From 92a20577940f5eef80dfc034a8b4df062d59c87c Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 19:30:46 -0300 Subject: [PATCH 01/17] fix(server): place rooms on adopted members, so a node cannot mint an id into a replica set (C25) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C13 gated replication, leadership and durability on membership in `replicas_for(room)`, and placement is HRW over the member set — a pure, publicly computable function — so a member could grind a node id that placed it on the room it wanted, introduce itself over gossip's join path, and reach all three gates from inside the replica set. Learning a member and placing rooms on it are now two admissions. The roster is what a node dials, probes and gossips about; the ring is built from the adopted members alone, and a gossip-learned member is pending until the cluster adopts it. Adoption cannot be a local predicate — placement must be identical on every node — so the evidence is disseminated instead: a node records only that it completed an identity-checked peer link to a member, that claim rides gossip attributed to the member the receiving link is bound to, and two already-adopted verifiers place a member. Verifier sets are grow-only, so adoption is a convergent fixpoint over them. --- ARCHITECTURE.md | 22 +- DECISIONS.md | 22 + KANBAN.md | 3 +- crates/core/src/lib.rs | 3 +- crates/core/src/protocol.rs | 45 +- crates/core/tests/protocol_gossip.rs | 87 ++- crates/server/src/gossip.rs | 70 ++- crates/server/src/membership.rs | 311 ++++++++-- crates/server/src/registry.rs | 78 ++- crates/server/src/runtime.rs | 31 +- crates/server/tests/adoption.rs | 810 +++++++++++++++++++++++++++ crates/server/tests/failover.rs | 2 +- crates/server/tests/gossip.rs | 119 ++-- crates/server/tests/peer_auth.rs | 3 + crates/server/tests/peer_identity.rs | 72 +-- crates/server/tests/peer_tls.rs | 1 + crates/server/tests/reaping.rs | 112 +++- 17 files changed, 1578 insertions(+), 213 deletions(-) create mode 100644 crates/server/tests/adoption.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9792db49..5351ac9c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -912,7 +912,7 @@ No Redis / Postgres dependencies. Cluster layer must be internal. ## Room-Based Sharding -Each room maps to a replica set. Consistent hashing on `room_id` for deterministic placement, horizontal scaling, balanced distribution. +Each room maps to a replica set. Consistent hashing on `room_id` for deterministic placement, horizontal scaling, balanced distribution. The set hashed over is the cluster's **adopted** members, not every node it has heard of (§Member Adoption). ## Leader Model @@ -954,11 +954,27 @@ The secret is a **bearer** credential, so the peer link carries it over an encry **Identity is declared, like transport.** A cluster cannot acquire per-node certificates at one instant, so an uncertified link is admitted at the id it claims — which binds one link to one identity and vouches for none — until `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` declares the rollout finished and refuses every certificate-less peer. A node that requires it while verifying no client certificate, while presenting no identity of its own, or while holding a member whose advertise address names no host, refuses to start: each would present as a cluster whose every peer link is refused. Two refusals do not wait for the policy. An advertise address that names no host is refused outright, this node's and any configured member's alike — no peer could dial it and no certificate could be bound to it — and a gossip-learned one is classified a *permanent* dial failure rather than redialed forever. That refusal lives in the address parse every consumer shares, so the host is read in one place. And a node that *verifies* client certificates while presenting one naming no host matching the id it dials under refuses to start whether or not identity is required, because that is the half of a *peer's* decision which is decidable locally: the peers apply this node's own rule to this node's own certificate. That condition is a **proxy** — this node's own listener standing in for whether its *peers* request a certificate — exact where the posture is uniform and wrong in both directions during a rolling change, so where the local evidence says no certificate is asked for the node warns rather than refusing, breaking no deployment that may be perfectly inert. Keeping the client-certificate posture uniform is therefore a deployment requirement rather than a recommendation: no local check can replace it, because the fact being predicted is on the other end of the link. A member's advertised *transport* is deliberately not among those preconditions — the scheme describes that member's own listener, while the link carrying its identity into this node is the one it dials, so a plaintext member still identifies itself; refusing it the cluster secret is `CRDTSYNC_CLUSTER_REQUIRE_TLS`'s separate declaration. Where certificates *are* configured they are decisive whether or not identity is required — a verified certificate that contradicts a link's claim, **or that names no host to support it**, is evidence rather than an opinion — so a cluster already running peer mTLS must have each node's certificate name that node's advertise host, and a node whose own certificate does not refuses to start even mid-rollout. Client-certificate verification in `request` mode is enough, so peer identity never becomes a certificate requirement for ordinary application clients. -**Three gates read the bound identity.** *Replication and leadership*: a `Replicate`/`ReplicateSnapshot` is honored only from a member that itself holds the room, since placement decides which members may ever lead it and one outside the replica set can hold no copy — so a member outside a room's replica set can neither push ops into it nor supersede its leader. The rejection drops the link, which is also the repair: two nodes' member sets differ for the propagation window of every join and reap, so a legitimate leader is transiently outside a room's replica set as a follower sees it, and *silently* discarding its frames would leave a permanent gap — the steady path mirrors only fresh commits, so nothing would re-send them. Dropping makes the leader redial, and the redial re-runs the late-joiner catch-up that closes the gap. *Membership growth*: an inbound `Gossip` may introduce only the node that sent it, **at its own id as its dial address**; a tuple naming an already-known member still merges whole, so a `Dead` verdict still travels, but an unknown member is adopted only from the node it names and only pointing at itself — constraining the id alone would leave the same channel open one field over, since the member set records the address a tuple carries and every node dials it. A joiner still converges — it learns the cluster from the seed it *dialed*, whose reply may introduce freely because the set a node dials is its own member set, and then introduces itself to each member directly. *Durability*: a `FollowerHeads` must name the member its link was admitted as, because a node is the only authority on what it durably holds; one member crediting another's watermark is what let majority-ack release a client `Accepted` for a write no majority held. +**Three gates read the bound identity.** *Replication and leadership*: a `Replicate`/`ReplicateSnapshot` is honored only from a member that itself holds the room, since placement decides which members may ever lead it and one outside the replica set can hold no copy — so a member outside a room's replica set can neither push ops into it nor supersede its leader. The rejection drops the link, which is also the repair: two nodes' member sets differ for the propagation window of every join and reap, so a legitimate leader is transiently outside a room's replica set as a follower sees it, and *silently* discarding its frames would leave a permanent gap — the steady path mirrors only fresh commits, so nothing would re-send them. Dropping makes the leader redial, and the redial re-runs the late-joiner catch-up that closes the gap. *Membership growth*: an inbound `Gossip` may introduce only the node that sent it, **at its own id as its dial address**; a tuple naming an already-known member still merges whole, so a `Dead` verdict still travels, but an unknown member is adopted only from the node it names and only pointing at itself — constraining the id alone would leave the same channel open one field over, since the member set records the address a tuple carries and every node dials it. A joiner still converges — it learns the cluster from the seed it *dialed*, whose reply may introduce freely because the set a node dials is its own member set, and then introduces itself to each member directly. What a self-introduction or a reply buys is a place on the *roster*, not in the ring: rooms are placed on adopted members (§Member Adoption). *Durability*: a `FollowerHeads` must name the member its link was admitted as, because a node is the only authority on what it durably holds; one member crediting another's watermark is what let majority-ack release a client `Accepted` for a write no majority held. **What identity cannot decide is which claim to leadership is legitimate.** Inside a room's replica set the epoch remains the only arbiter: a replica promoted over a leader it believes down must be able to supersede it, and no identity distinguishes that from a peer replica forging the bump. That is the election's question, answered by the HRW+epoch → Raft evolution (§Cluster Leadership), not by a stronger credential. -**Nor can identity alone decide who belongs in a room's replica set.** Placement is HRW over the member set — a pure, publicly computable function — so which rooms a node replicates follows from its node id, and the join path lets an unknown node introduce itself. A member can therefore *mint* an id that places it into a chosen room's replica set and reach the gates above from inside it. Identity bounds the mint space to the member's own certified host and stops it minting on anyone else's, which is why these gates hold against a member misreaching; against a *compromised* member they need placement computed over members each node has itself verified. That cannot be a local trust filter — placement must be identical on every node or the ring diverges and the cluster splits — so it is a cluster-wide change, and the prerequisite it is. +**Nor does identity alone decide who belongs in a room's replica set.** Placement is HRW over the member set — a pure, publicly computable function — so which rooms a node replicates follows from its node id, and the join path lets an unknown node introduce itself. A member could therefore *mint* an id that placed it into a chosen room's replica set and reach the gates above from inside it. Identity bounds the mint space to the member's own certified host and stops it minting on anyone else's; what closes it is that rooms are placed on *adopted* members only (§Member Adoption), and introducing yourself does not adopt you. + +## Member Adoption + +**Learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the placement ring is built from the **adopted** members alone. A member learned by gossip is *pending* — reachable and converging, but in no room's replica set and no room's quorum — until the cluster adopts it. Pending is not exile, and cannot be: a member nobody dials could never be verified, so a genuine joiner would never join. What it removes is the step from "a node said this id exists" to "rooms live on that id", which is the whole of the mint. + +**Adoption is the cluster's decision, never one node's.** Placement must be identical on every node or the ring diverges and the cluster splits, so "verified" cannot be a local predicate over a shared member set — two nodes would place the same room differently forever. So the *evidence* is disseminated instead of the verdict: a node records only what it knows first-hand — that it has completed an identity-checked peer link to a member, its dial having authenticated the acceptor against the address it dialed, or the member having dialed in behind a certificate that names it — and that claim rides the same anti-entropy that carries liveness, as one flag per member on the gossip tuple. A member is placed once **two** already-adopted members have verified it. The evidence is a grow-only set of verifiers, so it merges by union and converges however rounds interleave, and adoption is a pure function of it. + +**A claim is worth exactly the link that carried it.** A `verified` flag is recorded against the member the receiving link is bound to (§Peer Identity) and against nobody the payload names, so a member can assert its own verifications and no one else's; relaying another node's would make one member's word enough to place any id it liked. A tuple claiming the sender itself is dropped — a node's place in the ring is never its own to assert — and a claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. The cost of first-hand-only is that verifications travel by direct exchange rather than by relay: a joiner is placed once enough members have each reached it, which randomized peer selection delivers within a few gossip rounds rather than one. + +**Two members are the bar because one member is the attacker.** A single compromised member would otherwise vouch for the id it ground and place it itself; requiring a second means an honest member must have independently reached that id, which also keeps an unreachable id — one that would pollute placement and stall every quorum it landed in — out of the ring entirely. The bar clamps to the number of members that could possibly vouch, so a cluster too small to raise two can still grow. **Configured members are adopted from birth**: the operator's config is the root of trust a cluster starts from, and there is no earlier authority for it to be vouched for by. + +**A member is verified at the host its own id names.** A verification is admissible only where the member's advertise address and its node id name the same host — otherwise it would be verified at one host and dialed at another, and no link to either would vouch for it. A node id *is* an advertise address, so that costs a legitimate member nothing; what it closes is the gossip *reply* half, which adopts a member at whatever address its tuple carries because it comes from a node this one chose to dial. Such a member stays pending permanently rather than transiently. + +**Adoption is only as strong as the identity beneath it.** Under peer mTLS a verification means a certificate for that member's host answered there, so the mint space stays bounded by what §Peer Identity bounds it to — the member's own host, which is already a shared trust unit. With no certificates configured a verification means only that *something* answers at the address the id names, which still requires the minted id to be reachable and still denies a single member the power to place it, but vouches for no identity; `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` is what makes it vouch. For the same reason an *inbound* link verifies the member it names only when a certificate named it: an uncertified inbound claim is a member asserting an id rather than answering at one, and the deployment-wide secret makes that free. And under that policy an outbound dial to a **plaintext** member verifies nothing either — there was no certificate to check, and adopting on it would give rooms to a member whose every inbound link the cluster refuses, stalling their quorums. That reads a member's advertised transport, which is sound here and is not sound inbound: an outbound dial runs over exactly the transport that address declares, while an inbound link is one the member dialed and its own listener's scheme describes nothing about it. + +**Reaping takes adoption and vouches with it.** A member removed from the roster leaves the adopted set, its verifier set is dropped, and it is struck from every other member's — a departed node keeps vouching for nobody. A return is a fresh join and is verified again. ## Peer Transport diff --git a/DECISIONS.md b/DECISIONS.md index 228b2847..dcab5163 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -7,6 +7,28 @@ Log of design changes to [ARCHITECTURE.md](ARCHITECTURE.md) that implementation The entries below (2026-07-02) are a backfill: design changes made during the v0.1→v0.2 build that predate this log, recovered from the sessions and commit history. +## 2026-07-28 · C25 member adoption (#PR) · rooms are placed on the members the cluster has *verified*, and a node introducing itself joins the roster rather than the ring + +**Changed:** ARCHITECTURE gains §Member Adoption; §Peer Identity's "nor can identity alone decide who belongs in a room's replica set" paragraph is replaced by a pointer to it, §Room-Based Sharding says which member set is hashed over, and §Peer Identity's membership-growth clause says what a self-introduction and a gossip reply actually buy. + +**The defect.** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so which rooms a node replicates follows from its node id. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id that HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. C13 pinned this as a passing test; this unit inverts it. Two lesser ways in had the same shape: the gossip *reply* half adopts a member at whatever address its tuple carries, and a minted member polluted placement and quorum for the rooms it did **not** attack, so writes to them waited on acks that never came. + +**Why a local check was not available.** The obvious fix — place rooms only on members this node has itself verified — splits the cluster. Placement must be identical on every node or two nodes disagree about who leads a room permanently rather than for a propagation window, and "verified" is per-node by construction: each node has completed a different set of links, so the predicate never converges. The ring has to be a function of state that *does* converge. + +**So the evidence is disseminated, not the verdict.** A node records only what it knows first-hand — it completed an identity-checked peer link to a member — and that claim rides the same anti-entropy that already carries liveness, as one `verified` flag per tuple on `Message::Gossip`. A receiver records the flag **against the member its link is bound to** and against nobody the payload names, so a member can assert its own verifications and no one else's. A member is adopted once `ADOPTION_VERIFIERS = 2` already-adopted members have verified it. The evidence is a grow-only set of verifiers per member, so it merges by union and adoption is a pure function of it — convergent however rounds interleave, and computed to a fixpoint so adopting a member makes its own claims count without the outcome depending on the order members were visited in. + +**Two, not one, and not a majority.** One is the attacker: a single compromised member would vouch for the id it ground and place it itself. Two forces an honest member to have independently reached that id, which also keeps an *unreachable* minted id — the one that pollutes placement and stalls every quorum it lands in — out of the ring entirely. A majority was rejected: it buys nothing here, because an honest node dials every member it knows anyway, so the marginal verifier past the second is a formality that only delays a genuine joiner. The bar clamps to the number of members that could vouch, so a cluster smaller than it can still grow. Configured members are adopted from birth — the operator's config is the root of trust, and a cluster whose own seeds were pending could place no room at all. + +**Relaying verifications was rejected.** It would let adoption spread in one hop instead of by direct exchange, and it is exactly what a compromised member wants: with no signatures on the wire, "A and B verified X" is free to write. First-hand-only costs a joiner a few extra gossip rounds — randomized peer selection reaches every member — and is the only form of the claim that a receiver can attribute. + +**A verification is admissible only where a member's advertise address and its node id name the same host.** Otherwise the member is verified at one host and dialed at another and no link to either vouches for it. A node id *is* an advertise address, so this costs a legitimate member nothing; what it closes, permanently rather than transiently, is the reply half introducing an id that points somewhere else. + +**An inbound link verifies only behind a certificate.** A dial proves something answers at the address the id names; an inbound claim proves only that the sender reached here, and the cluster secret is deployment-wide, so treating an uncertified inbound claim as verification would let any secret-holder vouch for any id at no cost. Under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` an outbound dial to a **plaintext** member verifies nothing either, for the same reason: nothing was checked, and adopting on it hands rooms to a member whose every inbound link that same policy refuses, stalling their quorums. That gate reads the member's advertised transport, which C13's pass 3 rejected for the *inbound* direction and which is sound here — an outbound dial runs over exactly the transport the address declares, while an inbound link is one the member dialed and its own listener's scheme describes nothing about it. The outbound direction still vouches without certificates, which is the honest floor: with no certificates configured, adoption means the minted id must be reachable and no single member may place it, and nothing about identity. `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` is what makes it mean identity, and under it the mint space stays what §Peer Identity bounds it to — the member's own host, already a shared trust unit. + +**Pending is not exile.** A member the cluster has not adopted stays on the roster: dialed, probed, gossiped about and advertised back. It has to be, or nobody would ever reach it and no genuine joiner would ever join. `Membership::members` is therefore the roster and `adopted_members` the ring; the split is the whole change, and every gate that reads `replicas_for` inherited it without a line of its own. + +**Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. + ## 2026-07-28 · C26 refused-op signal (#370) · the SDK fold answers two counts, because at the seam with no server the applied count alone renders a permanent refusal as silence **Changed:** ARCHITECTURE's §SDK-Ergonomic-Surface gains the fold's report — `applyUpdate` and its per-language equivalents answer how many ops applied as they arrived *beside* how many no replica will ever hold — and that report joins the language-agnostic contract every SDK realizes. diff --git a/KANBAN.md b/KANBAN.md index ba12e22d..6ed5d378 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,6 +30,8 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#PR).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once `ADOPTION_VERIFIERS = 2` already-adopted members have verified it; the verifier sets are grow-only, so they merge by union and adoption is a pure fixpoint over them — convergent however rounds interleave, order-independent, pinned as such. **Two, not one:** one voucher is the attacker itself, and requiring a second forces an honest member to have independently *reached* the minted id, which also keeps an unreachable one — the shape that pollutes placement and stalls every quorum it lands in, the unit's second lesser way in — out of the ring entirely. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a genuine joiner); relaying another node's verifications was rejected (with no signatures on the wire, "A and B verified X" is free to write, which is the mint again). The bar clamps to the members that could vouch, so a small cluster still grows, and **configured members are adopted from birth** — the operator's config is the root of trust. **Three narrower rules close the rest.** A tuple claiming the sender itself is dropped, so no node can vouch for its own place. A claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. And a verification is admissible only where a member's advertise address and its node id name the same **host** — otherwise it is verified at one host and dialed at another — which closes the first lesser way in (the gossip *reply* half adopts a member at whatever address its tuple carries) permanently rather than transiently. **An inbound link verifies only behind a certificate that names the member**, since an uncertified inbound claim is a member asserting an id rather than answering at one and the cluster secret is deployment-wide; the outbound *direct* gossip round vouches either way — except that under that policy a **plaintext** member is not verified by a dial either (nothing was checked, and adopting on it hands rooms to a member whose every inbound link the same policy refuses), a gate that reads the member's advertised transport, which is sound outbound and is exactly what C13's pass 3 rejected inbound. That round is the honest floor — with no certificates configured, adoption means the minted id must be reachable and that no single member may place it, and nothing about identity, while `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` makes it mean identity and keeps the mint space bounded to the member's own host (already a shared trust unit per C13). An *indirect* (ping-req) confirmation is liveness only and vouches for nothing. Reaping takes a member's adoption and its vouches with it. C13's residual test is inverted and moved: `crates/server/tests/adoption.rs` (19 cases), covering the mint against all three gates, the threshold, the self-vouch, the pending-voucher, the host-coherence rule, both link directions, order-independence, reaping, and that a pending member is still dialed, probed and advertised. Wire: `Message::Gossip`'s tuple gains a trailing flag byte. See DECISIONS (2026-07-28). → *Cluster*. + **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. **C15 — `VersionFetch` served a captured room state with neither projection applied (crates/server) — DONE (#368).** `Message::VersionFetch` answered with `hub.version_state` verbatim, gated only on the deployment/schema-tier room `Read` that `version_room` resolves, while every other seam that hands a client a state blob runs `project_read_paths` then `project_zones` first. A version is the same bytes at an earlier sequence, so a reader that passed the room gate fetched the whole room out of any named version — withheld zones, doc-ACL-denied subtrees, and the full causal frontier naming every other replica's ops in the partitions it may not read. The fetch now runs the Subscribe composition in the same order over the version's own state, with two inputs taken from the channel rather than re-derived: its own `Subscription.zones` — the very set the live fan-out filters that channel's ops by, so the fetch narrows to the partitions the stream carries rather than to the actor's whole entitlement — and the channel-derived recipient C9 threads, so a version a reader adopts does not re-mint. **Element-scoped grants resolve against the version's tree, live records still govern.** Authorization is a decision about now; an element's redaction *path* is where it stood at capture. Resolving against the live room leaves the deny inert the moment the element leaves it — and an inert deny is not a narrower read, it is the whole version. **An unprojectable version is refused, not served:** version bytes come back off durable storage, so an undecodable one is reachable in a way a just-materialized snapshot is not, and the raw bytes still carry what a redaction would have cut; `zone_narrowing` is now the single home of "would the zone projection narrow at all", shared by the projection and that guard, and it decides *whole-zone* as "the set is exactly the declared id range" rather than "the set has as many members" — a set is resolved once, when a channel subscribes, and the room's governing schema lifts underneath it, so a set can both miss a declared partition and name ids no longer declared, and a count says nothing about which of the two it is (C30 is the cause). The refusal is non-closing, and it asks what a redaction is *configured over* for these bytes on this channel — the room holds a tuple, the channel holds a partial zone set — not what would actually have been cut: that answer needs the element index, the index needs the decode, and the decode is what failed. **Both seams now share one `project_served_state`**, rather than the fetch copying the snapshot arm's composition — duplicating it would have re-created, inside this unit, the shape the unit exists to answer. The *gate* is deliberately not widened to Subscribe's subtree-scoped admission — narrowing what a reader is served is this unit, widening who may fetch is not. Spec `crates/server/tests/version_projection.rs` (16) + `version_unreadable.rs` (5, non-Miri: it damages a durable versions file). Mutation-swept against an isolated copy of the committed tree, unmutated baseline 0: dropping either projection, `reads_all = true`, `recipient: None`, `session.client` instead of `for_channel`, the live element index, a narrowability test that ignores either half, serving an undecodable version, `zone_narrowing` always/never narrowing, `zone_narrowing` back to either half of its rule alone (the cardinality proxy, or containment without it), and a zone scope re-derived whole-room at fetch time instead of the channel's own each fail a named test — each anchored on the shared composition, so no mutation can land on a sibling seam and read as covered. @@ -423,7 +425,6 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C16 — the reveal back-fill's batch dedup drops the copy that was zone-protected (crates/server) — READY, no dependencies. Found by cold review during C11 (#359), reasoned not reproduced.** The live fan-out's reveal prefix force-stamps every shell and back-filled op with the placing move's zone, precisely so the per-channel `zone_filter` cannot drop the content while keeping the shell. C11 made the back-fill skip any op the batch already carries — correct, since the duplicate took that op out of its transaction's count — but the surviving copy is then the *batch's*, which carries the zone its author computed at emit time. `zone_of_op` resolves against the live tree, so in one `atomic_transact` an edit inside a node carries the node's origin zone while the move carries the destination's: the per-channel filter can now drop the content and keep the shell, leaving the reader with a materialised node it never fills. Reaching it needs a node crossing zones, which a named-branch write does un-tokened (`handle_ops` gates `batch_zone_crossings` on `branch == MAIN_BRANCH`). Wants either the batch copy stamped with the move's zone the way the prefix copy was, or the zone filter taught to keep a shell and its content together by node rather than by zone. The `registry.rs` comment claiming the co-travel guarantee overstates it until then. → *Server / Fan-out*. -**C25 — a member places itself into any room's replica set by minting a node id, so C13's leadership and durability gates do not hold against a compromised member (crates/server) — READY, no dependencies. Found during C13 (#365) and filed there as "C17", renumbered here because that id was already taken by the lamport-mint unit filed in #364; a prerequisite for C13's guarantees rather than a refinement of them.** C13 binds a peer link to a member and gates replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself. So an admitted member grinds an id that HRW places on the room it wants, introduces itself under it, and is inside that room's replica set: it can supersede the leader with a forged epoch, push ops in, and report heads that make majority-ack release a client `Accepted` for a write no majority holds. Peer identity bounds the mint space — a certified member can only mint on its own host, and cannot mint on anyone else's — but does not close it; with no certificates configured the binding vouches for nothing anyway, and a secret-holder can simply claim an existing member's id. Two lesser ways in have the same shape: the *reply* half of a gossip round adopts members freely (it comes from a node this one chose to dial), and a minted member also pollutes placement and quorum for rooms it does not attack, so writes to them wait on acks that never come. **The fix is placement over *verified* members**: a gossip-learned member is held pending — dialable and gossipable, but in no room's replica set and no room's quorum — until this node has itself completed an identity-checked peer link to it (its dial verifying the acceptor's certificate against the address, or the member dialing in with a certificate that names it). The hard part is not the check but the convergence: **placement must be identical on every node**, so "verified" cannot be a per-node predicate over the same member set or the ring diverges and the cluster splits. Adoption has to be a decision the cluster agrees on — a member is placeable once *some* threshold of members have verified it, disseminated through the same anti-entropy that already carries liveness, with the same incarnation/merge rules making it convergent. Needs a pending set in `Membership`, a promotion signal from the dial tasks, an adoption field on the gossip tuple, and placement built from adopted members only. C13 pins the residual as a passing test (`a_member_that_mints_a_node_id_places_itself_into_a_rooms_replica_set`), which this unit inverts. → *Cluster*. **C14 — a redacted op delta leaves the same re-mint hole, and no frame can carry the ids it withholds (crates/server + crates/core) — READY, no dependencies. Found during C9 (#357), reproduced.** C9 closed the *snapshot* seam: a projected snapshot now names the recipient's own ids, so adoption never frees one the room's log holds. An **uncompacted** room serves an op delta instead, and the per-op read filter withholds the recipient's own ops on paths it may no longer read — leaving a hole in its own run with nothing to repair it from. Measured on C9's own fixture: a reader with write at `/` and read on `/a` alone writes into `/b` (seqs 0,1) then `/a` (seq 2); it restarts, its delta withholds `/b`, `next_seq` reports 0, and its next write mints seq 0 straight onto its own withheld id — the write never lands. The state encoding was the carrier for the snapshot case; an `Ops` frame has no equivalent, so closing this needs one: a frontier field on the delta, or a rule that a reader's *own* ops are never withheld from it (which trades the hole for materialising ids in a subtree it may not read — the reason C9 did not simply stop scrubbing). Note the second shape also has to answer what an op carries beyond its id: its target and content may name denied structure. → *Server / Fan-out*. diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index e5c806ef..36e7f879 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -63,7 +63,8 @@ pub use marks::{MarkState, ResolvedMark}; pub use op::{destrand_split, split_groups, Op, OpId, OpKind, Tx, TxId, MAX_TX_MEMBERS}; pub use protocol::{ decode_header, decode_message, encode_header, encode_message, select_codec, BranchInfo, - Channel, DiffKind, ErrorCode, MemberState, Message, ProtocolError, CODEC_V1, SUPPORTED_CODECS, + Channel, DiffKind, ErrorCode, MemberAdvert, MemberState, Message, ProtocolError, CODEC_V1, + SUPPORTED_CODECS, }; pub use ranged::{RangeAnchor, RangedElement, RangedInit, RangedPayload}; pub use register::Register; diff --git a/crates/core/src/protocol.rs b/crates/core/src/protocol.rs index dfcb65d4..14d38595 100644 --- a/crates/core/src/protocol.rs +++ b/crates/core/src/protocol.rs @@ -131,6 +131,13 @@ pub enum MemberState { Dead, } +/// One cluster member as a [`Message::Gossip`] frame advertises it: its node id, the +/// advertise address a peer dials it at, its incarnation, its [`MemberState`], and +/// whether the **sender of that frame** has itself completed an identity-checked peer +/// link to it. The last is a first-hand claim about the sender, so a receiver +/// attributes it to the member its link is bound to and to nobody the payload names. +pub type MemberAdvert = (Vec, Vec, u64, MemberState, bool); + /// A closed set of failure reasons the server reports to a client. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum ErrorCode { @@ -415,18 +422,26 @@ pub enum Message { }, /// A node's advertisement of the cluster members it knows, for gossip /// membership discovery and SWIM-style failure detection: `members` is a set of - /// `(node_id, advertise_addr, incarnation, state)` tuples — the node id a peer - /// places with, the address it dials to reach that member, a monotonic per-node - /// refutation counter, and the member's [`MemberState`]. A receiver merges each - /// tuple into its own liveness view (anti-entropy: a higher incarnation wins, - /// and at equal incarnation the more-suspicious state wins), so a node that - /// boots knowing only a seed peer learns the whole cluster — and a node's - /// failure propagates to every node — within a few gossip rounds. Node-to-node - /// — never a client frame; a client that sends one commits a protocol - /// violation. - Gossip { - members: Vec<(Vec, Vec, u64, MemberState)>, - }, + /// `(node_id, advertise_addr, incarnation, state, verified)` tuples — the node id + /// a peer places with, the address it dials to reach that member, a monotonic + /// per-node refutation counter, the member's [`MemberState`], and whether the + /// *sender of this frame* has itself completed an identity-checked peer link to + /// that member. A receiver merges each tuple into its own liveness view + /// (anti-entropy: a higher incarnation wins, and at equal incarnation the + /// more-suspicious state wins), so a node that boots knowing only a seed peer + /// learns the whole cluster — and a node's failure propagates to every node — + /// within a few gossip rounds. + /// + /// `verified` is a **first-hand claim about the sender itself**, not a relayed + /// fact: it says "I have verified this member", so a receiver attributes it to + /// the member its link is bound to and to nobody else. That is what makes it + /// safe to act on — a member can assert only its own verifications, never + /// another's — and it is the evidence a room's replica set is placed over, so + /// that a node cannot mint an id into the placement ring by introducing itself. + /// + /// Node-to-node — never a client frame; a client that sends one commits a + /// protocol violation. + Gossip { members: Vec }, /// Requests the branches of `room` — the app-facing enumeration a client runs /// to discover a room's forks and published targets before subscribing one. /// Room-keyed rather than channel-keyed: branch management is a room-level @@ -848,11 +863,12 @@ pub fn encode_message(m: &Message) -> Vec { &mut out, u32::try_from(members.len()).expect("member count exceeds u32"), ); - for (node, addr, incarnation, state) in members { + for (node, addr, incarnation, state, verified) in members { put_bytes(&mut out, node); put_bytes(&mut out, addr); put_u64(&mut out, *incarnation); put_u8(&mut out, member_state_tag(*state)); + put_u8(&mut out, u8::from(*verified)); } } Message::BranchList { room } => { @@ -1249,7 +1265,8 @@ pub fn decode_message(bytes: &[u8]) -> Result { let addr = cur.bytes()?; let incarnation = cur.u64()?; let state = member_state(cur.u8()?)?; - members.push((node, addr, incarnation, state)); + let verified = cur.u8()? != 0; + members.push((node, addr, incarnation, state, verified)); } Message::Gossip { members } } diff --git a/crates/core/tests/protocol_gossip.rs b/crates/core/tests/protocol_gossip.rs index 0b27202a..c1506805 100644 --- a/crates/core/tests/protocol_gossip.rs +++ b/crates/core/tests/protocol_gossip.rs @@ -2,14 +2,16 @@ //! each with its liveness, for anti-entropy failure detection. //! //! `Gossip { members }` carries a length-prefixed list of `(node_id, -//! advertise_addr, incarnation, state)` tuples: the id a peer places with, the -//! address it dials, a monotonic per-node refutation counter, and the member's -//! SWIM state (`Alive` / `Suspect` / `Dead`). A receiver merges each tuple into -//! its own liveness view — a higher incarnation wins, and at equal incarnation a -//! more-suspicious state wins — so both the member set and its liveness converge -//! cluster-wide. Node-to-node, never a client frame. Decoding is total — a -//! truncation, a trailing byte, or a bad state tag is a `ProtocolError`, never a -//! panic. +//! advertise_addr, incarnation, state, verified)` tuples: the id a peer places +//! with, the address it dials, a monotonic per-node refutation counter, the +//! member's SWIM state (`Alive` / `Suspect` / `Dead`), and whether the sender has +//! itself completed an identity-checked peer link to that member. A receiver +//! merges each tuple into its own liveness view — a higher incarnation wins, and at +//! equal incarnation a more-suspicious state wins — so both the member set and its +//! liveness converge cluster-wide; the verification flag is recorded against the +//! sender, and enough of them place the member in the ring. Node-to-node, never a +//! client frame. Decoding is total — a truncation, a trailing byte, or a bad state +//! tag is a `ProtocolError`, never a panic. use crdtsync_core::protocol::{ decode_message, encode_message, MemberState, Message, ProtocolError, @@ -26,8 +28,18 @@ fn member( addr: &[u8], incarnation: u64, state: MemberState, -) -> (Vec, Vec, u64, MemberState) { - (node.to_vec(), addr.to_vec(), incarnation, state) +) -> (Vec, Vec, u64, MemberState, bool) { + (node.to_vec(), addr.to_vec(), incarnation, state, false) +} + +/// The same advertisement, carrying the sender's claim to have verified the member. +fn verified_member( + node: &[u8], + addr: &[u8], + incarnation: u64, + state: MemberState, +) -> (Vec, Vec, u64, MemberState, bool) { + (node.to_vec(), addr.to_vec(), incarnation, state, true) } #[test] @@ -80,6 +92,7 @@ fn gossip_round_trips_binary_and_empty_fields() { vec![0xFF, 0x00, 0x80, 0x7F], 999, MemberState::Dead, + false, ), ], }); @@ -131,12 +144,14 @@ fn a_gossip_count_past_the_payload_is_an_error() { #[test] fn a_bad_state_tag_is_an_error_not_a_panic() { // The state is a single tag byte; an unknown value is a decode error, never a - // silent misread or a panic. Build a one-member frame and clobber its trailing - // state byte with an out-of-range tag. + // silent misread or a panic. Build a one-member frame and clobber its state byte + // — the last but one, the verification flag trailing it — with an out-of-range + // tag. let mut bytes = encode_message(&Message::Gossip { members: vec![member(b"node-a", b"10.0.0.1:9000", 0, MemberState::Alive)], }); - *bytes.last_mut().unwrap() = 200; + let state_byte = bytes.len() - 2; + bytes[state_byte] = 200; assert!(matches!( decode_message(&bytes), Err(ProtocolError::BadTag { .. }) @@ -145,12 +160,12 @@ fn a_bad_state_tag_is_an_error_not_a_panic() { #[test] fn each_state_tag_is_distinct() { - // The three states encode to three distinct trailing tag bytes. + // The three states encode to three distinct tag bytes. let tag_of = |state| { let bytes = encode_message(&Message::Gossip { members: vec![member(b"n", b"a", 0, state)], }); - *bytes.last().unwrap() + bytes[bytes.len() - 2] }; let alive = tag_of(MemberState::Alive); let suspect = tag_of(MemberState::Suspect); @@ -160,6 +175,48 @@ fn each_state_tag_is_distinct() { assert_ne!(alive, dead); } +#[test] +fn gossip_round_trips_a_verification_claim() { + // The sender's claim to have verified a member rides per tuple and survives the + // round trip independently of the liveness beside it — a frame mixing verified + // and unverified members comes back as it went. + round_trips(Message::Gossip { + members: vec![ + verified_member(b"node-a", b"10.0.0.1:9000", 4, MemberState::Alive), + member(b"node-b", b"10.0.0.2:9000", 4, MemberState::Suspect), + ], + }); +} + +#[test] +fn a_verification_flag_is_a_distinct_trailing_byte() { + // Verified and unverified encode differently, in the byte after the state tag — + // so a decoder cannot read the claim off the state, nor drop it silently. + let flag_of = |m| { + let bytes = encode_message(&Message::Gossip { members: vec![m] }); + *bytes.last().unwrap() + }; + assert_ne!( + flag_of(member(b"n", b"a", 0, MemberState::Alive)), + flag_of(verified_member(b"n", b"a", 0, MemberState::Alive)), + ); +} + +#[test] +fn a_truncated_verification_flag_is_an_error_not_a_panic() { + // The flag is the last byte of a tuple, so a frame cut just before it is a + // truncation inside a field — an error, never a silently unverified member. + let bytes = encode_message(&Message::Gossip { + members: vec![verified_member( + b"node-a", + b"10.0.0.1:9000", + 0, + MemberState::Alive, + )], + }); + assert!(decode_message(&bytes[..bytes.len() - 1]).is_err()); +} + #[test] fn the_gossip_tag_is_distinct() { // Tag 26 is the gossip frame; the frame's first byte is that tag. diff --git a/crates/server/src/gossip.rs b/crates/server/src/gossip.rs index 420b9798..e2852012 100644 --- a/crates/server/src/gossip.rs +++ b/crates/server/src/gossip.rs @@ -4,12 +4,22 @@ //! A node need not know the whole cluster at boot: it seeds from one (or a few) //! peer addresses and learns the rest by gossip. Each round a node picks a random //! known peer and exchanges liveness with it — a push-pull [`Message::Gossip`] -//! carrying `(node_id, advertise_addr, incarnation, state)` tuples. The exchange is -//! a SWIM anti-entropy merge ([`Membership::merge_liveness`]): both sides come away -//! holding every member either knew and the more-authoritative liveness for each, -//! so a node that boots knowing only a seed converges on the full set within a few -//! rounds. Placement (rendezvous/HRW) is order-independent, so once two nodes have -//! learned the same members they place every room identically. +//! carrying `(node_id, advertise_addr, incarnation, state, verified)` tuples. The +//! exchange is a SWIM anti-entropy merge ([`Membership::merge_liveness`]): both +//! sides come away holding every member either knew and the more-authoritative +//! liveness for each, so a node that boots knowing only a seed converges on the full +//! set within a few rounds. Placement (rendezvous/HRW) is order-independent, so once +//! two nodes have learned the same members they place every room identically. +//! +//! The exchange carries **adoption** as well as liveness. Rooms are placed on the +//! members the cluster has adopted, not on every node it has heard of, so the +//! `verified` flag on each tuple is the sender's own claim to have completed an +//! identity-checked peer link to that member. A *direct* round — this node dialed the +//! peer, and on a TLS member the transport authenticated it before a byte was written +//! — is what produces such a claim here; an indirect (ping-req) confirmation is +//! liveness only, since a relay reaching a target says nothing about who this node +//! would reach. Enough claims from adopted members place a member in the ring; see +//! [`Membership`]. //! //! Failure detection rides the same exchange (no separate detector socket): a //! *successful* round with a peer is first-hand proof it is alive @@ -56,12 +66,13 @@ use crate::membership::Membership; use crate::placement::NodeId; /// One member's advertisement as the membership layer holds it: its [`NodeId`], -/// dial address, incarnation, and liveness state. -pub type GossipMember = (NodeId, Vec, u64, MemberState); +/// dial address, incarnation, liveness state, and whether the advertising node has +/// itself verified it. +pub type GossipMember = (NodeId, Vec, u64, MemberState, bool); /// One member's advertisement as it rides the wire — the [`NodeId`] is raw bytes, /// matching [`Message::Gossip`]'s payload. The receiver reconstitutes the id. -pub type GossipWireMember = (Vec, Vec, u64, MemberState); +pub type GossipWireMember = crdtsync_core::MemberAdvert; /// How often a node initiates a gossip round. Frequent enough that a fresh joiner /// converges within a few seconds, sparse enough that a steady-state cluster's @@ -88,22 +99,32 @@ pub fn gossip_frame(members: &[GossipMember]) -> Message { Message::Gossip { members: members .iter() - .map(|(node, addr, incarnation, state)| { - (node.as_bytes().to_vec(), addr.clone(), *incarnation, *state) + .map(|(node, addr, incarnation, state, verified)| { + ( + node.as_bytes().to_vec(), + addr.clone(), + *incarnation, + *state, + *verified, + ) }) .collect(), } } -/// Merge a gossiped liveness payload (as it arrives off the wire) into -/// `membership` — the SWIM anti-entropy merge. Higher incarnation wins, equal -/// incarnation the more-suspicious state wins, and a stale suspicion of self is -/// refuted; see [`Membership::merge_liveness`]. -pub fn merge_into(membership: &mut Membership, payload: Vec<(Vec, Vec, u64, MemberState)>) { +/// Merge a gossiped liveness payload from the member `sender` (as it arrives off the +/// wire) into `membership` — the SWIM anti-entropy merge. Higher incarnation wins, +/// equal incarnation the more-suspicious state wins, a stale suspicion of self is +/// refuted, and each `verified` flag is recorded as `sender`'s own claim; see +/// [`Membership::merge_liveness`]. +pub fn merge_into(membership: &mut Membership, sender: &NodeId, payload: Vec) { membership.merge_liveness( + sender, payload .into_iter() - .map(|(node, addr, incarnation, state)| (NodeId::from(node), addr, incarnation, state)), + .map(|(node, addr, incarnation, state, verified)| { + (NodeId::from(node), addr, incarnation, state, verified) + }), ); } @@ -119,11 +140,18 @@ pub fn merge_into(membership: &mut Membership, payload: Vec<(Vec, Vec, u /// only that the sender reached *here*, and the reverse path is what this node's own /// probe measures. The one-way partition it would otherwise tolerate is already /// covered by SWIM indirect probing. +/// +/// **Only the `initiator` verifies the peer**, for the same reason: it is the side +/// that dialed, so it is the side whose transport authenticated the far end against +/// the address it dialed. Being dialed proves the sender reached here and nothing +/// about who answers at the sender's own address. pub fn exchange(initiator: &mut Membership, peer: &mut Membership) { let peer_id = peer.self_id().clone(); - peer.merge_liveness(initiator.known_liveness()); - initiator.merge_liveness(peer.known_liveness()); + let initiator_id = initiator.self_id().clone(); + peer.merge_liveness(&initiator_id, initiator.known_liveness()); + initiator.merge_liveness(&peer_id, peer.known_liveness()); initiator.note_gossip_reachable(&peer_id); + initiator.note_verified(&peer_id); } /// Pick a random peer to gossip to from `members`, excluding `self_id`, returning @@ -162,7 +190,7 @@ pub fn choose_relays( ) -> Vec<(NodeId, Vec)> { let mut eligible: Vec<(NodeId, Vec)> = members .iter() - .filter(|(node, _, _, state)| { + .filter(|(node, _, _, state, _)| { node != self_id && node != target && *state != MemberState::Dead }) .map(|(node, addr, ..)| (node.clone(), addr.clone())) @@ -240,7 +268,7 @@ pub async fn gossip_exchange( server: ClientId, dialer: &PeerDialer, frame: Message, -) -> Option, Vec, u64, MemberState)>> { +) -> Option> { relay_roundtrip(addr, server, dialer, frame, GOSSIP_TIMEOUT, |m| match m { Message::Gossip { members } => Some(members), _ => None, diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index b26708ac..614b3ecc 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -35,11 +35,35 @@ //! first live replica in HRW order, so a dead placement primary's rooms promote to //! the next live replica rather than stranding, and a refuted or recovered node //! reclaims them. +//! +//! Learning a member and *placing* rooms on it are two different admissions. The +//! roster above is what a node dials, probes and gossips; the [`Cluster`] placement +//! is built from the **adopted** members only, and a member learned by gossip is +//! held *pending* until the cluster has vouched for it. Placement is HRW over the +//! member set — a pure and publicly computable function — so which rooms a node +//! replicates follows from its node id, and an unchecked join path would let a node +//! mint an id that places it into any room's replica set. Pending keeps such a node +//! dialable and gossipable (so it converges, and so a genuine joiner is reached) +//! while it holds no room and counts toward no room's quorum. +//! +//! Adoption is a decision the *cluster* makes, never one node: placement must be +//! identical everywhere or the ring splits, so it cannot be a local predicate over a +//! shared member set. A node records only what it knows first-hand — that it has +//! completed an identity-checked peer link to a member ([`note_verified`]) — and +//! that claim rides gossip attributed to the node that made it. A member is adopted +//! once [`ADOPTION_VERIFIERS`] *already-adopted* members have verified it, so the +//! evidence is a grow-only set that merges the same way liveness does and converges +//! on the same anti-entropy. A member never verifies itself, so no node can place +//! itself; the members a node was *configured* with are adopted from birth, since +//! the operator's config is the root of trust the cluster starts from. +//! +//! [`note_verified`]: Membership::note_verified use std::collections::{HashMap, HashSet}; use crdtsync_core::MemberState; +use crate::dial::member_host; use crate::placement::{Cluster, NodeId}; /// Consecutive failed direct gossip probes to a member before this node escalates @@ -105,6 +129,17 @@ impl MemberLiveness { /// room, primary first. Clamps to the member count, so a small cluster resolves. pub const DEFAULT_REPLICATION_FACTOR: usize = 3; +/// How many adopted members must have completed an identity-checked peer link to a +/// gossip-learned member before it is adopted — placed on rooms and counted toward +/// their quorums. More than one, because the whole point is that no *single* member +/// can put a node of its choosing into the placement ring: one compromised member +/// grinding a node id needs an honest member to have independently reached that id +/// too. Small, because every honest member probes every member it knows on its own +/// gossip cadence, so a genuine joiner clears the bar within a few rounds. Clamped +/// to the adopted-member count ([`Membership::adoption_quorum`]) so a cluster too +/// small to raise this many verifiers can still grow. +pub const ADOPTION_VERIFIERS: usize = 2; + /// A malformed static membership configuration, surfaced at startup instead of a /// panic or a silently wrong member set. #[derive(Debug, Clone, PartialEq, Eq)] @@ -181,6 +216,23 @@ pub struct Membership { /// retention outlives any in-flight gossip that could reference the member, so the /// prune never resurrects it. reaped: HashMap, + /// The members rooms are actually placed on — the subset of the roster the + /// [`Cluster`] above is built from. `self` and every *configured* member are + /// adopted from birth; a member learned by gossip joins once + /// [`ADOPTION_VERIFIERS`] adopted members have verified it. Adoption is sticky + /// until the member is reaped: a ring that un-placed a member on evidence going + /// stale would move rooms off a node the cluster still holds. + adopted: HashSet, + /// Who has verified each member: for every member, the nodes that reported + /// completing an identity-checked peer link to it. A node inserts itself here + /// from its own links, and inserts a peer only from that peer's own gossip + /// (attributed to the member its link is bound to), so every entry is a + /// first-hand claim by the node named — a member can vouch for itself only, and + /// never *for* itself ([`note_verified`](Self::note_verified) refuses that). + /// Grow-only per member, so it merges by union and converges however gossip + /// interleaves. Claims by members that are not (yet) adopted are retained but do + /// not count, so a pending member cannot vouch another pending member in. + verifiers: HashMap>, } impl Membership { @@ -206,6 +258,10 @@ impl Membership { .iter() .map(|node| (node.clone(), MemberLiveness::new(0, MemberState::Alive))) .collect(); + // A configured member is adopted from birth: the operator's config is the + // root of trust a cluster starts from, and there is no earlier authority for + // it to be vouched for by. + let adopted: HashSet = members.iter().cloned().collect(); Self { self_id, cluster: Cluster::new(members), @@ -214,6 +270,8 @@ impl Membership { liveness, addrs, reaped: HashMap::new(), + adopted, + verifiers: HashMap::new(), } } @@ -249,8 +307,18 @@ impl Membership { &self.self_id } - /// The canonical (sorted, de-duplicated) member set, self included. - pub fn members(&self) -> &[NodeId] { + /// The canonical (sorted, de-duplicated) roster, self included — every member + /// this node knows, whether or not the cluster has adopted it. What a node + /// dials, probes and gossips; [`adopted_members`](Self::adopted_members) is the + /// narrower set rooms are placed on. + pub fn members(&self) -> Vec { + self.roster() + } + + /// The canonical (sorted, de-duplicated) *adopted* member set — the members + /// rooms are placed on and counted toward quorum. Equal to the roster in a + /// cluster with no pending joiners. + pub fn adopted_members(&self) -> &[NodeId] { self.cluster.nodes() } @@ -260,15 +328,13 @@ impl Membership { self.add_members(std::iter::once((node, addr))); } - /// Union a batch of learned members in, rebuilding the [`Cluster`] placement - /// once if any were genuinely new. Idempotent: a member already known is - /// skipped, so a re-gossip of a fully-known set rebuilds no placement (no - /// churn). A member with an empty node id is dropped — it is neither placeable - /// nor dialable, so a malformed gossip pair cannot poison the set. When new - /// members land, placement is rebuilt from the canonicalized (sorted, de-duped) - /// set, so every node that has learned the same members places every room - /// identically regardless of the order it learned them in. `self` is a member - /// from construction and is never relearned. + /// Union a batch of learned members into the roster — dialable, probable and + /// gossipable, but **pending**: a member learned this way is placed on no room + /// until the cluster adopts it ([`note_verified`](Self::note_verified)). + /// Idempotent: a member already known is skipped, so a re-gossip of a fully-known + /// set changes nothing (no churn). A member with an empty node id is dropped — it + /// is neither placeable nor dialable, so a malformed gossip pair cannot poison the + /// set. `self` is a member from construction and is never relearned. pub fn add_members(&mut self, members: impl IntoIterator)>) { let mut added = false; for (node, addr) in members { @@ -288,25 +354,35 @@ impl Membership { added = true; } if added { - self.cluster = Cluster::new(self.addrs.keys().cloned()); + self.rebuild_placement(); } } + /// The roster in canonical (sorted) order — every member this node knows, + /// pending ones included. What a node dials, probes and gossips, as against the + /// [`Cluster`] placement, which holds the adopted members only. + fn roster(&self) -> Vec { + let mut roster: Vec = self.addrs.keys().cloned().collect(); + roster.sort(); + roster + } + /// The members this node knows, each with its dial address — the payload a - /// node gossips. Canonical order (the sorted member set), so the advertisement - /// is deterministic. A member's address falls back to its node-id bytes if none - /// was recorded, keeping every member dialable. + /// node gossips. Canonical order (the sorted roster), so the advertisement is + /// deterministic. A member's address falls back to its node-id bytes if none was + /// recorded, keeping every member dialable. Pending members are included: a node + /// still unadopted has to be reachable and gossiped about, or it could never be + /// verified into the ring. pub fn known_members(&self) -> Vec<(NodeId, Vec)> { - self.cluster - .nodes() - .iter() + self.roster() + .into_iter() .map(|node| { let addr = self .addrs - .get(node) + .get(&node) .cloned() .unwrap_or_else(|| node.as_bytes().to_vec()); - (node.clone(), addr) + (node, addr) }) .collect() } @@ -334,6 +410,19 @@ impl Membership { &self.cluster } + /// Whether `node`'s advertise address declares a TLS transport — whether a dial + /// to it authenticates the far end before a byte is written. Unlike the *inbound* + /// direction, where a member's advertised scheme describes its own listener and + /// says nothing about the link carrying its identity here, an outbound dial runs + /// over exactly the transport that address declares. + pub fn advertises_tls(&self, node: &NodeId) -> bool { + self.addrs + .get(node) + .and_then(|addr| std::str::from_utf8(addr).ok()) + .and_then(|addr| crate::dial::PeerEndpoint::parse(addr).ok()) + .is_some_and(|endpoint| endpoint.is_tls()) + } + /// The ordered replica set for `room`, primary first — the placement Unit 3 /// routing and Unit 4 replication consult. pub fn replicas_for(&self, room: &[u8]) -> Vec { @@ -402,29 +491,134 @@ impl Membership { } /// The gossip liveness payload this node advertises: every known member with - /// its dial address, current incarnation, and state — canonical (sorted) order, - /// so the advertisement is deterministic. `self` rides at its own incarnation, - /// always `Alive`. - pub fn known_liveness(&self) -> Vec<(NodeId, Vec, u64, MemberState)> { - self.cluster - .nodes() - .iter() + /// its dial address, current incarnation, state, and whether **this node** has + /// verified it — canonical (sorted) order, so the advertisement is deterministic. + /// `self` rides at its own incarnation, always `Alive`. + /// + /// The last field is first-hand and self-scoped: a node advertises the links *it* + /// completed, never one it heard about, so the claim a receiver acts on is always + /// the claim of the member whose link carried it. Relaying another node's + /// verifications would make one member's word enough to place any id it liked. + pub fn known_liveness(&self) -> Vec<(NodeId, Vec, u64, MemberState, bool)> { + self.roster() + .into_iter() .map(|node| { let addr = self .addrs - .get(node) + .get(&node) .cloned() .unwrap_or_else(|| node.as_bytes().to_vec()); - ( - node.clone(), - addr, - self.incarnation(node), - self.gossip_state(node), - ) + let incarnation = self.incarnation(&node); + let state = self.gossip_state(&node); + let verified = self.verified_by_self(&node); + (node, addr, incarnation, state, verified) }) .collect() } + /// Whether this node has itself verified `node`. `self` is not verified by + /// itself — a node vouching for its own place in the ring is exactly what + /// adoption exists to refuse — and needs no vouching, since a node is a member of + /// its own view from construction. + fn verified_by_self(&self, node: &NodeId) -> bool { + self.verifiers + .get(node) + .is_some_and(|vs| vs.contains(&self.self_id)) + } + + /// Whether `node` is adopted — placed on rooms and counted toward their quorums, + /// as against merely known (dialed, probed and gossiped about). + pub fn is_adopted(&self, node: &NodeId) -> bool { + self.adopted.contains(node) + } + + /// Record that this node has itself completed an identity-checked peer link to + /// `node` — its dial verifying the acceptor's certificate against the address, or + /// `node` dialing in with a certificate that names it. This is the one piece of + /// evidence a node produces on its own; adoption is what the cluster does with + /// enough of them. + /// + /// Three members are never verified. `self`, because a node's own place in the + /// ring is not its to vouch for. A node outside the roster, because a link is + /// evidence about a member and there is no member here to be evidence about — + /// learning one is gossip's job, and a link that could add one would restore the + /// unchecked join this exists to close. And a member whose advertise address + /// names a different **host** than its own node id: it would be verified at one + /// host and dialed at another, so no link to either can vouch for it. A node id + /// *is* an advertise address, so that costs a legitimate member nothing — it is + /// the reply half of a gossip round, which adopts a member at whatever address + /// its tuple carries, that can introduce the mismatch. + pub fn note_verified(&mut self, node: &NodeId) { + if !self.can_be_verified(node) { + return; + } + let me = self.self_id.clone(); + if self.verifiers.entry(node.clone()).or_default().insert(me) { + self.rebuild_placement(); + } + } + + /// Whether a verification of `node` is admissible evidence at all — the bar every + /// verifier claim clears, this node's own and a peer's alike. See + /// [`note_verified`](Self::note_verified) for what each clause refuses. + fn can_be_verified(&self, node: &NodeId) -> bool { + if self.is_self(node) { + return false; + } + let Some(addr) = self.addrs.get(node) else { + return false; + }; + matches!( + (member_host(addr), member_host(node.as_bytes())), + (Some(addr_host), Some(id_host)) if addr_host == id_host + ) + } + + /// How many adopted members must vouch for a pending one before it is adopted: + /// [`ADOPTION_VERIFIERS`], clamped to the number of members that could possibly + /// vouch. Without the clamp a cluster smaller than the constant could never grow + /// — a single-node cluster has nobody but itself to verify a joiner — and with it + /// the bar is always "every adopted member that exists, up to the constant". + fn adoption_quorum(&self) -> usize { + ADOPTION_VERIFIERS.min(self.adopted.len()) + } + + /// How many *adopted* members have verified `node`. A claim by a member that is + /// itself pending does not count: two nodes minted together would otherwise vouch + /// each other into the ring without any established member ever reaching either. + fn adopted_verifier_count(&self, node: &NodeId) -> usize { + self.verifiers + .get(node) + .map(|vs| vs.iter().filter(|v| self.adopted.contains(*v)).count()) + .unwrap_or(0) + } + + /// Adopt every pending member the evidence now carries, then rebuild the + /// [`Cluster`] over the adopted set. Run to a fixpoint, because adopting a member + /// makes its own verifications count: each round admits every member that meets + /// the bar against the same adopted set, so the outcome is a function of the + /// merged state alone and never of the order the rounds happened to visit + /// members in. Placement is then rebuilt from the canonicalized (sorted, + /// de-duped) adopted set, so two nodes holding the same evidence place every room + /// identically. + fn rebuild_placement(&mut self) { + loop { + let quorum = self.adoption_quorum(); + let newly: Vec = self + .addrs + .keys() + .filter(|node| !self.adopted.contains(*node)) + .filter(|node| self.adopted_verifier_count(node) >= quorum) + .cloned() + .collect(); + if newly.is_empty() { + break; + } + self.adopted.extend(newly); + } + self.cluster = Cluster::new(self.adopted.iter().cloned()); + } + /// Record a *successful* direct gossip exchange with `node` — first-hand proof /// it is alive. Clears its failed-probe count and restores it to `Alive` at the /// known incarnation. It does **not** bump the incarnation — only the member @@ -511,9 +705,18 @@ impl Membership { self.liveness.remove(node); self.addrs.remove(node); self.relay_down.remove(node); + // A reaped member is no longer placed and no longer vouches: it leaves + // the adopted set, its own verifier set goes with it, and it is struck + // from every other member's. Were it to return it would be a fresh join + // and would have to be verified again. + self.adopted.remove(node); + self.verifiers.remove(node); + for vs in self.verifiers.values_mut() { + vs.remove(node); + } } if !to_reap.is_empty() { - self.cluster = Cluster::new(self.addrs.keys().cloned()); + self.rebuild_placement(); } to_reap } @@ -526,8 +729,9 @@ impl Membership { self.reaped.contains_key(node) } - /// Merge a gossiped liveness payload into this node's view — the SWIM anti- - /// entropy of failure detection. For each `(node, addr, incarnation, state)`: + /// Merge a gossiped liveness payload from the member `sender` into this node's + /// view — the SWIM anti-entropy of failure detection. For each + /// `(node, addr, incarnation, state, verified)`: /// /// - a member this node does not know is learned (address recorded, placement /// rebuilt) at the advertised incarnation and state — the same union @@ -542,18 +746,32 @@ impl Membership { /// its own incarnation above the received one and re-asserts `Alive`, so its /// correction wins everywhere the stale suspicion reached. /// + /// The `verified` flag is `sender`'s own claim to have completed an + /// identity-checked peer link to the member the tuple names, and it is recorded + /// **against `sender`** — the member this link is bound to — rather than against + /// whoever the payload might name. That is what keeps the evidence first-hand: a + /// member can add itself to another's verifier set and to nobody else's, so no + /// member can manufacture the verifiers that would place an id it minted. A claim + /// naming the sender itself is dropped; the verifier sets are grow-only, so the + /// merge is a union and converges however rounds interleave. + /// /// A malformed pair (empty node id) is dropped, as on the additive path. Order- /// independent and idempotent: two nodes that received the same updates in any /// order converge on the same liveness. pub fn merge_liveness( &mut self, - payload: impl IntoIterator, u64, MemberState)>, + sender: &NodeId, + payload: impl IntoIterator, u64, MemberState, bool)>, ) { let mut rebuilt = false; - for (node, addr, incarnation, state) in payload { + let mut claimed = Vec::new(); + for (node, addr, incarnation, state, verified) in payload { if node.as_bytes().is_empty() { continue; } + if verified && &node != sender && !self.is_self(&node) { + claimed.push(node.clone()); + } if self.is_self(&node) { self.refute_if_stale(incarnation, state); continue; @@ -592,8 +810,21 @@ impl Membership { } } } + // Record the sender's verifications only for members that survived the merge + // above — a claim about a node that was dropped as malformed, or that is + // tombstoned, is a claim about no member of this view — and only where the + // member is coherently addressed, the same bar this node's own links clear. + for node in claimed { + if self.can_be_verified(&node) { + rebuilt |= self + .verifiers + .entry(node) + .or_default() + .insert(sender.clone()); + } + } if rebuilt { - self.cluster = Cluster::new(self.addrs.keys().cloned()); + self.rebuild_placement(); } } diff --git a/crates/server/src/registry.rs b/crates/server/src/registry.rs index 2b751b6b..b1f851bf 100644 --- a/crates/server/src/registry.rs +++ b/crates/server/src/registry.rs @@ -596,12 +596,12 @@ impl Registry { if conn.peer.is_some() { return false; } - match conn.cert_hosts.as_deref() { + let certified = match conn.cert_hosts.as_deref() { // No certificate at all. The claim stands on its own, which is what a // deployment that has not issued per-node certificates has; one that has // says so and this is refused. None if self.require_peer_identity => return false, - None => {} + None => false, // A certificate *was* presented and verified, so it decides — including // when it names no host, which binds nothing. Treating that as certless // would make a certificate widen what a link may claim. @@ -612,9 +612,22 @@ impl Registry { { return false } - Some(_) => {} + Some(_) => true, + }; + let member = NodeId::from(claimed.to_vec()); + conn.peer = Some(member.clone()); + // An inbound link is a verification of the member it names only when a + // certificate named it. Admission alone is no evidence: the cluster secret is + // deployment-wide, so an uncertified claim is a member asserting an id rather + // than answering at one, and taking it as verification would let any + // secret-holder place any id it liked. A dial in the other direction still + // vouches without certificates — it at least proves the far end answers at the + // address the id names. + if certified { + if let Some(membership) = &mut self.membership { + membership.note_verified(&member); + } } - conn.peer = Some(NodeId::from(claimed.to_vec())); true } @@ -775,13 +788,37 @@ impl Registry { /// This node's known cluster members with liveness — the payload it gossips. /// Empty in single-node mode (no membership), so a non-cluster node advertises /// nothing. - pub fn known_liveness(&self) -> Vec<(NodeId, Vec, u64, MemberState)> { + pub fn known_liveness(&self) -> Vec<(NodeId, Vec, u64, MemberState, bool)> { self.membership .as_ref() .map(Membership::known_liveness) .unwrap_or_default() } + /// Record that this node has itself completed an identity-checked peer link to + /// `node` — the promotion signal a room's placement is built from. Driven by the + /// gossip loop's *direct* round, which dials the member and (on a TLS member) + /// authenticates it before a byte of the cluster secret is written; an + /// indirect-only round confirms a relay reached it and vouches for nothing here. + /// Inert in single-node mode. + /// + /// A deployment that *requires* an identified peer requires the dial to have + /// authenticated one, so a plaintext member is not verified this way: the dial had + /// no certificate to check, and a round that proves only "something answers here" + /// would let such a member take rooms in a cluster whose every inbound link from it + /// is refused — stalling their quorums. Without that policy the same round is the + /// honest floor, and vouches for reachability alone. + pub fn note_peer_verified(&mut self, node: &NodeId) { + let require_identity = self.require_peer_identity; + let Some(membership) = &mut self.membership else { + return; + }; + if require_identity && !membership.advertises_tls(node) { + return; + } + membership.note_verified(node); + } + /// Merge a gossiped liveness payload into this node's membership — the SWIM /// anti-entropy merge that both grows the member set and converges its liveness /// toward a cluster-wide view. Inert in single-node mode (no membership). @@ -793,12 +830,28 @@ impl Registry { /// it is how a joiner learns the cluster in one round. The *inbound* half is the one /// an unknown peer can reach, and it filters first — a peer introduces only itself, /// at its own address ([`apply_gossip`](Self::apply_gossip)). - pub fn merge_gossip(&mut self, members: Vec<(Vec, Vec, u64, MemberState)>) { + /// + /// What the reply half hands over freely is the *roster*, and a member reaching the + /// roster this way is **pending**: dialed, probed and gossiped about, but on no + /// room and in no room's quorum until the cluster verifies it. So the freedom costs + /// what it always should have — a joiner converges in one round — without letting + /// the node this one dialed choose who takes a place in the ring. + /// + /// `sender` is the member the payload came from, and every `verified` flag in it is + /// recorded as that member's own first-hand claim. + pub fn merge_gossip( + &mut self, + sender: &NodeId, + members: Vec<(Vec, Vec, u64, MemberState, bool)>, + ) { if let Some(membership) = &mut self.membership { membership.merge_liveness( + sender, members .into_iter() - .map(|(node, addr, inc, state)| (NodeId::from(node), addr, inc, state)), + .map(|(node, addr, inc, state, verified)| { + (NodeId::from(node), addr, inc, state, verified) + }), ); } } @@ -853,11 +906,18 @@ impl Registry { /// A joining node still converges: it learns the cluster from the seed it *dialed* /// (see [`merge_gossip`](Self::merge_gossip), the reply path) and then introduces /// itself to each member directly. + /// + /// **A self-introduction joins the roster, not the ring.** The member is dialed, + /// probed and gossiped about, and placed on no room until the cluster has verified + /// it — otherwise a member would grind a node id that HRW placed on the room it + /// wanted and be inside that room's replica set the moment it said so. Its own + /// tuple cannot carry that verification either: a claim naming the sender is + /// dropped. fn apply_gossip( &mut self, id: ConnId, sender: &NodeId, - members: Vec<(Vec, Vec, u64, MemberState)>, + members: Vec<(Vec, Vec, u64, MemberState, bool)>, ) -> bool { let Some(membership) = &self.membership else { return false; @@ -871,7 +931,7 @@ impl Registry { membership.is_member(&node) || (&node == sender && addr == node.as_bytes()) }) .collect(); - self.merge_gossip(members); + self.merge_gossip(sender, members); let reply = crate::gossip::gossip_frame(&self.known_liveness()); if let Some(conn) = self.conns.get_mut(&id) { conn.outbox.push(reply); diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index 289d231c..16d82a64 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -249,9 +249,13 @@ enum Cmd { /// success, `learned` carries the liveness the peer advertised back (the /// registry merges it) and the peer is noted reachable; on failure (`learned` /// is `None`) the peer is noted unreachable — the gossip-driven failover signal. + /// `direct` says the round reached `peer` by *this node's own dial* rather than + /// through a relay's second opinion, which is what makes it evidence of the peer's + /// identity and not merely of its liveness. GossipRound { peer: NodeId, learned: Option>, + direct: bool, }, /// The blob-fetch plane asks whether `identity` may retrieve blob `blob_id`, /// resolved against the live rooms' references (see @@ -733,14 +737,25 @@ async fn registry_actor( Cmd::GossipSnapshot { reply } => { let _ = reply.send(reg.known_liveness()); } - Cmd::GossipRound { peer, learned } => { + Cmd::GossipRound { + peer, + learned, + direct, + } => { // The next client delivery recomputes placement and // effective leadership off the grown set and updated // liveness, so there is nothing to flush here. match learned { Some(members) => { - reg.note_gossip_probe(peer, true); - reg.merge_gossip(members); + reg.note_gossip_probe(peer.clone(), true); + // This node dialed the peer and the transport + // authenticated it before a byte was written, so the + // round is its own first-hand verification of that + // member — the evidence its place in the ring rests on. + if direct { + reg.note_peer_verified(&peer); + } + reg.merge_gossip(&peer, members); } None => reg.note_gossip_probe(peer, false), } @@ -1145,11 +1160,19 @@ async fn gossip_loop( // A direct success carries the liveness the peer advertised back; an // indirect-only success reports the peer reachable but has nothing to merge // (an empty learned set); an all-failed round reports it unreachable (`None`). + let reached_directly = direct.is_some(); let learned = match direct { Some(members) => Some(members), None => reachable.then(Vec::new), }; - if cmds.send(Cmd::GossipRound { peer, learned }).is_err() { + if cmds + .send(Cmd::GossipRound { + peer, + learned, + direct: reached_directly, + }) + .is_err() + { return; } } diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs new file mode 100644 index 00000000..390e1db3 --- /dev/null +++ b/crates/server/tests/adoption.rs @@ -0,0 +1,810 @@ +//! C25 — rooms are placed on *adopted* members, so a node cannot mint an id into a +//! replica set. +//! +//! C13 bound a peer link to a member and gated replication, leadership and durability +//! on it. Every one of those gates asks the same question — is the sender in +//! `replicas_for(room)` — and placement is HRW over the member set, a pure and +//! publicly computable function, so **which rooms a node replicates follows from its +//! node id**. The join path lets an unknown node introduce itself. So an admitted +//! member ground an id that HRW placed on the room it wanted, introduced itself under +//! it, and was inside that room's replica set: it could supersede the leader with a +//! forged epoch, push ops in, and report heads that made majority-ack release a client +//! `Accepted` for a write no majority held. Two lesser ways in had the same shape: the +//! reply half of a gossip round adopted members freely, and a minted member polluted +//! placement and quorum for the rooms it did *not* attack, so writes to them waited on +//! acks that never came. +//! +//! Learning a member and placing rooms on it are now two admissions. A gossip-learned +//! member is **pending** — dialed, probed and gossiped about, but in no room's replica +//! set and no room's quorum — until the cluster adopts it. Adoption cannot be a local +//! predicate, because placement must be identical on every node or the ring splits, so +//! a node records only what it knows first-hand (it completed an identity-checked peer +//! link to the member), that claim rides the same anti-entropy liveness does attributed +//! to the node that made it, and a member is placed once `ADOPTION_VERIFIERS` +//! already-adopted members have verified it. A member never verifies itself, so no node +//! can place itself; configured members are adopted from birth, the operator's config +//! being the root of trust. +//! +//! These drive the registry and the membership in process (no sockets), so they are +//! deterministic and run under Miri. + +use std::sync::Arc; + +use crdtsync_core::protocol::Channel; +use crdtsync_core::{ClientId, Document, MemberState, Message, Op, Scalar}; +use crdtsync_server::auth::Identity; +use crdtsync_server::membership::{Membership, ADOPTION_VERIFIERS}; +use crdtsync_server::placement::{Cluster, NodeId}; +use crdtsync_server::{ConnId, ManualClock, Registry}; + +const CH: Channel = Channel(0); + +/// One voucher can be the attacker itself, so the bar is more than one — the premise +/// the test below rests on, checked where it cannot drift. +const _: () = assert!(ADOPTION_VERIFIERS > 1); + +/// The deployment's cluster secret — what every node in one cluster holds. +const SECRET: &[u8] = b"cluster-secret-of-at-least-32-bytes"; + +/// This node's advertise address in the in-process cluster below. +const SELF_ADDR: &str = "10.0.0.6:9000"; + +/// The per-room replication factor. Five, so a room's majority is three and one +/// follower's genuine ack is not on its own enough to release a write. +const N: usize = 5; + +fn cid(first: u8) -> ClientId { + let mut b = [0u8; 16]; + b[0] = first; + ClientId::from_bytes(b) +} + +fn doc(first: u8) -> Document { + Document::new(cid(first)) +} + +/// The static member set every node's view is built from — larger than the +/// replication factor, so a node leads some rooms and holds no replica of others. +fn members_str() -> String { + (0..9) + .map(|i| format!("10.0.0.{i}:9000")) + .collect::>() + .join(",") +} + +fn membership_for(self_addr: &str) -> Membership { + Membership::from_static_config(None, Some(self_addr), &members_str(), N).unwrap() +} + +/// A clustered registry with the cluster secret configured. +fn registry() -> Registry { + let mut r = Registry::new(cid(0xFF)); + r.set_clock(Arc::new(ManualClock::new(0))); + r.set_membership(membership_for(SELF_ADDR)); + r.set_cluster_secret(SECRET.to_vec()); + r +} + +/// A connection admitted to the peer plane as the member `node`, presenting no +/// certificate — what a deployment that has issued none has. +fn peer_as(r: &mut Registry, node: &NodeId) -> ConnId { + let id = r.connect(); + assert!( + r.deliver( + id, + Message::PeerAuth { + node: node.as_bytes().to_vec(), + secret: SECRET.to_vec(), + }, + ), + "the cluster secret admits a peer that names itself", + ); + id +} + +/// A connection admitted as `node` behind a verified client certificate that names +/// `node`'s own host — the binding C13 established, and the inbound half of a +/// verification. +fn certified_peer_as(r: &mut Registry, node: &NodeId) -> ConnId { + let host = + crdtsync_server::dial::member_host(node.as_bytes()).expect("the member names a host"); + let id = r.connect_cert_authenticated(Identity::new(b"peer".to_vec()), vec![host.into_bytes()]); + assert!(r.deliver( + id, + Message::PeerAuth { + node: node.as_bytes().to_vec(), + secret: SECRET.to_vec(), + }, + )); + id +} + +/// The frame a member sends to introduce itself: its own tuple, at its own address. +/// `verified` is the claim it makes about having reached that member — for a +/// self-introduction, a claim about itself. +fn introduces(node: &NodeId, verified: bool) -> Message { + Message::Gossip { + members: vec![( + node.as_bytes().to_vec(), + node.as_bytes().to_vec(), + 0, + MemberState::Alive, + verified, + )], + } +} + +/// The wire tuples introducing each of `nodes` at its own address, unverified — the +/// payload the *reply* half of a gossip round hands over, which introduces freely. +fn advertisements(nodes: &[&NodeId]) -> Vec<(Vec, Vec, u64, MemberState, bool)> { + nodes + .iter() + .map(|node| { + ( + node.as_bytes().to_vec(), + node.as_bytes().to_vec(), + 0, + MemberState::Alive, + false, + ) + }) + .collect() +} + +/// A room this node is the placement primary of — the room an attack from outside +/// the replica set targets. +fn room_self_leads(m: &Membership) -> Vec { + (0..1_000_000) + .map(|i| format!("room-{i}").into_bytes()) + .find(|room| m.is_primary_for(room)) + .expect("a room self leads") +} + +/// An id HRW would place into `room`'s replica set once adopted — what a member +/// grinds for when it wants to hold a room it was never placed on. All on one host, +/// which is the mint space peer identity leaves open (C13): a certified member can +/// mint only on its own host. +fn minted_for(m: &Membership, room: &[u8]) -> NodeId { + (0..1_000_000) + .map(|i| NodeId::from(format!("10.9.9.9:{i}"))) + .find(|node| { + let mut grown: Vec = m.adopted_members().to_vec(); + grown.push(node.clone()); + Cluster::new(grown).replicas(room, N).contains(node) + }) + .expect("an id the room places on") +} + +/// A replication frame for `room` at `epoch`, carrying one op. +fn replicate(d: &mut Document, room: &[u8], epoch: u64) -> Message { + let ops = d.transact(|tx| tx.register(b"k", Scalar::Int(1))); + Message::Replicate { + room: room.to_vec(), + branch: b"main".to_vec(), + epoch, + base_seq: 0, + ops, + } +} + +/// An authenticated client on `r`, handshake drained. +fn client(r: &mut Registry) -> ConnId { + let id = r.connect(); + r.deliver( + id, + Message::Hello { + client: cid(1), + app_id: Vec::new(), + schema_version: 0, + codecs: Vec::new(), + }, + ); + r.deliver( + id, + Message::Auth { + credential: b"cred".to_vec(), + }, + ); + r.take_outbox(id); + id +} + +fn sub(room: &[u8]) -> Message { + Message::Subscribe { + channel: CH, + room: room.to_vec(), + branch: Vec::new(), + zone: Vec::new(), + last_seen_seq: 0, + } +} + +fn write() -> Vec { + doc(1).transact(|tx| tx.register(b"age", Scalar::Int(30))) +} + +/// Whether `outbox` carries a write-ack `Accepted` on `CH`. +fn has_accepted(outbox: &[Message]) -> bool { + outbox + .iter() + .any(|m| matches!(m, Message::Accepted { channel, .. } if *channel == CH)) +} + +/// Commit `r`'s write on a room this node leads and return the room, the client, and +/// the room's genuine followers. +fn led_room_with_a_withheld_write(r: &mut Registry) -> (Vec, ConnId, Vec) { + let m = membership_for(SELF_ADDR); + let room = room_self_leads(&m); + let me = NodeId::from(SELF_ADDR); + let followers: Vec = m + .replicas_for(&room) + .into_iter() + .filter(|n| n != &me) + .collect(); + let c = client(r); + r.deliver(c, sub(&room)); + r.take_outbox(c); + r.deliver( + c, + Message::Ops { + channel: CH, + ops: write(), + }, + ); + assert!( + r.take_outbox(c).is_empty(), + "the Accepted is withheld until a majority holds the write", + ); + (room, c, followers) +} + +/// The liveness tuple `m` advertises for `node`, or `None` if it advertises none. +fn advertised(m: &Membership, node: &NodeId) -> Option<(Vec, u64, MemberState, bool)> { + m.known_liveness() + .into_iter() + .find(|(n, ..)| n == node) + .map(|(_, addr, inc, state, verified)| (addr, inc, state, verified)) +} + +/// Whether `m` advertises `node` as one it has itself verified. +fn verified_by(m: &Membership, node: &NodeId) -> bool { + advertised(m, node).map(|(.., v)| v).unwrap_or(false) +} + +// --- the defect: a minted id reaches no room --- + +#[test] +fn a_member_that_mints_a_node_id_does_not_enter_a_rooms_replica_set() { + // The inversion of C13's residual. The id still places — HRW is a pure function of + // the member set and grinding one that lands on a chosen room is a few tries — but + // introducing itself no longer makes it a member rooms are placed on. + let m = membership_for(SELF_ADDR); + let room = room_self_leads(&m); + let minted = minted_for(&m, &room); + let mut r = registry(); + + let p = peer_as(&mut r, &minted); + assert!( + r.deliver(p, introduces(&minted, false)), + "the join path still admits an unknown node's self-introduction", + ); + + let view = r.membership().expect("clustered"); + assert!( + view.is_member(&minted), + "the minted node is learned — it has to be dialable to ever be verified", + ); + assert!( + !view.is_adopted(&minted), + "but the cluster has not adopted it" + ); + assert!( + !view.replicas_for(&room).contains(&minted), + "so no room places on it", + ); + assert_eq!( + view.replicas_for(&room), + m.replicas_for(&room), + "and the room's replica set is the one it always was", + ); +} + +#[test] +fn a_minted_member_cannot_supersede_the_leader_with_a_forged_epoch() { + // The leadership gate, reached from outside the replica set. Before adoption the + // minted member was inside it and any epoch above this node's stripped it of the + // room; now the frame comes from a member that replicates nothing and the link + // goes, leaving the epoch where the leader left it. + let m = membership_for(SELF_ADDR); + let room = room_self_leads(&m); + let minted = minted_for(&m, &room); + let mut r = registry(); + let c = client(&mut r); + r.deliver(c, sub(&room)); + r.deliver( + c, + Message::Ops { + channel: CH, + ops: write(), + }, + ); + let epoch = r.highest_epoch(&room); + + let p = peer_as(&mut r, &minted); + assert!(r.deliver(p, introduces(&minted, false))); + assert!( + !r.deliver(p, replicate(&mut doc(9), &room, epoch + 100)), + "the forged frame drops the link", + ); + assert_eq!( + r.highest_epoch(&room), + epoch, + "and leaves this node's leadership epoch untouched", + ); +} + +#[test] +fn a_minted_member_cannot_release_an_accepted_no_majority_holds() { + // The durability gate. A room's majority is counted over its replica set, so a + // member that mints its way in acks as one of five and the leader releases the + // client's `Accepted` on a write only two nodes hold. Pending, its ack counts for + // nothing — and the write still releases the moment a real majority holds it, so + // this is a narrower quorum and not a stalled one. + let m = membership_for(SELF_ADDR); + let mut r = registry(); + let (room, c, followers) = led_room_with_a_withheld_write(&mut r); + let minted = minted_for(&m, &room); + let seq = r.hub().seq(&room); + + let p = peer_as(&mut r, &minted); + assert!(r.deliver(p, introduces(&minted, false))); + + r.record_replica_ack(minted.clone(), &room, seq); + r.record_replica_ack(followers[0].clone(), &room, seq); + assert!( + !has_accepted(&r.take_outbox(c)), + "self plus one genuine follower is not a majority of five, whatever the \ + minted member acks", + ); + + r.record_replica_ack(followers[1].clone(), &room, seq); + assert!( + has_accepted(&r.take_outbox(c)), + "a genuine majority still releases the write", + ); +} + +#[test] +fn a_pending_member_does_not_enlarge_the_quorum_of_a_room_it_does_not_attack() { + // The lesser way in: a minted member landing in the ring raises the majority of + // every room it places on, so writes to rooms it never meant to touch wait on acks + // it never sends. Pending, it changes no room's replica set at all, so the same + // genuine acks release the same write. + let m = membership_for(SELF_ADDR); + let mut r = registry(); + let (room, c, followers) = led_room_with_a_withheld_write(&mut r); + let seq = r.hub().seq(&room); + + // A member minted for a *different* room — the collateral case. + let other = (0..1_000_000) + .map(|i| format!("room-{i}").into_bytes()) + .find(|candidate| candidate != &room && m.is_primary_for(candidate)) + .expect("a second room self leads"); + let minted = minted_for(&m, &other); + let p = peer_as(&mut r, &minted); + assert!(r.deliver(p, introduces(&minted, false))); + assert_eq!( + r.membership().expect("clustered").replicas_for(&room), + m.replicas_for(&room), + "the written room's replica set is untouched", + ); + + r.record_replica_ack(followers[0].clone(), &room, seq); + r.record_replica_ack(followers[1].clone(), &room, seq); + assert!( + has_accepted(&r.take_outbox(c)), + "the room's majority is the one it always had", + ); +} + +// --- a node cannot vouch for itself --- + +#[test] +fn a_member_cannot_verify_itself_into_the_ring() { + // The claim is a member's own, so the obvious forgery is to make it about itself. + // A tuple naming the sender is dropped: a node's place in the ring is never its own + // to assert, which is the whole reason adoption is a cluster decision. + let m = membership_for(SELF_ADDR); + let room = room_self_leads(&m); + let minted = minted_for(&m, &room); + let mut r = registry(); + + let p = peer_as(&mut r, &minted); + assert!(r.deliver(p, introduces(&minted, true))); + assert!(!r.membership().expect("clustered").is_adopted(&minted)); +} + +#[test] +fn this_node_never_advertises_itself_as_verified() { + // The same rule from the inside: a node holds no verification of itself to + // advertise, so it never contributes one to its own adoption anywhere. + let mut m = membership_for(SELF_ADDR); + let me = NodeId::from(SELF_ADDR); + m.note_verified(&me); + assert!(!verified_by(&m, &me)); +} + +// --- the threshold --- + +#[test] +fn one_members_verification_does_not_place_a_node() { + // Adoption takes more than one member, because a single compromised member is + // exactly the attacker here: it would otherwise vouch for the id it ground and + // place it itself. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + + m.merge_liveness( + &NodeId::from("10.0.0.1:9000"), + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + assert!(!m.is_adopted(&joiner)); +} + +#[test] +fn enough_members_verifications_place_a_node() { + // The other side: the bar is reachable, and clearing it puts the member in every + // room its id places on. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + for voucher in ["10.0.0.1:9000", "10.0.0.2:9000"] { + m.merge_liveness( + &NodeId::from(voucher), + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + } + assert!(m.is_adopted(&joiner)); + assert!(m.adopted_members().contains(&joiner)); + assert!( + (0..64) + .map(|i| format!("room-{i}").into_bytes()) + .any(|room| m.replicas_for(&room).contains(&joiner)), + "and it now holds rooms", + ); +} + +#[test] +fn the_same_members_verification_twice_is_still_one_voucher() { + // The evidence is a set of members, not a count of frames, so a member that + // re-gossips its own claim every round does not accumulate into a majority of one. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + for _ in 0..8 { + m.merge_liveness( + &NodeId::from("10.0.0.1:9000"), + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + } + assert!(!m.is_adopted(&joiner)); +} + +#[test] +fn a_pending_members_verification_does_not_count() { + // Two nodes minted together would otherwise vouch each other in without a single + // established member ever reaching either. Only an adopted member's claim counts. + let mut m = membership_for(SELF_ADDR); + let first = NodeId::from("10.9.9.9:9000"); + let second = NodeId::from("10.9.9.9:9001"); + for node in [&first, &second] { + m.add_member((*node).clone(), node.as_bytes().to_vec()); + } + let claim = |about: &NodeId| { + [( + about.clone(), + about.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )] + }; + m.merge_liveness(&second, claim(&first)); + m.merge_liveness(&first, claim(&second)); + // One genuine member reaches the first — still one short, because the pending + // sibling's word is worth nothing. + m.merge_liveness(&NodeId::from("10.0.0.1:9000"), claim(&first)); + assert!(!m.is_adopted(&first)); + assert!(!m.is_adopted(&second)); +} + +// --- the address a member is verified at is the address its id names --- + +#[test] +fn a_member_advertised_at_another_hosts_address_is_never_adopted() { + // The other lesser way in: the reply half of a gossip round adopts a member at + // whatever address its tuple carries, so a dialed node can introduce an id pointing + // somewhere else entirely. Such a member would be verified at one host and dialed + // at another, so no link to either vouches for it and it stays pending however many + // members claim it. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone(), b"10.6.6.6:9000".to_vec()); + for voucher in ["10.0.0.1:9000", "10.0.0.2:9000", "10.0.0.3:9000"] { + m.merge_liveness( + &NodeId::from(voucher), + [( + joiner.clone(), + b"10.6.6.6:9000".to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + } + assert!(!m.is_adopted(&joiner)); + // Nor does this node's own link to it vouch, for the same reason. + m.note_verified(&joiner); + assert!(!verified_by(&m, &joiner)); + assert!(!m.is_adopted(&joiner)); +} + +#[test] +fn a_member_this_view_never_learned_is_not_verified_into_existence() { + // A link is evidence *about* a member; where this view holds no member there is + // nothing to be evidence about, and a link that could add one would restore the + // unchecked join this closes. Learning a member stays gossip's job. + let mut m = membership_for(SELF_ADDR); + let stranger = NodeId::from("10.9.9.9:9000"); + m.note_verified(&stranger); + assert!(!m.is_member(&stranger)); + assert!(!m.is_adopted(&stranger)); +} + +// --- what a link means --- + +#[test] +fn only_the_dialing_side_verifies_the_other() { + // A dial authenticates the far end against the address it dialed; being dialed + // proves the sender reached here and nothing about who answers at the sender's own + // address. So the initiator comes away holding a verification and the peer does not. + let mut a = Membership::from_static_config(None, Some("10.0.0.1:9000"), "", N).unwrap(); + let mut b = Membership::from_static_config(None, Some("10.0.0.2:9000"), "", N).unwrap(); + let a_id = NodeId::from("10.0.0.1:9000"); + let b_id = NodeId::from("10.0.0.2:9000"); + + crdtsync_server::gossip::exchange(&mut a, &mut b); + assert!(verified_by(&a, &b_id), "the dialer verified the peer"); + assert!(!verified_by(&b, &a_id), "the dialed node verified nobody"); +} + +#[test] +fn an_uncertified_inbound_link_does_not_verify_the_member_it_claims() { + // The cluster secret is one deployment-wide value, so an uncertified claim is a + // member asserting an id rather than answering at one. Taking it as verification + // would let any secret-holder vouch for any id it liked, which is the mint again + // one step over. + let m = membership_for(SELF_ADDR); + let room = room_self_leads(&m); + let minted = minted_for(&m, &room); + let mut r = registry(); + + let p = peer_as(&mut r, &minted); + assert!(r.deliver(p, introduces(&minted, false))); + // Re-admit on a second link: repetition is not evidence either. + let again = peer_as(&mut r, &minted); + assert!(r.deliver(again, introduces(&minted, false))); + assert!(!verified_by(r.membership().expect("clustered"), &minted)); +} + +#[test] +fn a_certified_inbound_link_verifies_the_member_it_names() { + // The other half of the promotion signal: a member dialing in behind a certificate + // that names its own host is verified by this node, exactly as this node's own dial + // to it would be. One verification is still not adoption in a cluster this size. + let m = membership_for(SELF_ADDR); + let room = room_self_leads(&m); + let minted = minted_for(&m, &room); + let mut r = registry(); + + let p = peer_as(&mut r, &minted); + assert!(r.deliver(p, introduces(&minted, false))); + certified_peer_as(&mut r, &minted); + + let view = r.membership().expect("clustered"); + assert!(verified_by(view, &minted), "the certificate vouches"); + assert!( + !view.is_adopted(&minted), + "and one voucher is short of the bar", + ); +} + +// --- convergence --- + +#[test] +fn adoption_is_independent_of_the_order_the_evidence_arrives() { + // Placement must be identical on every node or the ring splits, so adoption has to + // be a function of the evidence and not of the order it landed in. + let joiner = NodeId::from("10.9.9.9:9000"); + let vouchers = ["10.0.0.1:9000", "10.0.0.2:9000", "10.0.0.3:9000"]; + let claim = [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )]; + + let mut forward = membership_for(SELF_ADDR); + forward.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + for v in vouchers { + forward.merge_liveness(&NodeId::from(v), claim.clone()); + } + + let mut backward = membership_for(SELF_ADDR); + backward.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + for v in vouchers.iter().rev() { + backward.merge_liveness(&NodeId::from(*v), claim.clone()); + } + + assert_eq!(forward.adopted_members(), backward.adopted_members()); + for i in 0..32 { + let room = format!("room-{i}").into_bytes(); + assert_eq!(forward.replicas_for(&room), backward.replicas_for(&room)); + } +} + +#[test] +fn a_configured_member_is_adopted_from_birth() { + // The operator's config is the root of trust a cluster starts from — there is no + // earlier authority for it to be vouched for by, and a cluster whose own seeds were + // pending could never place a room at all. + let m = membership_for(SELF_ADDR); + assert_eq!(m.members(), m.adopted_members().to_vec()); + for i in 0..9 { + assert!(m.is_adopted(&NodeId::from(format!("10.0.0.{i}:9000")))); + } +} + +#[test] +fn a_reaped_member_loses_its_place_and_its_vouches() { + // Reaping removes a durably-gone member from the roster; it must take its adoption + // and its word with it, or a departed node would keep vouching for joiners nobody + // can reach. + let mut m = membership_for(SELF_ADDR); + let departing = NodeId::from("10.0.0.1:9000"); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + m.merge_liveness( + &departing, + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + + for _ in 0..crdtsync_server::membership::DEAD_AFTER_FAILURES { + m.note_gossip_unreachable(&departing); + } + for _ in 0..crdtsync_server::membership::REAP_AFTER_DEAD_TICKS { + m.reap_dead(); + } + assert!(!m.is_member(&departing)); + assert!(!m.is_adopted(&departing)); + + // Its vouch went with it: one further genuine claim is now the first, not the + // second. + m.merge_liveness( + &NodeId::from("10.0.0.2:9000"), + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + assert!(!m.is_adopted(&joiner)); +} + +// --- a pending member is still a member --- + +#[test] +fn a_pending_member_is_dialed_probed_and_gossiped_about() { + // Pending is not exile: a member nobody dials can never be verified, so it would + // never be adopted and a genuine joiner would never join. It rides the roster, the + // gossip advertisement, and the indirect-probe roster exactly as an adopted member + // does — it simply holds no room. + let m = membership_for(SELF_ADDR); + let room = room_self_leads(&m); + let minted = minted_for(&m, &room); + let mut r = registry(); + + let p = peer_as(&mut r, &minted); + assert!(r.deliver(p, introduces(&minted, false))); + + assert!( + r.known_members().iter().any(|(node, _)| node == &minted), + "the roster carries it, so the gossip loop dials it", + ); + let advert = advertised(r.membership().expect("clustered"), &minted); + assert_eq!( + advert, + Some((minted.as_bytes().to_vec(), 0, MemberState::Alive, false)), + "and it is advertised, unverified, at its own address", + ); + // The reply this node just queued carries it too, so the joiner's own view converges. + let replied = r.take_outbox(p).into_iter().any(|msg| match msg { + Message::Gossip { members } => members + .iter() + .any(|(node, ..)| node.as_slice() == minted.as_bytes()), + _ => false, + }); + assert!(replied, "the gossip reply advertises it back"); +} + +#[test] +fn a_plaintext_dial_does_not_verify_where_an_identified_peer_is_required() { + // A deployment that requires an identified peer requires the dial to have + // authenticated one. A plaintext member is dialed with no certificate to check, so + // the round proves only that something answers there — and adopting on that would + // give rooms to a member whose every inbound link is refused, stalling their + // quorums. A `wss://` member is dialed over a transport that authenticates it, and + // is verified. + let mut r = registry(); + r.set_require_peer_identity(true); + let plain = NodeId::from("10.9.9.9:9000"); + let tls = NodeId::from("wss://10.9.9.8:9000"); + r.merge_gossip( + &NodeId::from("10.0.0.1:9000"), + advertisements(&[&plain, &tls]), + ); + + r.note_peer_verified(&plain); + r.note_peer_verified(&tls); + + let view = r.membership().expect("clustered"); + assert!( + !verified_by(view, &plain), + "a plaintext dial checked nothing" + ); + assert!( + verified_by(view, &tls), + "a TLS dial authenticated the far end" + ); +} + +#[test] +fn a_plaintext_dial_still_verifies_where_no_identity_is_required() { + // The honest floor, so the refusal above is the policy and not a blanket one. With + // no certificates configured a completed dial vouches for reachability at the + // address the id names — which is less than identity, and still more than a member + // saying an id exists. + let mut r = registry(); + let plain = NodeId::from("10.9.9.9:9000"); + r.merge_gossip(&NodeId::from("10.0.0.1:9000"), advertisements(&[&plain])); + r.note_peer_verified(&plain); + assert!(verified_by(r.membership().expect("clustered"), &plain)); +} diff --git a/crates/server/tests/failover.rs b/crates/server/tests/failover.rs index fe323c16..d6e73b2e 100644 --- a/crates/server/tests/failover.rs +++ b/crates/server/tests/failover.rs @@ -203,7 +203,7 @@ fn all_replicas_live_effective_equals_placement() { assert_eq!(m.is_effective_primary_for(&room), m.is_primary_for(&room)); // Every member is live until proven down. for member in m.members() { - assert!(m.is_live(member)); + assert!(m.is_live(&member)); } } } diff --git a/crates/server/tests/gossip.rs b/crates/server/tests/gossip.rs index 97895cb8..991e1582 100644 --- a/crates/server/tests/gossip.rs +++ b/crates/server/tests/gossip.rs @@ -58,6 +58,23 @@ fn sample_rooms() -> Vec> { .collect() } +/// Drive two gossip rounds per ordered pair, so every node has dialed every other +/// twice — what a real node's randomized peer choice reaches within a handful of +/// rounds. The roster converges much faster than this; the *ring* waits on it, +/// because a node adopts a member off links it completed itself. Two passes, not +/// one: the first completes the links, and the second carries each node's own +/// verifications to the others, which is the only way they travel. +fn all_pairs(a: &mut Membership, b: &mut Membership, c: &mut Membership) { + for _ in 0..2 { + exchange(a, b); + exchange(b, a); + exchange(a, c); + exchange(c, a); + exchange(b, c); + exchange(c, b); + } +} + /// Assert two memberships place every sample room identically — same ordered /// replica set and same primary. fn placement_agrees(x: &Membership, y: &Membership) { @@ -77,10 +94,12 @@ fn placement_agrees(x: &Membership, y: &Membership) { /// The wire payload a membership advertises — its known members with liveness, as /// raw bytes. -fn payload(m: &Membership) -> Vec<(Vec, Vec, u64, MemberState)> { +fn payload(m: &Membership) -> Vec<(Vec, Vec, u64, MemberState, bool)> { m.known_liveness() .into_iter() - .map(|(node, addr, inc, state)| (node.as_bytes().to_vec(), addr, inc, state)) + .map(|(node, addr, inc, state, verified)| { + (node.as_bytes().to_vec(), addr, inc, state, verified) + }) .collect() } @@ -96,8 +115,16 @@ fn tuple( addr: &str, inc: u64, state: MemberState, -) -> (NodeId, Vec, u64, MemberState) { - (node.clone(), addr.as_bytes().to_vec(), inc, state) +) -> (NodeId, Vec, u64, MemberState, bool) { + (node.clone(), addr.as_bytes().to_vec(), inc, state, false) +} + +/// The member a directly-merged liveness payload is attributed to. These payloads +/// carry no verification claims, so the merge does not turn on which member sent +/// them — but every anti-entropy frame arrives from one, and the merge records its +/// claims against it. +fn sender() -> NodeId { + NodeId::from_addr(B) } /// The cluster secret these nodes share — what admits a node-to-node link to a @@ -174,8 +201,13 @@ fn a_seed_only_node_converges_on_the_full_cluster() { assert_eq!(member_set(&b), full, "B converged"); assert_eq!(member_set(&c), full, "C converged"); + // The roster converges in two rounds; the *ring* waits on the cluster having + // reached each newcomer, since adoption rests on links a node completed itself. + all_pairs(&mut a, &mut b, &mut c); + // Placement converged: all three compute the same replica set and primary for - // every sample room. + // every sample room, over all three members. + assert_eq!(a.adopted_members(), full, "A places on all three"); placement_agrees(&a, &b); placement_agrees(&b, &c); } @@ -192,6 +224,7 @@ fn convergence_holds_whatever_the_gossip_order() { exchange(&mut b, &mut a); exchange(&mut c, &mut b); } + all_pairs(&mut a, &mut b, &mut c); let full = vec![ NodeId::from_addr(A), NodeId::from_addr(B), @@ -241,7 +274,7 @@ fn re_gossiping_a_fully_known_set_changes_nothing() { // Union the node's own advertisement back into itself — a re-gossip of an // already-known set. let own = payload(&m); - merge_into(&mut m, own); + merge_into(&mut m, &sender(), own); assert_eq!(member_set(&m), members_before, "no member churn"); assert_eq!(m.known_members(), known_before, "addresses unchanged"); @@ -280,13 +313,15 @@ fn a_member_with_an_empty_node_id_is_dropped() { // The same guard holds through the wire merge path. merge_into( &mut m, + &sender(), vec![ - (Vec::new(), Vec::new(), 0, MemberState::Alive), + (Vec::new(), Vec::new(), 0, MemberState::Alive, false), ( C.as_bytes().to_vec(), C.as_bytes().to_vec(), 0, MemberState::Alive, + false, ), ], ); @@ -331,12 +366,16 @@ fn a_single_node_registry_knows_no_members_and_ignores_gossip() { assert!(r.known_members().is_empty(), "no members to advertise"); // Merging a gossip payload is inert with no membership. - r.merge_gossip(vec![( - A.as_bytes().to_vec(), - A.as_bytes().to_vec(), - 0, - MemberState::Alive, - )]); + r.merge_gossip( + &sender(), + vec![( + A.as_bytes().to_vec(), + A.as_bytes().to_vec(), + 0, + MemberState::Alive, + false, + )], + ); assert!(r.known_members().is_empty()); assert!(r.membership().is_none()); @@ -350,6 +389,7 @@ fn a_single_node_registry_knows_no_members_and_ignores_gossip() { A.as_bytes().to_vec(), 0, MemberState::Alive, + false, )], }, ); @@ -375,6 +415,13 @@ fn a_member_learned_by_gossip_becomes_a_placement_target() { .all(|room| m.primary_for(room) == Some(NodeId::from_addr(self_addr)))); m.add_member(newcomer_id.clone(), newcomer.as_bytes().to_vec()); + // Learned, but pending: it is dialed and gossiped about, and placed nowhere. + assert!(sample_rooms() + .iter() + .all(|room| m.primary_for(room) == Some(NodeId::from_addr(self_addr)))); + // This node then reaches it over an identity-checked link, which is the whole + // cluster's verdict here — there is nobody else to raise. + m.note_verified(&newcomer_id); let room = sample_rooms() .into_iter() @@ -400,12 +447,16 @@ fn a_registry_grows_its_membership_from_a_gossip_frame() { r.set_membership(seeded(A, B)); assert_eq!(r.known_members().len(), 2); - r.merge_gossip(vec![( - C.as_bytes().to_vec(), - C.as_bytes().to_vec(), - 0, - MemberState::Alive, - )]); + r.merge_gossip( + &sender(), + vec![( + C.as_bytes().to_vec(), + C.as_bytes().to_vec(), + 0, + MemberState::Alive, + false, + )], + ); let members: Vec = r .known_members() .into_iter() @@ -434,6 +485,7 @@ fn the_gossip_frame_carries_every_known_member_with_its_address_and_liveness() { addr.as_bytes().to_vec(), 0, MemberState::Alive, + false, ))); } } @@ -552,14 +604,14 @@ fn a_falsely_suspected_node_refutes_and_the_cluster_restores_it() { let c_id = NodeId::from_addr(C); let stale = vec![tuple(&c_id, C, 0, MemberState::Suspect)]; - a.merge_liveness(stale.clone()); - b.merge_liveness(stale.clone()); + a.merge_liveness(&sender(), stale.clone()); + b.merge_liveness(&sender(), stale.clone()); assert_eq!(a.gossip_state(&c_id), MemberState::Suspect); assert_eq!(b.gossip_state(&c_id), MemberState::Suspect); // C receives the same suspicion about itself and refutes: incarnation climbs // above the received 0, state re-asserted Alive. - c.merge_liveness(stale); + c.merge_liveness(&sender(), stale); assert_eq!(c.gossip_state(&c_id), MemberState::Alive); assert!( c.incarnation(&c_id) > 0, @@ -582,11 +634,11 @@ fn refutation_overrides_even_a_dead_verdict() { let mut a = full_node(A); let mut c = full_node(C); let c_id = NodeId::from_addr(C); - a.merge_liveness(vec![tuple(&c_id, C, 4, MemberState::Dead)]); + a.merge_liveness(&sender(), vec![tuple(&c_id, C, 4, MemberState::Dead)]); assert!(!a.is_live(&c_id)); // C hears the Dead@4 about itself, refutes above 4, re-disseminates Alive. - c.merge_liveness(vec![tuple(&c_id, C, 4, MemberState::Dead)]); + c.merge_liveness(&sender(), vec![tuple(&c_id, C, 4, MemberState::Dead)]); assert!(c.incarnation(&c_id) > 4); exchange(&mut c, &mut a); assert_eq!(a.gossip_state(&c_id), MemberState::Alive); @@ -599,14 +651,14 @@ fn refutation_overrides_even_a_dead_verdict() { fn a_higher_incarnation_always_wins_the_merge() { let mut a = full_node(A); let c_id = NodeId::from_addr(C); - a.merge_liveness(vec![tuple(&c_id, C, 5, MemberState::Dead)]); + a.merge_liveness(&sender(), vec![tuple(&c_id, C, 5, MemberState::Dead)]); assert_eq!(a.gossip_state(&c_id), MemberState::Dead); // A fresher (higher-incarnation) Alive supersedes the older Dead. - a.merge_liveness(vec![tuple(&c_id, C, 6, MemberState::Alive)]); + a.merge_liveness(&sender(), vec![tuple(&c_id, C, 6, MemberState::Alive)]); assert_eq!(a.gossip_state(&c_id), MemberState::Alive); assert_eq!(a.incarnation(&c_id), 6); // A stale lower-incarnation Dead is ignored. - a.merge_liveness(vec![tuple(&c_id, C, 5, MemberState::Dead)]); + a.merge_liveness(&sender(), vec![tuple(&c_id, C, 5, MemberState::Dead)]); assert_eq!(a.gossip_state(&c_id), MemberState::Alive); } @@ -615,12 +667,12 @@ fn at_equal_incarnation_the_more_suspicious_state_wins() { let mut a = full_node(A); let c_id = NodeId::from_addr(C); // Alive -> Suspect -> Dead all at incarnation 3, each more suspicious wins. - a.merge_liveness(vec![tuple(&c_id, C, 3, MemberState::Suspect)]); + a.merge_liveness(&sender(), vec![tuple(&c_id, C, 3, MemberState::Suspect)]); assert_eq!(a.gossip_state(&c_id), MemberState::Suspect); - a.merge_liveness(vec![tuple(&c_id, C, 3, MemberState::Dead)]); + a.merge_liveness(&sender(), vec![tuple(&c_id, C, 3, MemberState::Dead)]); assert_eq!(a.gossip_state(&c_id), MemberState::Dead); // A less-suspicious state at the same incarnation does not un-do it. - a.merge_liveness(vec![tuple(&c_id, C, 3, MemberState::Alive)]); + a.merge_liveness(&sender(), vec![tuple(&c_id, C, 3, MemberState::Alive)]); assert_eq!(a.gossip_state(&c_id), MemberState::Dead); } @@ -636,11 +688,11 @@ fn liveness_merge_is_order_independent() { ]; let mut fwd = full_node(A); for u in updates.iter().cloned() { - fwd.merge_liveness(vec![u]); + fwd.merge_liveness(&sender(), vec![u]); } let mut rev = full_node(A); for u in updates.iter().rev().cloned() { - rev.merge_liveness(vec![u]); + rev.merge_liveness(&sender(), vec![u]); } assert_eq!(fwd.gossip_state(&c_id), rev.gossip_state(&c_id)); assert_eq!(fwd.incarnation(&c_id), rev.incarnation(&c_id)); @@ -716,7 +768,7 @@ fn a_relay_reconnect_does_not_resurrect_a_gossip_dead_node() { a.mark_node_live(&c_id); assert!(!a.is_live(&c_id), "gossip-Dead survives a relay reconnect"); // Only a gossip refutation (higher-incarnation Alive) restores it. - a.merge_liveness(vec![tuple(&c_id, C, 99, MemberState::Alive)]); + a.merge_liveness(&sender(), vec![tuple(&c_id, C, 99, MemberState::Alive)]); assert!(a.is_live(&c_id)); } @@ -767,6 +819,7 @@ async fn a_gossip_exchange_over_the_socket_merges_and_replies() { C.as_bytes().to_vec(), 0, MemberState::Alive, + false, )], }; let learned = gossip_exchange(&addr, cid(0xEE), &plaintext_dialer_as(C), frame) diff --git a/crates/server/tests/peer_auth.rs b/crates/server/tests/peer_auth.rs index aaa5a1a2..3e74927a 100644 --- a/crates/server/tests/peer_auth.rs +++ b/crates/server/tests/peer_auth.rs @@ -307,6 +307,7 @@ fn a_bare_socket_cannot_gossip_membership() { b"10.9.9.9:9000".to_vec(), 7, MemberState::Alive, + false, )], }, ); @@ -548,6 +549,7 @@ fn a_single_node_deployment_refuses_every_node_to_node_frame() { b"10.9.9.9:9000".to_vec(), 7, MemberState::Alive, + false, )], }, Message::FollowerHeads { @@ -682,6 +684,7 @@ fn an_admitted_peer_may_send_every_node_to_node_frame() { b"10.9.9.9:9000".to_vec(), 7, MemberState::Alive, + false, )], }, )); diff --git a/crates/server/tests/peer_identity.rs b/crates/server/tests/peer_identity.rs index e2f8b7c8..1298332a 100644 --- a/crates/server/tests/peer_identity.rs +++ b/crates/server/tests/peer_identity.rs @@ -25,16 +25,15 @@ //! //! What that closes: a member outside a room's replica set can no longer touch that //! room at all, a peer introduces only itself and only at its own address, and a node -//! reports only its own heads. Two limits are pinned below as passing tests rather +//! reports only its own heads. One limit is pinned below as a passing test rather //! than left implied. Inside a room's replica set the epoch is still the only arbiter //! — a genuinely promoted replica must be able to supersede a stale leader, and //! nothing here tells it apart from a peer replica forging the bump, which needs a -//! real election. And placement follows from a node id while the join path lets an -//! unknown node introduce itself, so a member can *mint* an id that places it into a -//! room's replica set and reach these gates from inside it; peer identity bounds that -//! to the member's own certified host, and closing it needs placement over verified -//! members (C17), which is a cluster-wide change because placement must be identical -//! on every node. +//! real election. What was a second limit — that placement follows from a node id +//! while the join path lets an unknown node introduce itself, so a member could *mint* +//! an id that placed it into a room's replica set and reach these gates from inside it +//! — is closed by C25: rooms are placed on *adopted* members, and a node introducing +//! itself is pending until the cluster has verified it. See `adoption.rs`. //! //! Most of these drive the registry in process (no sockets), so they are //! deterministic and run under Miri; the socket tests at the end stand up a real @@ -379,53 +378,6 @@ fn a_replica_of_the_room_still_supersedes_a_stale_leader() { ); } -#[test] -fn a_member_that_mints_a_node_id_places_itself_into_a_rooms_replica_set() { - // The second limit, pinned rather than implied — and the reason C17 is a - // prerequisite for these gates to hold against a *compromised* member rather than - // a follow-on. Placement is HRW over the member set, a pure and publicly - // computable function, so which rooms a node replicates follows from its node id; - // and the join path lets an unknown node introduce itself. A member can therefore - // mint an id, land in a room's replica set, and reach the leadership gate from - // inside it. Peer identity bounds the mint space to the member's own certified - // host — it cannot mint on anyone else's — but does not close it, because closing - // it means placing rooms only on members this node has itself verified, and - // placement must be identical on every node or the ring diverges. - let m = membership_for(SELF_ADDR); - let room = room_self_leads_without(&m, &outsider_of(&m, b"room-0")); - let mut r = registry(); - commit_write(&mut r, &room); - let epoch = r.highest_epoch(&room); - - // Grind an id that HRW places on the room, then join under it. - let minted = (0..1_000_000) - .map(|i| NodeId::from(format!("10.9.9.9:{i}"))) - .find(|node| { - let mut grown = membership_for(SELF_ADDR); - grown.add_member(node.clone(), node.as_bytes().to_vec()); - grown.replicas_for(&room).contains(node) - }) - .expect("an id the room places on"); - let p = peer_as(&mut r, &minted); - assert!(r.deliver( - p, - Message::Gossip { - members: vec![( - minted.as_bytes().to_vec(), - minted.as_bytes().to_vec(), - 0, - MemberState::Alive, - )], - }, - )); - assert!(r.deliver(p, replicate(&mut doc(9), &room, epoch + 100))); - assert_eq!( - r.highest_epoch(&room), - epoch + 100, - "a minted member reaches the leadership gate from inside the replica set", - ); -} - // --- membership: a peer introduces itself and nobody else --- #[test] @@ -445,6 +397,7 @@ fn a_member_cannot_introduce_a_foreign_address_into_the_member_set() { b"evil.example:9000".to_vec(), 7, MemberState::Alive, + false, )], }, ), @@ -468,22 +421,24 @@ fn a_member_cannot_hide_a_foreign_address_among_legitimate_tuples() { let joiner = NodeId::from("10.9.9.9:9000"); let mut r = registry(); let p = peer_as(&mut r, &joiner); - let mut members: Vec<(Vec, Vec, u64, MemberState)> = m + let mut members: Vec<(Vec, Vec, u64, MemberState, bool)> = m .known_liveness() .into_iter() - .map(|(node, addr, inc, state)| (node.as_bytes().to_vec(), addr, inc, state)) + .map(|(node, addr, inc, state, v)| (node.as_bytes().to_vec(), addr, inc, state, v)) .collect(); members.push(( joiner.as_bytes().to_vec(), joiner.as_bytes().to_vec(), 0, MemberState::Alive, + false, )); members.push(( b"evil.example:9000".to_vec(), b"evil.example:9000".to_vec(), 9, MemberState::Alive, + false, )); assert!(r.deliver(p, Message::Gossip { members })); assert!( @@ -516,6 +471,7 @@ fn a_member_cannot_introduce_itself_at_an_address_it_chose() { b"ws://evil.example:9000".to_vec(), 0, MemberState::Alive, + false, )], }, )); @@ -543,6 +499,7 @@ fn a_liveness_verdict_about_a_known_member_still_disseminates() { subject.as_bytes().to_vec(), 5, MemberState::Dead, + false, )], }, )); @@ -568,6 +525,7 @@ fn a_joining_node_introduces_itself() { joiner.as_bytes().to_vec(), 0, MemberState::Alive, + false, )], }, )); @@ -843,7 +801,7 @@ fn an_identified_member_still_sends_every_node_to_node_frame() { members: m .known_liveness() .into_iter() - .map(|(node, addr, inc, state)| (node.as_bytes().to_vec(), addr, inc, state)) + .map(|(node, addr, inc, state, v)| (node.as_bytes().to_vec(), addr, inc, state, v)) .collect(), }, )); diff --git a/crates/server/tests/peer_tls.rs b/crates/server/tests/peer_tls.rs index 2688fb97..0b5f61db 100644 --- a/crates/server/tests/peer_tls.rs +++ b/crates/server/tests/peer_tls.rs @@ -465,6 +465,7 @@ async fn anti_entropy_and_ping_req_round_trip_over_tls() { b"10.0.0.9:9000".to_vec(), 1, MemberState::Alive, + false, )]), ) }) diff --git a/crates/server/tests/reaping.rs b/crates/server/tests/reaping.rs index ea011025..1e0fd343 100644 --- a/crates/server/tests/reaping.rs +++ b/crates/server/tests/reaping.rs @@ -47,6 +47,14 @@ fn kill(m: &mut Membership, node: &NodeId) { assert_eq!(m.gossip_state(node), MemberState::Dead); } +/// The member a directly-merged liveness payload is attributed to. These payloads +/// carry no verification claims, so the merge does not turn on which member sent +/// them — but every anti-entropy frame arrives from one, and the merge records its +/// claims against it. +fn sender() -> NodeId { + nid(B) +} + /// Run `n` reap checks, returning every id reaped across them. fn reap_n(m: &mut Membership, n: u32) -> Vec { let mut reaped = Vec::new(); @@ -138,7 +146,16 @@ fn a_recovered_member_before_reaping_is_kept() { reap_n(&mut m, REAP_AFTER_DEAD_TICKS - 1); // D refutes with a higher incarnation just before the reap tick — its reap // clock resets and it is not removed. - m.merge_liveness([(d.clone(), D.as_bytes().to_vec(), 5, MemberState::Alive)]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + 5, + MemberState::Alive, + false, + )], + ); assert_eq!(m.gossip_state(&d), MemberState::Alive); reap_n(&mut m, REAP_AFTER_DEAD_TICKS); assert!( @@ -187,7 +204,16 @@ fn a_reaped_member_is_not_relearned_from_stale_gossip() { assert!(!m.is_member(&d)); // A peer that has not yet reaped D keeps gossiping it Dead at the same // incarnation. The tombstone must keep it out — no reap-then-resurrect. - m.merge_liveness([(d.clone(), D.as_bytes().to_vec(), d_inc, MemberState::Dead)]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + d_inc, + MemberState::Dead, + false, + )], + ); assert!( !m.is_member(&d), "stale Dead gossip does not resurrect a reaped member" @@ -239,12 +265,16 @@ fn a_returned_member_with_a_higher_incarnation_is_resurrected() { assert!(!m.is_member(&d)); // D returns Alive at a higher incarnation (it refuted before rejoining) — it // escapes the tombstone and rejoins the roster. - m.merge_liveness([( - d.clone(), - D.as_bytes().to_vec(), - reap_inc + 1, - MemberState::Alive, - )]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + reap_inc + 1, + MemberState::Alive, + false, + )], + ); assert!(m.is_member(&d), "a genuinely-returned node rejoins"); assert_eq!(m.gossip_state(&d), MemberState::Alive); } @@ -259,13 +289,31 @@ fn a_crash_restarted_member_rejoins_at_incarnation_zero() { let d = nid(D); // Refute D up to a non-zero incarnation first, so "reaped incarnation" > 0 and a // naive incarnation gate would permanently exclude the rebooted D. - m.merge_liveness([(d.clone(), D.as_bytes().to_vec(), 7, MemberState::Alive)]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + 7, + MemberState::Alive, + false, + )], + ); kill(&mut m, &d); assert!(m.incarnation(&d) >= 7); reap_n(&mut m, REAP_AFTER_DEAD_TICKS); assert!(!m.is_member(&d), "D was reaped"); // Rebooted D gossips itself Alive at incarnation 0. - m.merge_liveness([(d.clone(), D.as_bytes().to_vec(), 0, MemberState::Alive)]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + 0, + MemberState::Alive, + false, + )], + ); assert!( m.is_member(&d), "a crash-restarted node at incarnation 0 still rejoins" @@ -296,7 +344,16 @@ fn a_tombstone_within_retention_still_blocks_resurrection() { "the tombstone is retained below the threshold" ); // Stale Dead gossip within retention still cannot resurrect it. - m.merge_liveness([(d.clone(), D.as_bytes().to_vec(), d_inc, MemberState::Dead)]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + d_inc, + MemberState::Dead, + false, + )], + ); assert!( !m.is_member(&d), "a retained tombstone still blocks stale-gossip resurrection" @@ -358,7 +415,16 @@ fn a_pruned_member_reappearing_is_a_fresh_join() { assert!(!m.is_tombstoned(&d)); // With the tombstone gone, D returning Alive at incarnation 0 is a plain fresh // join — no resurrection gate, and it is not tombstoned again. - m.merge_liveness([(d.clone(), D.as_bytes().to_vec(), 0, MemberState::Alive)]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + 0, + MemberState::Alive, + false, + )], + ); assert!(m.is_member(&d), "a pruned member rejoins as a fresh member"); assert_eq!(m.gossip_state(&d), MemberState::Alive); assert!(!m.is_tombstoned(&d)); @@ -385,11 +451,29 @@ fn a_higher_incarnation_dead_does_not_resurrect() { kill(&mut m, &d); reap_n(&mut m, REAP_AFTER_DEAD_TICKS); assert!(!m.is_member(&d)); - m.merge_liveness([(d.clone(), D.as_bytes().to_vec(), 99, MemberState::Dead)]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + 99, + MemberState::Dead, + false, + )], + ); assert!( !m.is_member(&d), "a higher-incarnation Dead does not resurrect a reaped member" ); - m.merge_liveness([(d.clone(), D.as_bytes().to_vec(), 99, MemberState::Suspect)]); + m.merge_liveness( + &sender(), + [( + d.clone(), + D.as_bytes().to_vec(), + 99, + MemberState::Suspect, + false, + )], + ); assert!(!m.is_member(&d), "nor does a Suspect"); } From 169b17f735f5895ad3cb5bbb8c713a81cb169f4c Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 19:36:46 -0300 Subject: [PATCH 02/17] test(server): pin that the ground id is one placement really puts on the room --- DECISIONS.md | 2 +- KANBAN.md | 2 +- crates/server/tests/adoption.rs | 35 +++++++++++++++++++++++++++------ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index dcab5163..01a77e40 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -7,7 +7,7 @@ Log of design changes to [ARCHITECTURE.md](ARCHITECTURE.md) that implementation The entries below (2026-07-02) are a backfill: design changes made during the v0.1→v0.2 build that predate this log, recovered from the sessions and commit history. -## 2026-07-28 · C25 member adoption (#PR) · rooms are placed on the members the cluster has *verified*, and a node introducing itself joins the roster rather than the ring +## 2026-07-28 · C25 member adoption (#369) · rooms are placed on the members the cluster has *verified*, and a node introducing itself joins the roster rather than the ring **Changed:** ARCHITECTURE gains §Member Adoption; §Peer Identity's "nor can identity alone decide who belongs in a room's replica set" paragraph is replaced by a pointer to it, §Room-Based Sharding says which member set is hashed over, and §Peer Identity's membership-growth clause says what a self-introduction and a gossip reply actually buy. diff --git a/KANBAN.md b/KANBAN.md index 6ed5d378..33805c08 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,7 +30,7 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ -**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#PR).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once `ADOPTION_VERIFIERS = 2` already-adopted members have verified it; the verifier sets are grow-only, so they merge by union and adoption is a pure fixpoint over them — convergent however rounds interleave, order-independent, pinned as such. **Two, not one:** one voucher is the attacker itself, and requiring a second forces an honest member to have independently *reached* the minted id, which also keeps an unreachable one — the shape that pollutes placement and stalls every quorum it lands in, the unit's second lesser way in — out of the ring entirely. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a genuine joiner); relaying another node's verifications was rejected (with no signatures on the wire, "A and B verified X" is free to write, which is the mint again). The bar clamps to the members that could vouch, so a small cluster still grows, and **configured members are adopted from birth** — the operator's config is the root of trust. **Three narrower rules close the rest.** A tuple claiming the sender itself is dropped, so no node can vouch for its own place. A claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. And a verification is admissible only where a member's advertise address and its node id name the same **host** — otherwise it is verified at one host and dialed at another — which closes the first lesser way in (the gossip *reply* half adopts a member at whatever address its tuple carries) permanently rather than transiently. **An inbound link verifies only behind a certificate that names the member**, since an uncertified inbound claim is a member asserting an id rather than answering at one and the cluster secret is deployment-wide; the outbound *direct* gossip round vouches either way — except that under that policy a **plaintext** member is not verified by a dial either (nothing was checked, and adopting on it hands rooms to a member whose every inbound link the same policy refuses), a gate that reads the member's advertised transport, which is sound outbound and is exactly what C13's pass 3 rejected inbound. That round is the honest floor — with no certificates configured, adoption means the minted id must be reachable and that no single member may place it, and nothing about identity, while `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` makes it mean identity and keeps the mint space bounded to the member's own host (already a shared trust unit per C13). An *indirect* (ping-req) confirmation is liveness only and vouches for nothing. Reaping takes a member's adoption and its vouches with it. C13's residual test is inverted and moved: `crates/server/tests/adoption.rs` (19 cases), covering the mint against all three gates, the threshold, the self-vouch, the pending-voucher, the host-coherence rule, both link directions, order-independence, reaping, and that a pending member is still dialed, probed and advertised. Wire: `Message::Gossip`'s tuple gains a trailing flag byte. See DECISIONS (2026-07-28). → *Cluster*. +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once `ADOPTION_VERIFIERS = 2` already-adopted members have verified it; the verifier sets are grow-only, so they merge by union and adoption is a pure fixpoint over them — convergent however rounds interleave, order-independent, pinned as such. **Two, not one:** one voucher is the attacker itself, and requiring a second forces an honest member to have independently *reached* the minted id, which also keeps an unreachable one — the shape that pollutes placement and stalls every quorum it lands in, the unit's second lesser way in — out of the ring entirely. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a genuine joiner); relaying another node's verifications was rejected (with no signatures on the wire, "A and B verified X" is free to write, which is the mint again). The bar clamps to the members that could vouch, so a small cluster still grows, and **configured members are adopted from birth** — the operator's config is the root of trust. **Three narrower rules close the rest.** A tuple claiming the sender itself is dropped, so no node can vouch for its own place. A claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. And a verification is admissible only where a member's advertise address and its node id name the same **host** — otherwise it is verified at one host and dialed at another — which closes the first lesser way in (the gossip *reply* half adopts a member at whatever address its tuple carries) permanently rather than transiently. **An inbound link verifies only behind a certificate that names the member**, since an uncertified inbound claim is a member asserting an id rather than answering at one and the cluster secret is deployment-wide; the outbound *direct* gossip round vouches either way — except that under that policy a **plaintext** member is not verified by a dial either (nothing was checked, and adopting on it hands rooms to a member whose every inbound link the same policy refuses), a gate that reads the member's advertised transport, which is sound outbound and is exactly what C13's pass 3 rejected inbound. That round is the honest floor — with no certificates configured, adoption means the minted id must be reachable and that no single member may place it, and nothing about identity, while `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` makes it mean identity and keeps the mint space bounded to the member's own host (already a shared trust unit per C13). An *indirect* (ping-req) confirmation is liveness only and vouches for nothing. Reaping takes a member's adoption and its vouches with it. C13's residual test is inverted and moved: `crates/server/tests/adoption.rs` (19 cases), covering the mint against all three gates, the threshold, the self-vouch, the pending-voucher, the host-coherence rule, both link directions, order-independence, reaping, and that a pending member is still dialed, probed and advertised. Wire: `Message::Gossip`'s tuple gains a trailing flag byte. See DECISIONS (2026-07-28). → *Cluster*. **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index 390e1db3..76d0d72a 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -134,9 +134,10 @@ fn introduces(node: &NodeId, verified: bool) -> Message { } } -/// The wire tuples introducing each of `nodes` at its own address, unverified — the -/// payload the *reply* half of a gossip round hands over, which introduces freely. -fn advertisements(nodes: &[&NodeId]) -> Vec<(Vec, Vec, u64, MemberState, bool)> { +/// The wire tuples introducing each of `nodes` at its own address, carrying the +/// sender's `verified` claim about each — the payload the *reply* half of a gossip +/// round hands over, which introduces freely. +fn advertisements(nodes: &[&NodeId], verified: bool) -> Vec { nodes .iter() .map(|node| { @@ -145,12 +146,21 @@ fn advertisements(nodes: &[&NodeId]) -> Vec<(Vec, Vec, u64, MemberState, node.as_bytes().to_vec(), 0, MemberState::Alive, - false, + verified, ) }) .collect() } +/// Have the cluster verify `node` to the bar, as `ADOPTION_VERIFIERS` members each +/// reporting their own completed link to it would. +fn cluster_verifies(r: &mut Registry, node: &NodeId) { + for i in 0..ADOPTION_VERIFIERS { + let voucher = NodeId::from(format!("10.0.0.{i}:9000")); + r.merge_gossip(&voucher, advertisements(&[node], true)); + } +} + /// A room this node is the placement primary of — the room an attack from outside /// the replica set targets. fn room_self_leads(m: &Membership) -> Vec { @@ -307,6 +317,16 @@ fn a_member_that_mints_a_node_id_does_not_enter_a_rooms_replica_set() { m.replicas_for(&room), "and the room's replica set is the one it always was", ); + + // The grind was real, and pending is the whole of what kept it out: verified to the + // bar by the cluster, that same id takes its place on that same room. + cluster_verifies(&mut r, &minted); + let view = r.membership().expect("clustered"); + assert!(view.is_adopted(&minted)); + assert!( + view.replicas_for(&room).contains(&minted), + "the id was one placement puts on the room", + ); } #[test] @@ -779,7 +799,7 @@ fn a_plaintext_dial_does_not_verify_where_an_identified_peer_is_required() { let tls = NodeId::from("wss://10.9.9.8:9000"); r.merge_gossip( &NodeId::from("10.0.0.1:9000"), - advertisements(&[&plain, &tls]), + advertisements(&[&plain, &tls], false), ); r.note_peer_verified(&plain); @@ -804,7 +824,10 @@ fn a_plaintext_dial_still_verifies_where_no_identity_is_required() { // saying an id exists. let mut r = registry(); let plain = NodeId::from("10.9.9.9:9000"); - r.merge_gossip(&NodeId::from("10.0.0.1:9000"), advertisements(&[&plain])); + r.merge_gossip( + &NodeId::from("10.0.0.1:9000"), + advertisements(&[&plain], false), + ); r.note_peer_verified(&plain); assert!(verified_by(r.membership().expect("clustered"), &plain)); } From e6113c491f3c298d2a3d2541bb3f751522834610 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 20:06:15 -0300 Subject: [PATCH 03/17] fix(server): adoption is derived from trust-unit evidence, and only this node's own dial verifies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review broke the first cut four ways, each of them the premise underneath a rule rather than the rule itself. - Sticky adoption made the ring a function of history: a member adopted before its vouchers were reaped stayed placed on the node that saw that order and was never placed on one that did not. The adopted set is now derived from the configured members plus the evidence, to a fixpoint, on every change. - Counting distinct verifier node ids was the mint one level up: a certificate names a host and a host mints unlimited ids, so one machine holding two of them raised the whole bar. The bar counts distinct verifier hosts and excludes the candidate's own. - An inbound link is no verification, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. - A member's recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it. A member is dialed at its own node id; the advertised address is dropped. The bar is also a constant now rather than clamped to the adopted count, with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by. On the reply half of a round a claim counts only where the dial established the sender. The residual is pinned as a passing test and filed as C27: a member that owns a host owns every id under it, and honest nodes verify one truthfully — closing that needs the ring to weigh trust units rather than node ids. --- ARCHITECTURE.md | 18 +- DECISIONS.md | 20 +- KANBAN.md | 5 +- crates/server/src/gossip.rs | 33 +++ crates/server/src/membership.rs | 189 +++++++++++------ crates/server/src/registry.rs | 109 +++++++--- crates/server/src/runtime.rs | 50 ++--- crates/server/tests/adoption.rs | 357 +++++++++++++++++++++++++++----- 8 files changed, 587 insertions(+), 194 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5351ac9c..d2341f97 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -964,17 +964,23 @@ The secret is a **bearer** credential, so the peer link carries it over an encry **Learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the placement ring is built from the **adopted** members alone. A member learned by gossip is *pending* — reachable and converging, but in no room's replica set and no room's quorum — until the cluster adopts it. Pending is not exile, and cannot be: a member nobody dials could never be verified, so a genuine joiner would never join. What it removes is the step from "a node said this id exists" to "rooms live on that id", which is the whole of the mint. -**Adoption is the cluster's decision, never one node's.** Placement must be identical on every node or the ring diverges and the cluster splits, so "verified" cannot be a local predicate over a shared member set — two nodes would place the same room differently forever. So the *evidence* is disseminated instead of the verdict: a node records only what it knows first-hand — that it has completed an identity-checked peer link to a member, its dial having authenticated the acceptor against the address it dialed, or the member having dialed in behind a certificate that names it — and that claim rides the same anti-entropy that carries liveness, as one flag per member on the gossip tuple. A member is placed once **two** already-adopted members have verified it. The evidence is a grow-only set of verifiers, so it merges by union and converges however rounds interleave, and adoption is a pure function of it. +**Adoption is the cluster's decision, never one node's.** Placement must be identical on every node or the ring diverges and the cluster splits, so "verified" cannot be a local predicate over a shared member set — two nodes would place the same room differently forever. So the *evidence* is disseminated instead of the verdict: a node records only what it knows first-hand, and that claim rides the same anti-entropy that carries liveness, as one flag per member on the gossip tuple. A member is placed once **two** already-adopted trust units have verified it. The evidence is a grow-only set of verifiers, so it merges by union. -**A claim is worth exactly the link that carried it.** A `verified` flag is recorded against the member the receiving link is bound to (§Peer Identity) and against nobody the payload names, so a member can assert its own verifications and no one else's; relaying another node's would make one member's word enough to place any id it liked. A tuple claiming the sender itself is dropped — a node's place in the ring is never its own to assert — and a claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. The cost of first-hand-only is that verifications travel by direct exchange rather than by relay: a joiner is placed once enough members have each reached it, which randomized peer selection delivers within a few gossip rounds rather than one. +**The adopted set is derived, never accumulated.** It is recomputed from the configured members plus the verifier evidence every time either changes, so it is a pure function of state: two nodes holding the same evidence hold the same ring however they came by it. Accumulating instead — adopting on the way past and never revisiting — would make the ring a function of *history*: a member adopted before its vouchers were reaped would stay placed on the node that saw that order and never be placed on one that did not, and the split would be permanent. -**Two members are the bar because one member is the attacker.** A single compromised member would otherwise vouch for the id it ground and place it itself; requiring a second means an honest member must have independently reached that id, which also keeps an unreachable id — one that would pollute placement and stall every quorum it landed in — out of the ring entirely. The bar clamps to the number of members that could possibly vouch, so a cluster too small to raise two can still grow. **Configured members are adopted from birth**: the operator's config is the root of trust a cluster starts from, and there is no earlier authority for it to be vouched for by. +**Only this node's own dial verifies.** The dial goes to the address the member's id names and the transport authenticates the far end before a byte is written; that is an act this node chose to take. An *inbound* link never verifies the member it names, however well a certificate names it — a member chooses when to dial in and how often, so a vouch earned that way is one the member caused rather than one this node made, and a certificate names a host, which mints as many node ids as it likes, so a member could dial in under each ground id in turn and be vouched for under every one. An *indirect* (ping-req) confirmation is liveness only: it says a relay reaches the target, which this node did not observe and cannot attribute. -**A member is verified at the host its own id names.** A verification is admissible only where the member's advertise address and its node id name the same host — otherwise it would be verified at one host and dialed at another, and no link to either would vouch for it. A node id *is* an advertise address, so that costs a legitimate member nothing; what it closes is the gossip *reply* half, which adopts a member at whatever address its tuple carries because it comes from a node this one chose to dial. Such a member stays pending permanently rather than transiently. +**A verifier is a trust unit, and a trust unit is a host.** A host is what a certificate names (§Peer Identity) and one host holds as many node ids as it likes, so the bar counts the distinct *hosts* of the adopted members that vouched, not the ids. Counting ids would let one machine holding two of them raise the whole bar by itself. A member's own host is excluded from its own count for the same reason: a member vouching for a sibling on its own host is vouching for itself. -**Adoption is only as strong as the identity beneath it.** Under peer mTLS a verification means a certificate for that member's host answered there, so the mint space stays bounded by what §Peer Identity bounds it to — the member's own host, which is already a shared trust unit. With no certificates configured a verification means only that *something* answers at the address the id names, which still requires the minted id to be reachable and still denies a single member the power to place it, but vouches for no identity; `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` is what makes it vouch. For the same reason an *inbound* link verifies the member it names only when a certificate named it: an uncertified inbound claim is a member asserting an id rather than answering at one, and the deployment-wide secret makes that free. And under that policy an outbound dial to a **plaintext** member verifies nothing either — there was no certificate to check, and adopting on it would give rooms to a member whose every inbound link the cluster refuses, stalling their quorums. That reads a member's advertised transport, which is sound here and is not sound inbound: an outbound dial runs over exactly the transport that address declares, while an inbound link is one the member dialed and its own listener's scheme describes nothing about it. +**A claim is worth exactly the link that carried it.** A `verified` flag is recorded against the member the receiving link is bound to (§Peer Identity) and against nobody the payload names, so a member can assert its own verifications and no one else's; relaying another node's would make one member's word enough to place any id it liked. A tuple claiming the sender itself is dropped — a node's place in the ring is never its own to assert — and a claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. On the *reply* half of a round, the claim counts only where the dial established who answered: under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a `wss://` member's certificate does and a plaintext member's transport does not, and an unattributable claim must not become an adopted member's vouch. The cost of first-hand-only is that verifications travel by direct exchange rather than by relay: a joiner is placed once enough members have each reached it, which randomized peer selection delivers in a handful of gossip rounds rather than one. -**Reaping takes adoption and vouches with it.** A member removed from the roster leaves the adopted set, its verifier set is dropped, and it is struck from every other member's — a departed node keeps vouching for nobody. A return is a fresh join and is verified again. +**Two trust units, because one of them is the attacker.** A single compromised member would otherwise vouch for the id it ground and place it itself; requiring a second means an honest member must have independently reached that id, which also keeps an unreachable id — one that would pollute placement and stall every quorum it landed in — out of the ring entirely. The bar is a **constant**, not a fraction of the cluster, so two nodes never disagree about what the evidence has to show. **Configured members are adopted from birth**: the operator's config is the root of trust a cluster starts from, and there is no earlier authority for it to be vouched for by. The one exception is a node configured with *no peers at all*, which has no cluster to be outvoted by and which the constant would freeze forever — its ring is the members it has itself reached. That is the single-node deployment; give every node that is meant to join a cluster a seed peer, or it takes its ring from whoever reaches it. + +**A member is dialed at its own node id.** A node id *is* an advertise address, so a second address for the same member is a second, unauthenticated name for one thing — and the ring turns on it, because a node dials a member in order to verify it. Keeping the advertised address made the roster first-write-wins over a field any peer may set: whoever advertised a member first decided where every later dial went, so two nodes that saw it in a different order verified different endpoints and placed rooms differently, forever. The address a gossip tuple carries is therefore dropped, which also ends the *reply* half's freedom to point the ring anywhere. + +**Reaping takes adoption and vouches with it.** A member removed from the roster leaves the configured set, its verifier set is dropped, and it is struck from every other member's, so the ring is recomputed without its word. A return is a fresh join and is verified again. + +**Adoption is only as strong as the identity beneath it, and it does not bound a host.** Under peer mTLS a verification means a certificate for that member's host answered there, so no member can manufacture another's verification. **With no certificates configured none of that holds**: a verification means only that *something* answers at the address the id names, and a secret-holder can bind a link to any member id it likes, so it can raise the whole bar itself. `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` is what makes the bar a bar. Even then one limit remains, deliberately: **placement keys on node ids and a certificate names a host**, so a member that owns a host owns every id under it — it answers at each ground id, and the honest nodes that dial it verify one truthfully. Adoption bounds the mint to the member's own host and to ids the cluster can actually reach; it cannot bound it further, because every verification involved is genuine. Closing that needs the ring to weigh trust units rather than ids — a placement change, not an evidence one. ## Peer Transport diff --git a/DECISIONS.md b/DECISIONS.md index 01a77e40..c4aa91c7 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -7,7 +7,7 @@ Log of design changes to [ARCHITECTURE.md](ARCHITECTURE.md) that implementation The entries below (2026-07-02) are a backfill: design changes made during the v0.1→v0.2 build that predate this log, recovered from the sessions and commit history. -## 2026-07-28 · C25 member adoption (#369) · rooms are placed on the members the cluster has *verified*, and a node introducing itself joins the roster rather than the ring +## 2026-07-28 · C25 member adoption (#369) · rooms are placed on members the cluster has verified, adoption is derived from that evidence rather than accumulated, and a verifier is a trust unit rather than a node id **Changed:** ARCHITECTURE gains §Member Adoption; §Peer Identity's "nor can identity alone decide who belongs in a room's replica set" paragraph is replaced by a pointer to it, §Room-Based Sharding says which member set is hashed over, and §Peer Identity's membership-growth clause says what a self-introduction and a gossip reply actually buy. @@ -15,17 +15,23 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Why a local check was not available.** The obvious fix — place rooms only on members this node has itself verified — splits the cluster. Placement must be identical on every node or two nodes disagree about who leads a room permanently rather than for a propagation window, and "verified" is per-node by construction: each node has completed a different set of links, so the predicate never converges. The ring has to be a function of state that *does* converge. -**So the evidence is disseminated, not the verdict.** A node records only what it knows first-hand — it completed an identity-checked peer link to a member — and that claim rides the same anti-entropy that already carries liveness, as one `verified` flag per tuple on `Message::Gossip`. A receiver records the flag **against the member its link is bound to** and against nobody the payload names, so a member can assert its own verifications and no one else's. A member is adopted once `ADOPTION_VERIFIERS = 2` already-adopted members have verified it. The evidence is a grow-only set of verifiers per member, so it merges by union and adoption is a pure function of it — convergent however rounds interleave, and computed to a fixpoint so adopting a member makes its own claims count without the outcome depending on the order members were visited in. +**So the evidence is disseminated, not the verdict.** A node records only what it knows first-hand — it completed an identity-checked peer link to a member — and that claim rides the same anti-entropy that already carries liveness, as one `verified` flag per tuple on `Message::Gossip`. A receiver records the flag **against the member its link is bound to** and against nobody the payload names, so a member can assert its own verifications and no one else's. A member is adopted once two already-adopted trust units have verified it. -**Two, not one, and not a majority.** One is the attacker: a single compromised member would vouch for the id it ground and place it itself. Two forces an honest member to have independently reached that id, which also keeps an *unreachable* minted id — the one that pollutes placement and stalls every quorum it lands in — out of the ring entirely. A majority was rejected: it buys nothing here, because an honest node dials every member it knows anyway, so the marginal verifier past the second is a formality that only delays a genuine joiner. The bar clamps to the number of members that could vouch, so a cluster smaller than it can still grow. Configured members are adopted from birth — the operator's config is the root of trust, and a cluster whose own seeds were pending could place no room at all. +**Adoption is derived, never accumulated — the first thing cold review overturned.** The first cut adopted on the way past and kept the member adopted ("sticky", so that a ring would not un-place a node the cluster still holds). That makes the ring a function of *history*: reaping a member removes its vouch, so a node that adopted a joiner before the reap keeps it placed and a node that saw the evidence after never places it, and the split is permanent. The adopted set is now recomputed from `configured` + `verifiers` on every change, to a fixpoint (adopting a member makes its own vouches count), so it is a pure function of state. -**Relaying verifications was rejected.** It would let adoption spread in one hop instead of by direct exchange, and it is exactly what a compromised member wants: with no signatures on the wire, "A and B verified X" is free to write. First-hand-only costs a joiner a few extra gossip rounds — randomized peer selection reaches every member — and is the only form of the claim that a receiver can attribute. +**A verifier is a trust unit, and a trust unit is a host — the second thing review overturned.** Counting distinct *node ids* was the same mistake one level up: a certificate names a host (C13) and a host mints as many ids as it likes, so a member holding two ids on one host raised the whole bar by itself. The bar now counts the distinct hosts of the adopted members that vouched, and excludes the candidate's own host — a member vouching for a sibling on its own host is vouching for itself. -**A verification is admissible only where a member's advertise address and its node id name the same host.** Otherwise the member is verified at one host and dialed at another and no link to either vouches for it. A node id *is* an advertise address, so this costs a legitimate member nothing; what it closes, permanently rather than transiently, is the reply half introducing an id that points somewhere else. +**An inbound link verifies nothing — the third.** The first cut treated an admitted link whose certificate named the member as a verification. It is not: the *member* chooses when to dial in and how often, so the vouch is one it caused rather than one this node made, and because the certificate names a host it could dial in under each ground id in turn and be vouched for under every one. Verification is this node's own dial and nothing else; an indirect (ping-req) confirmation is liveness only, since a relay reaching a target is not something this node observed. -**An inbound link verifies only behind a certificate.** A dial proves something answers at the address the id names; an inbound claim proves only that the sender reached here, and the cluster secret is deployment-wide, so treating an uncertified inbound claim as verification would let any secret-holder vouch for any id at no cost. Under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` an outbound dial to a **plaintext** member verifies nothing either, for the same reason: nothing was checked, and adopting on it hands rooms to a member whose every inbound link that same policy refuses, stalling their quorums. That gate reads the member's advertised transport, which C13's pass 3 rejected for the *inbound* direction and which is sound here — an outbound dial runs over exactly the transport the address declares, while an inbound link is one the member dialed and its own listener's scheme describes nothing about it. The outbound direction still vouches without certificates, which is the honest floor: with no certificates configured, adoption means the minted id must be reachable and no single member may place it, and nothing about identity. `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` is what makes it mean identity, and under it the mint space stays what §Peer Identity bounds it to — the member's own host, already a shared trust unit. +**A member is dialed at its own node id — the fourth.** A member's recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, because a node dials a member in order to verify it. Whoever advertised a member first decided where every later dial went, so two nodes that saw it in a different order verified different endpoints and placed rooms differently, permanently — reachable by advertising any joiner at a bogus address. A node id *is* an advertise address, so the second name is simply dropped: the dial address is now a function of the id alone, which every node agrees on by construction. That also ends the reply half's freedom to point the ring anywhere, at the root rather than by a filter. -**Pending is not exile.** A member the cluster has not adopted stays on the roster: dialed, probed, gossiped about and advertised back. It has to be, or nobody would ever reach it and no genuine joiner would ever join. `Membership::members` is therefore the roster and `adopted_members` the ring; the split is the whole change, and every gate that reads `replicas_for` inherited it without a line of its own. +**The bar is a constant.** Clamping it to the adopted-member count made the bar itself node-local and non-monotonic: a node configured with no peers used a bar of one, and reaping could *lower* the bar and retroactively adopt a member on a stale vouch. It is now the same constant everywhere, with one exception keyed on configuration rather than on the running set — a node configured with **no peers at all** has no cluster to be outvoted by and the constant would freeze it forever, so its ring is what it has itself reached. That is the single-node deployment; a node meant to join a cluster is given a seed peer, which is adopted from birth and lifts it past the case at boot. + +**Two, not one, and not a majority.** One is the attacker: a single compromised member would vouch for the id it ground and place it itself. Two forces an honest member to have independently reached that id, which also keeps an *unreachable* minted id — the one that pollutes placement and stalls every quorum it lands in — out of the ring entirely. A majority was rejected: it buys nothing here, because an honest node dials every member it knows anyway, so the marginal verifier past the second is a formality that only delays a genuine joiner. Relaying verifications was rejected too: with no signatures on the wire, "A and B verified X" is free for a compromised member to write, and first-hand-only is the only form a receiver can attribute. The cost is that verifications travel by direct exchange, so a joiner is placed in a handful of rounds rather than one. + +**A claim is worth the link that carried it, on both halves of a round.** On the inbound half the link is bound to the sender (C13). On the *reply* half — a payload from a node this one dialed — the dial is what establishes the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped: nothing authenticated them, and an unattributable claim must not become an adopted member's word. That reads the member's advertised transport, which C13's pass 3 rejected for the inbound direction and which is sound here — an outbound dial runs over exactly the transport the address declares, while an inbound link is one the member dialed and its own listener's scheme describes nothing about it. The liveness in the same payload still merges: reachability is nobody's identity. + +**The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. diff --git a/KANBAN.md b/KANBAN.md index 33805c08..e1cab53c 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,7 +30,7 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ -**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once `ADOPTION_VERIFIERS = 2` already-adopted members have verified it; the verifier sets are grow-only, so they merge by union and adoption is a pure fixpoint over them — convergent however rounds interleave, order-independent, pinned as such. **Two, not one:** one voucher is the attacker itself, and requiring a second forces an honest member to have independently *reached* the minted id, which also keeps an unreachable one — the shape that pollutes placement and stalls every quorum it lands in, the unit's second lesser way in — out of the ring entirely. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a genuine joiner); relaying another node's verifications was rejected (with no signatures on the wire, "A and B verified X" is free to write, which is the mint again). The bar clamps to the members that could vouch, so a small cluster still grows, and **configured members are adopted from birth** — the operator's config is the root of trust. **Three narrower rules close the rest.** A tuple claiming the sender itself is dropped, so no node can vouch for its own place. A claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. And a verification is admissible only where a member's advertise address and its node id name the same **host** — otherwise it is verified at one host and dialed at another — which closes the first lesser way in (the gossip *reply* half adopts a member at whatever address its tuple carries) permanently rather than transiently. **An inbound link verifies only behind a certificate that names the member**, since an uncertified inbound claim is a member asserting an id rather than answering at one and the cluster secret is deployment-wide; the outbound *direct* gossip round vouches either way — except that under that policy a **plaintext** member is not verified by a dial either (nothing was checked, and adopting on it hands rooms to a member whose every inbound link the same policy refuses), a gate that reads the member's advertised transport, which is sound outbound and is exactly what C13's pass 3 rejected inbound. That round is the honest floor — with no certificates configured, adoption means the minted id must be reachable and that no single member may place it, and nothing about identity, while `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` makes it mean identity and keeps the mint space bounded to the member's own host (already a shared trust unit per C13). An *indirect* (ping-req) confirmation is liveness only and vouches for nothing. Reaping takes a member's adoption and its vouches with it. C13's residual test is inverted and moved: `crates/server/tests/adoption.rs` (19 cases), covering the mint against all three gates, the threshold, the self-vouch, the pending-voucher, the host-coherence rule, both link directions, order-independence, reaping, and that a pending member is still dialed, probed and advertised. Wire: `Message::Gossip`'s tuple gains a trailing flag byte. See DECISIONS (2026-07-28). → *Cluster*. +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. @@ -425,6 +425,9 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C16 — the reveal back-fill's batch dedup drops the copy that was zone-protected (crates/server) — READY, no dependencies. Found by cold review during C11 (#359), reasoned not reproduced.** The live fan-out's reveal prefix force-stamps every shell and back-filled op with the placing move's zone, precisely so the per-channel `zone_filter` cannot drop the content while keeping the shell. C11 made the back-fill skip any op the batch already carries — correct, since the duplicate took that op out of its transaction's count — but the surviving copy is then the *batch's*, which carries the zone its author computed at emit time. `zone_of_op` resolves against the live tree, so in one `atomic_transact` an edit inside a node carries the node's origin zone while the move carries the destination's: the per-channel filter can now drop the content and keep the shell, leaving the reader with a materialised node it never fills. Reaching it needs a node crossing zones, which a named-branch write does un-tokened (`handle_ops` gates `batch_zone_crossings` on `branch == MAIN_BRANCH`). Wants either the batch copy stamped with the move's zone the way the prefix copy was, or the zone filter taught to keep a shell and its content together by node rather than by zone. The `registry.rs` comment claiming the co-travel guarantee overstates it until then. → *Server / Fan-out*. +**C27 — a member that owns a host owns every ring position under it, because placement keys on node ids while a certificate names a host (crates/server) — READY, no dependencies. Found by cold review during C25 (#369) and pinned there as a passing test (`a_certified_member_still_mints_ids_on_its_own_host`).** C25 made rooms place on *adopted* members and made adoption the cluster's decision, which closes minting on anyone else's host, minting an id nobody can reach, and any single member manufacturing the evidence. What it cannot close is the mint on a member's **own** host: HRW scores a node id, C13 binds a link to the *host* a certificate names, and a host answers at as many ports as it likes — so a compromised member grinds `own-host:` until HRW places it on the room it wants, answers there, and every honest node that dials it verifies it **truthfully**. No evidence rule can help, because no one is lying. The consequence is the one C25 removed everywhere else: a member chooses which rooms it replicates, and from inside the replica set it reaches the leadership and durability gates. **The fix is a placement change, not an evidence one: weigh the ring by trust unit.** Score hosts rather than node ids, so a room's replica set is `n` distinct hosts and which rooms a host holds follows from a name it cannot grind; the member serving a host's position is then a local choice among that host's adopted members, and it does not matter which, since they are one trust unit already. That is also the better replication property — `n` replicas on `n` failure domains instead of `n` ids that may share a machine — and it makes the `replicas_for(room).contains(sender)` gate ask about exactly what the certificate vouches for. Costs to settle: two members on one host stop both replicating a room (a capacity change, not a correctness one), the replication factor becomes a host count, and every gate/redirect/quorum that names a member has to resolve host → member. Note this is *not* reachable without a compromised member's certificate: without peer mTLS the whole binding vouches for nothing anyway (C13's residual), and with it the mint is bounded to a host the operator already trusts to run a node. → *Cluster*. + +**C26 — the SDK apply seams report a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — READY, no dependencies. Split out of C19 (#367) rather than folded in.** `crdtsync_client_receive`/`WasmClient::receive` fold a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learns how many ops applied *now* and nothing else. C19 made the distinction answerable — `Op::is_admissible` is public and pure — but did not plumb it: a buffered op (which a later arrival commits) and an op no replica will ever hold are both reported as "not counted", so an SDK peered with a buggy or hostile writer sees edits vanish with no signal, and the two cases call for opposite responses (wait vs. surface a client bug). `client.rs` needs no change on this axis — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it, so nothing is missed. The server side is closed: `handle_ops` answers `ErrorCode::MalformedOp` and the ingest seams drop such a record, so this is the *offline / P2P / relay* seam, where no server sits between the writer and the fold. Wants a refused count (or a refused-op list) beside the applied one across FFI, wasm, JS, Python and Go — and is worth taking with **C23**, which has the same shape: a real refusal the SDKs render as an edit that silently did nothing. → *SDK / Ops*. **C14 — a redacted op delta leaves the same re-mint hole, and no frame can carry the ids it withholds (crates/server + crates/core) — READY, no dependencies. Found during C9 (#357), reproduced.** C9 closed the *snapshot* seam: a projected snapshot now names the recipient's own ids, so adoption never frees one the room's log holds. An **uncompacted** room serves an op delta instead, and the per-op read filter withholds the recipient's own ops on paths it may no longer read — leaving a hole in its own run with nothing to repair it from. Measured on C9's own fixture: a reader with write at `/` and read on `/a` alone writes into `/b` (seqs 0,1) then `/a` (seq 2); it restarts, its delta withholds `/b`, `next_seq` reports 0, and its next write mints seq 0 straight onto its own withheld id — the write never lands. The state encoding was the carrier for the snapshot case; an `Ops` frame has no equivalent, so closing this needs one: a frontier field on the delta, or a rule that a reader's *own* ops are never withheld from it (which trades the hole for materialising ids in a subtree it may not read — the reason C9 did not simply stop scrubbing). Note the second shape also has to answer what an op carries beyond its id: its target and content may name denied structure. → *Server / Fan-out*. diff --git a/crates/server/src/gossip.rs b/crates/server/src/gossip.rs index e2852012..4d4331b4 100644 --- a/crates/server/src/gossip.rs +++ b/crates/server/src/gossip.rs @@ -227,6 +227,39 @@ pub fn probe_outcome(direct_reachable: bool, indirect: &[Option]) -> bool direct_reachable || indirect_reachable(indirect) } +/// How one SWIM probe round reached its target — the whole verdict a round carries +/// back, since liveness and identity are different questions and a round answers them +/// differently. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum GossipRoundOutcome { + /// This node dialed the member and it answered. On a TLS member the transport + /// authenticated the far end against the address dialed before a byte was + /// written, so this is the only outcome that is evidence about *who* is there. + Direct, + /// The direct probe failed but a relay reported the target reachable. Evidence + /// that the member is alive, and about nobody's identity: it says a *relay* + /// reaches the target, which this node did not observe and cannot attribute. + Relayed, + /// Neither the direct probe nor any relay reached the member. + Unreachable, +} + +impl GossipRoundOutcome { + /// Fold a round's direct and indirect results into its outcome. + pub fn of(direct_reachable: bool, indirect: &[Option]) -> Self { + match (direct_reachable, indirect_reachable(indirect)) { + (true, _) => GossipRoundOutcome::Direct, + (false, true) => GossipRoundOutcome::Relayed, + (false, false) => GossipRoundOutcome::Unreachable, + } + } + + /// Whether the round found the member alive, by either path. + pub fn reachable(self) -> bool { + self != GossipRoundOutcome::Unreachable + } +} + /// Ask the relay at `relay_addr` to report whether it can reach `target_addr` on /// this node's behalf: dial it, send a [`Message::PingReq`], and read back the /// [`Message::PingAck`]'s verdict. `Some(reachable)` is the relay's answer — its diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index 614b3ecc..67641b1b 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -136,8 +136,10 @@ pub const DEFAULT_REPLICATION_FACTOR: usize = 3; /// grinding a node id needs an honest member to have independently reached that id /// too. Small, because every honest member probes every member it knows on its own /// gossip cadence, so a genuine joiner clears the bar within a few rounds. Clamped -/// to the adopted-member count ([`Membership::adoption_quorum`]) so a cluster too -/// small to raise this many verifiers can still grow. +/// Not clamped to the cluster's size: the bar is the same constant on every node, so +/// two nodes never disagree about what the evidence has to show. A cluster with fewer +/// than this many adopted members therefore grows by *configuration* rather than by +/// gossip — which is where a cluster that small gets its members anyway. pub const ADOPTION_VERIFIERS: usize = 2; /// A malformed static membership configuration, surfaced at startup instead of a @@ -216,12 +218,16 @@ pub struct Membership { /// retention outlives any in-flight gossip that could reference the member, so the /// prune never resurrects it. reaped: HashMap, + /// The members this node was *configured* with — `self` and the seed peers. The + /// root of trust a cluster starts from, adopted from birth because there is no + /// earlier authority for them to be vouched for by. + configured: HashSet, /// The members rooms are actually placed on — the subset of the roster the - /// [`Cluster`] above is built from. `self` and every *configured* member are - /// adopted from birth; a member learned by gossip joins once - /// [`ADOPTION_VERIFIERS`] adopted members have verified it. Adoption is sticky - /// until the member is reaped: a ring that un-placed a member on evidence going - /// stale would move rooms off a node the cluster still holds. + /// [`Cluster`] above is built from: the configured members plus every + /// gossip-learned member the evidence now carries. **Derived, never accumulated** + /// — [`rebuild_placement`](Self::rebuild_placement) recomputes it from + /// `configured` + `verifiers` — so it is a pure function of state and two nodes + /// holding the same evidence hold the same ring, however they came by it. adopted: HashSet, /// Who has verified each member: for every member, the nodes that reported /// completing an identity-checked peer link to it. A node inserts itself here @@ -261,7 +267,7 @@ impl Membership { // A configured member is adopted from birth: the operator's config is the // root of trust a cluster starts from, and there is no earlier authority for // it to be vouched for by. - let adopted: HashSet = members.iter().cloned().collect(); + let configured: HashSet = members.iter().cloned().collect(); Self { self_id, cluster: Cluster::new(members), @@ -270,7 +276,8 @@ impl Membership { liveness, addrs, reaped: HashMap::new(), - adopted, + adopted: configured.clone(), + configured, verifiers: HashMap::new(), } } @@ -322,8 +329,9 @@ impl Membership { self.cluster.nodes() } - /// Learn a member, dialable at `addr` — the anti-entropy union gossip applies - /// for each `(node, addr)` pair a peer advertises. See [`add_members`](Self::add_members). + /// Learn a member — the anti-entropy union gossip applies for each `(node, addr)` + /// pair a peer advertises. The advertised `addr` is **not** what the member is + /// dialed at; see [`add_members`](Self::add_members). pub fn add_member(&mut self, node: NodeId, addr: Vec) { self.add_members(std::iter::once((node, addr))); } @@ -335,6 +343,16 @@ impl Membership { /// set changes nothing (no churn). A member with an empty node id is dropped — it /// is neither placeable nor dialable, so a malformed gossip pair cannot poison the /// set. `self` is a member from construction and is never relearned. + /// + /// **A member is recorded at its own id, whatever address the pair carries.** A + /// node id *is* an advertise address, so a second address for the same member is a + /// second, unauthenticated name for one thing — and the ring turns on it, because a + /// node dials a member to verify it. Keeping the advertised one made the roster + /// first-write-wins over a field any peer may set: whoever advertised a member + /// first decided where every later dial went, so two nodes that saw it in a + /// different order verified different endpoints and placed rooms differently, + /// forever. Ignoring it makes the dial address a function of the id alone, which + /// every node agrees on by construction. pub fn add_members(&mut self, members: impl IntoIterator)>) { let mut added = false; for (node, addr) in members { @@ -348,9 +366,11 @@ impl Membership { { continue; } + let _ = addr; self.liveness .insert(node.clone(), MemberLiveness::new(0, MemberState::Alive)); - self.addrs.insert(node, addr); + let dial = node.as_bytes().to_vec(); + self.addrs.insert(node, dial); added = true; } if added { @@ -516,14 +536,22 @@ impl Membership { .collect() } + /// Whether `verifier` has reported a completed identity-checked peer link to + /// `node` — one entry of the evidence adoption is decided from. A claim is + /// admitted only from the member whose link carried it, so this reads "what + /// `verifier` itself told this node about `node`", never a relayed opinion. + pub fn has_verified(&self, verifier: &NodeId, node: &NodeId) -> bool { + self.verifiers + .get(node) + .is_some_and(|vs| vs.contains(verifier)) + } + /// Whether this node has itself verified `node`. `self` is not verified by /// itself — a node vouching for its own place in the ring is exactly what /// adoption exists to refuse — and needs no vouching, since a node is a member of /// its own view from construction. fn verified_by_self(&self, node: &NodeId) -> bool { - self.verifiers - .get(node) - .is_some_and(|vs| vs.contains(&self.self_id)) + self.has_verified(&self.self_id, node) } /// Whether `node` is adopted — placed on rooms and counted toward their quorums, @@ -538,16 +566,13 @@ impl Membership { /// evidence a node produces on its own; adoption is what the cluster does with /// enough of them. /// - /// Three members are never verified. `self`, because a node's own place in the - /// ring is not its to vouch for. A node outside the roster, because a link is + /// Two members are never verified. `self`, because a node's own place in the ring + /// is not its to vouch for. And a node outside the roster, because a link is /// evidence about a member and there is no member here to be evidence about — - /// learning one is gossip's job, and a link that could add one would restore the - /// unchecked join this exists to close. And a member whose advertise address - /// names a different **host** than its own node id: it would be verified at one - /// host and dialed at another, so no link to either can vouch for it. A node id - /// *is* an advertise address, so that costs a legitimate member nothing — it is - /// the reply half of a gossip round, which adopts a member at whatever address - /// its tuple carries, that can introduce the mismatch. + /// learning one is gossip's job, a link that could add one would restore the + /// unchecked join this exists to close, and peer admission takes the id a link + /// claims, so a certified member could otherwise open a link per port on its own + /// host and grow the evidence without limit. pub fn note_verified(&mut self, node: &NodeId) { if !self.can_be_verified(node) { return; @@ -562,53 +587,57 @@ impl Membership { /// verifier claim clears, this node's own and a peer's alike. See /// [`note_verified`](Self::note_verified) for what each clause refuses. fn can_be_verified(&self, node: &NodeId) -> bool { - if self.is_self(node) { - return false; - } - let Some(addr) = self.addrs.get(node) else { - return false; - }; - matches!( - (member_host(addr), member_host(node.as_bytes())), - (Some(addr_host), Some(id_host)) if addr_host == id_host - ) - } - - /// How many adopted members must vouch for a pending one before it is adopted: - /// [`ADOPTION_VERIFIERS`], clamped to the number of members that could possibly - /// vouch. Without the clamp a cluster smaller than the constant could never grow - /// — a single-node cluster has nobody but itself to verify a joiner — and with it - /// the bar is always "every adopted member that exists, up to the constant". - fn adoption_quorum(&self) -> usize { - ADOPTION_VERIFIERS.min(self.adopted.len()) + !self.is_self(node) && self.addrs.contains_key(node) } - /// How many *adopted* members have verified `node`. A claim by a member that is - /// itself pending does not count: two nodes minted together would otherwise vouch - /// each other into the ring without any established member ever reaching either. - fn adopted_verifier_count(&self, node: &NodeId) -> usize { - self.verifiers - .get(node) - .map(|vs| vs.iter().filter(|v| self.adopted.contains(*v)).count()) - .unwrap_or(0) - } - - /// Adopt every pending member the evidence now carries, then rebuild the - /// [`Cluster`] over the adopted set. Run to a fixpoint, because adopting a member - /// makes its own verifications count: each round admits every member that meets - /// the bar against the same adopted set, so the outcome is a function of the - /// merged state alone and never of the order the rounds happened to visit - /// members in. Placement is then rebuilt from the canonicalized (sorted, - /// de-duped) adopted set, so two nodes holding the same evidence place every room - /// identically. + /// How many **trust units** have vouched for `node`: the distinct hosts of the + /// adopted members that verified it, its own host excluded. + /// + /// Hosts rather than node ids, because a host is the unit a certificate names + /// (§Peer Identity) and one host mints as many node ids as it likes. Counting ids + /// would let a member that holds two of them on one host raise the bar by itself, + /// which is the mint one level up. Its own host is excluded for the same reason: a + /// member vouching for a sibling on its own host is vouching for itself. A member + /// whose id names no host counts for nobody — there is no unit to attribute it to. + fn verifier_units(&self, node: &NodeId) -> usize { + let Some(own_host) = member_host(node.as_bytes()) else { + return 0; + }; + let Some(verifiers) = self.verifiers.get(node) else { + return 0; + }; + verifiers + .iter() + .filter(|v| self.adopted.contains(*v)) + .filter_map(|v| member_host(v.as_bytes())) + .filter(|host| host != &own_host) + .collect::>() + .len() + } + + /// Recompute which members rooms are placed on, then rebuild the [`Cluster`] over + /// them. **Derived from scratch, never accumulated:** the adopted set is the + /// configured members plus every roster member [`ADOPTION_VERIFIERS`] adopted + /// trust units have verified, so it is a pure function of `configured` + + /// `verifiers` and two nodes holding the same evidence hold the same ring however + /// they came by it. Accumulating instead would make it a function of history — + /// a member adopted before its vouchers were reaped would stay placed on the node + /// that saw that order and never be placed on one that did not, and the ring would + /// split permanently. + /// + /// Run to a fixpoint, because adopting a member makes its own verifications count: + /// each round admits every member that meets the bar against the same adopted set, + /// so the outcome never depends on the order members were visited in. Terminates + /// because each round strictly grows a set bounded by the roster. fn rebuild_placement(&mut self) { + self.adopted.clone_from(&self.configured); + let bar = self.adoption_bar(); loop { - let quorum = self.adoption_quorum(); let newly: Vec = self .addrs .keys() .filter(|node| !self.adopted.contains(*node)) - .filter(|node| self.adopted_verifier_count(node) >= quorum) + .filter(|node| self.verifier_units(node) >= bar) .cloned() .collect(); if newly.is_empty() { @@ -619,6 +648,25 @@ impl Membership { self.cluster = Cluster::new(self.adopted.iter().cloned()); } + /// How many trust units must vouch for a member before it is placed: + /// [`ADOPTION_VERIFIERS`], except for a node **configured with no peers at all**. + /// + /// That exception is narrow and it keys on configuration, not on the running set, + /// so it is a fixed property of a node rather than a bar that moves as the ring + /// grows. A node whose config names no peer has no cluster to be outvoted by, and + /// the constant would freeze it forever — a second voucher can only ever come from + /// a member it has already adopted — so the cluster it places on is exactly the + /// members it has itself reached. That is the single-node deployment; a node + /// meant to join a cluster is given a seed peer, which is adopted from birth and + /// lifts it past this case at boot. Configure one: a node with none takes the ring + /// from whoever reaches it. + fn adoption_bar(&self) -> usize { + match self.configured.len() <= 1 { + true => 1, + false => ADOPTION_VERIFIERS, + } + } + /// Record a *successful* direct gossip exchange with `node` — first-hand proof /// it is alive. Clears its failed-probe count and restores it to `Alive` at the /// known incarnation. It does **not** bump the incarnation — only the member @@ -705,11 +753,12 @@ impl Membership { self.liveness.remove(node); self.addrs.remove(node); self.relay_down.remove(node); - // A reaped member is no longer placed and no longer vouches: it leaves - // the adopted set, its own verifier set goes with it, and it is struck - // from every other member's. Were it to return it would be a fresh join - // and would have to be verified again. - self.adopted.remove(node); + // A reaped member no longer vouches: its own verifier set goes with it and + // it is struck from every other member's, so the ring is recomputed without + // its word. It leaves `configured` too — a configured member that departed + // durably is gone, and keeping it would place rooms on a node the roster no + // longer holds. Were it to return it would be a fresh join, verified again. + self.configured.remove(node); self.verifiers.remove(node); for vs in self.verifiers.values_mut() { vs.remove(node); @@ -793,9 +842,11 @@ impl Membership { } match self.liveness.get_mut(&node) { None => { + let _ = addr; self.liveness .insert(node.clone(), MemberLiveness::new(incarnation, state)); - self.addrs.insert(node, addr); + let dial = node.as_bytes().to_vec(); + self.addrs.insert(node, dial); rebuilt = true; } Some(m) => { diff --git a/crates/server/src/registry.rs b/crates/server/src/registry.rs index b1f851bf..99b56674 100644 --- a/crates/server/src/registry.rs +++ b/crates/server/src/registry.rs @@ -23,6 +23,7 @@ use crate::auto_version::{ AutoVersionState, }; use crate::clock::{Clock, SystemClock}; +use crate::gossip::GossipRoundOutcome; use crate::leadership::LeadershipEpochs; use crate::membership::Membership; use crate::placement::NodeId; @@ -596,12 +597,12 @@ impl Registry { if conn.peer.is_some() { return false; } - let certified = match conn.cert_hosts.as_deref() { + match conn.cert_hosts.as_deref() { // No certificate at all. The claim stands on its own, which is what a // deployment that has not issued per-node certificates has; one that has // says so and this is refused. None if self.require_peer_identity => return false, - None => false, + None => {} // A certificate *was* presented and verified, so it decides — including // when it names no host, which binds nothing. Treating that as certless // would make a certificate widen what a link may claim. @@ -612,22 +613,17 @@ impl Registry { { return false } - Some(_) => true, - }; - let member = NodeId::from(claimed.to_vec()); - conn.peer = Some(member.clone()); - // An inbound link is a verification of the member it names only when a - // certificate named it. Admission alone is no evidence: the cluster secret is - // deployment-wide, so an uncertified claim is a member asserting an id rather - // than answering at one, and taking it as verification would let any - // secret-holder place any id it liked. A dial in the other direction still - // vouches without certificates — it at least proves the far end answers at the - // address the id names. - if certified { - if let Some(membership) = &mut self.membership { - membership.note_verified(&member); - } - } + Some(_) => {} + } + conn.peer = Some(NodeId::from(claimed.to_vec())); + // Admitting a link is **not** a verification of the member it names, however + // well the certificate names it. A member chooses when to dial in and how + // often, so a vouch earned that way is one the member caused rather than one + // this node independently made — and a certificate names a *host*, which mints + // as many node ids as it likes, so a member could dial in under each ground id + // in turn and have this node vouch for every one of them. Verification is this + // node's own dial and nothing else (see + // [`note_peer_verified`](Self::note_peer_verified)). true } @@ -809,14 +805,12 @@ impl Registry { /// is refused — stalling their quorums. Without that policy the same round is the /// honest floor, and vouches for reachability alone. pub fn note_peer_verified(&mut self, node: &NodeId) { - let require_identity = self.require_peer_identity; - let Some(membership) = &mut self.membership else { - return; - }; - if require_identity && !membership.advertises_tls(node) { + if !self.dial_establishes_identity(node) { return; } - membership.note_verified(node); + if let Some(membership) = &mut self.membership { + membership.note_verified(node); + } } /// Merge a gossiped liveness payload into this node's membership — the SWIM @@ -838,11 +832,30 @@ impl Registry { /// the node this one dialed choose who takes a place in the ring. /// /// `sender` is the member the payload came from, and every `verified` flag in it is - /// recorded as that member's own first-hand claim. + /// recorded as that member's own first-hand claim — but only where `sender`'s + /// identity was established. This is the *reply* half of a round this node drove, + /// so what establishes it is the dial: a deployment that requires an identified + /// peer gets one from a `wss://` member's certificate and nothing at all from a + /// plaintext one, and an unattributable claim must not become an adopted member's + /// vouch. The liveness in the payload still merges either way — a member's + /// reachability is not a claim about anyone's identity. pub fn merge_gossip( &mut self, sender: &NodeId, members: Vec<(Vec, Vec, u64, MemberState, bool)>, + ) { + let attributable = self.dial_establishes_identity(sender); + self.merge_gossip_attributed(sender, members, attributable); + } + + /// Merge a gossip payload from `sender`, saying explicitly whether its `verified` + /// flags may be attributed to it — the seam both halves of a round share, each + /// deciding attribution from the link it actually holds. + fn merge_gossip_attributed( + &mut self, + sender: &NodeId, + members: Vec<(Vec, Vec, u64, MemberState, bool)>, + attributable: bool, ) { if let Some(membership) = &mut self.membership { membership.merge_liveness( @@ -850,12 +863,33 @@ impl Registry { members .into_iter() .map(|(node, addr, inc, state, verified)| { - (NodeId::from(node), addr, inc, state, verified) + ( + NodeId::from(node), + addr, + inc, + state, + verified && attributable, + ) }), ); } } + /// Whether a dial to `member` establishes who answered — a TLS member's + /// certificate does, a plaintext member's transport does not, and a deployment + /// that has not declared identity required takes the dial at face value. The + /// member's *advertised* transport is the right thing to read here and the wrong + /// thing to read of an inbound link: an outbound dial runs over exactly the + /// transport that address declares, while an inbound link is one the member dialed + /// and its own listener's scheme describes nothing about it. + fn dial_establishes_identity(&self, member: &NodeId) -> bool { + !self.require_peer_identity + || self + .membership + .as_ref() + .is_some_and(|m| m.advertises_tls(member)) + } + /// Run one reap check over the cluster membership: remove members that have /// stayed `Dead` past the bounded dead-time ([`Membership::reap_dead`]), so a /// durably-departed node stops lingering as a placement replica. Driven once per @@ -882,6 +916,24 @@ impl Registry { } } + /// Record how one SWIM probe round reached `node` — the whole of what a round + /// means to this view, in one place rather than split across the caller. + /// + /// Liveness and *identity* are different questions and a round answers them + /// differently. Every reachable outcome is proof the member is alive, whichever + /// path found it. Only [`Direct`](GossipRoundOutcome::Direct) is proof about the + /// member itself: this node dialed the address the id names and the transport + /// authenticated the far end before a byte was written, which is what a + /// verification claims. A relay's second opinion says a *relay* reaches the target, + /// which vouches for nobody here — if it did, one member confirming a target would + /// place it, and the mint would be open one step over. + pub fn note_gossip_round(&mut self, node: NodeId, outcome: GossipRoundOutcome) { + self.note_gossip_probe(node.clone(), outcome.reachable()); + if outcome == GossipRoundOutcome::Direct { + self.note_peer_verified(&node); + } + } + /// Apply an inbound [`Message::Gossip`] from the member `sender` on peer /// connection `id`: merge the advertised liveness into this node's view, then /// answer with this node's own so the exchange syncs both directions (push-pull @@ -931,7 +983,10 @@ impl Registry { membership.is_member(&node) || (&node == sender && addr == node.as_bytes()) }) .collect(); - self.merge_gossip(sender, members); + // The link this arrived on is bound to `sender` (C13), and under a deployment + // that requires an identified peer it is bound by a certificate that names it, + // so the claims it carries are attributable to that member. + self.merge_gossip_attributed(sender, members, true); let reply = crate::gossip::gossip_frame(&self.known_liveness()); if let Some(conn) = self.conns.get_mut(&id) { conn.outbox.push(reply); diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index 16d82a64..3416a117 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -39,7 +39,7 @@ use tokio_tungstenite::tungstenite::Message as WsMessage; use crate::auth::{AllowAll, Identity, Verifier}; use crate::authz::{Authorizer, PermitAll}; use crate::dial::{PeerDialer, PeerStream}; -use crate::gossip::{GossipMember, GossipWireMember}; +use crate::gossip::{GossipMember, GossipRoundOutcome, GossipWireMember}; use crate::membership::Membership; use crate::placement::NodeId; use crate::webhook::{WebhookConfig, WebhookSink}; @@ -249,13 +249,13 @@ enum Cmd { /// success, `learned` carries the liveness the peer advertised back (the /// registry merges it) and the peer is noted reachable; on failure (`learned` /// is `None`) the peer is noted unreachable — the gossip-driven failover signal. - /// `direct` says the round reached `peer` by *this node's own dial* rather than - /// through a relay's second opinion, which is what makes it evidence of the peer's - /// identity and not merely of its liveness. + /// `outcome` says how the round reached `peer` — by this node's own dial, through + /// a relay's second opinion, or not at all — which is what separates evidence of + /// the peer's identity from evidence of its mere liveness. GossipRound { peer: NodeId, - learned: Option>, - direct: bool, + learned: Vec, + outcome: GossipRoundOutcome, }, /// The blob-fetch plane asks whether `identity` may retrieve blob `blob_id`, /// resolved against the live rooms' references (see @@ -740,25 +740,16 @@ async fn registry_actor( Cmd::GossipRound { peer, learned, - direct, + outcome, } => { // The next client delivery recomputes placement and // effective leadership off the grown set and updated - // liveness, so there is nothing to flush here. - match learned { - Some(members) => { - reg.note_gossip_probe(peer.clone(), true); - // This node dialed the peer and the transport - // authenticated it before a byte was written, so the - // round is its own first-hand verification of that - // member — the evidence its place in the ring rests on. - if direct { - reg.note_peer_verified(&peer); - } - reg.merge_gossip(&peer, members); - } - None => reg.note_gossip_probe(peer, false), - } + // liveness, so there is nothing to flush here. The round is + // noted *before* its payload merges, so a peer this node has + // just verified can vouch in the same round rather than the + // next one. + reg.note_gossip_round(peer.clone(), outcome); + reg.merge_gossip(&peer, learned); } Cmd::AuthorizeBlob { identity, @@ -1150,26 +1141,21 @@ async fn gossip_loop( // On a direct failure, ask a few other members for a second opinion (SWIM // ping-req) before counting the failure toward suspicion: a peer any relay // still reaches is not falsely suspected. The direct and indirect signals - // fold through the same `probe_outcome` the spec tests drive. + // fold through the same outcome the spec tests drive. let indirect = if direct.is_some() { Vec::new() } else { indirect_probe(server, &dialer, &members, &self_id, &peer, &peer_addr).await }; - let reachable = crate::gossip::probe_outcome(direct.is_some(), &indirect); + let outcome = GossipRoundOutcome::of(direct.is_some(), &indirect); // A direct success carries the liveness the peer advertised back; an - // indirect-only success reports the peer reachable but has nothing to merge - // (an empty learned set); an all-failed round reports it unreachable (`None`). - let reached_directly = direct.is_some(); - let learned = match direct { - Some(members) => Some(members), - None => reachable.then(Vec::new), - }; + // indirect-only success has nothing to merge, and neither does a failed round. + let learned = direct.unwrap_or_default(); if cmds .send(Cmd::GossipRound { peer, learned, - direct: reached_directly, + outcome, }) .is_err() { diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index 76d0d72a..e86802e4 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -18,12 +18,27 @@ //! member is **pending** — dialed, probed and gossiped about, but in no room's replica //! set and no room's quorum — until the cluster adopts it. Adoption cannot be a local //! predicate, because placement must be identical on every node or the ring splits, so -//! a node records only what it knows first-hand (it completed an identity-checked peer -//! link to the member), that claim rides the same anti-entropy liveness does attributed -//! to the node that made it, and a member is placed once `ADOPTION_VERIFIERS` -//! already-adopted members have verified it. A member never verifies itself, so no node -//! can place itself; configured members are adopted from birth, the operator's config -//! being the root of trust. +//! a node records only what it knows first-hand — its **own dial** completed and the +//! transport authenticated the far end — and that claim rides the same anti-entropy +//! liveness does, attributed to the member the receiving link is bound to. A member is +//! placed once `ADOPTION_VERIFIERS` already-adopted **trust units** have verified it. +//! +//! Four rules make that hold. The adopted set is *derived* from the configured members +//! plus the evidence, never accumulated, so the ring is a function of state and not of +//! the order a node saw things in. A verifier is a **host**, because a certificate +//! names a host and a host mints unlimited node ids — and a member's own host is +//! excluded from its own count. An **inbound** link never verifies, however certified: +//! the member chooses when to dial in, so the vouch would be one it caused. And a +//! member is dialed at its **own id**, so no peer can decide where a later dial goes by +//! being the first to advertise an address. +//! +//! Two limits are pinned below as passing tests rather than left implied. With no +//! certificates configured a secret-holder binds a link to any member id it likes +//! (C13's own residual), so the bar is not a bar and what remains is reachability. And +//! under peer mTLS a member that owns a host owns every id under it — it answers at +//! each ground id and honest nodes verify one truthfully — so adoption bounds the mint +//! to the member's own host and no further; closing that needs the ring to weigh trust +//! units rather than ids (C27). //! //! These drive the registry and the membership in process (no sockets), so they are //! deterministic and run under Miri. @@ -33,6 +48,7 @@ use std::sync::Arc; use crdtsync_core::protocol::Channel; use crdtsync_core::{ClientId, Document, MemberState, Message, Op, Scalar}; use crdtsync_server::auth::Identity; +use crdtsync_server::gossip::GossipRoundOutcome; use crdtsync_server::membership::{Membership, ADOPTION_VERIFIERS}; use crdtsync_server::placement::{Cluster, NodeId}; use crdtsync_server::{ConnId, ManualClock, Registry}; @@ -405,19 +421,22 @@ fn a_pending_member_does_not_enlarge_the_quorum_of_a_room_it_does_not_attack() { let (room, c, followers) = led_room_with_a_withheld_write(&mut r); let seq = r.hub().seq(&room); - // A member minted for a *different* room — the collateral case. - let other = (0..1_000_000) - .map(|i| format!("room-{i}").into_bytes()) - .find(|candidate| candidate != &room && m.is_primary_for(candidate)) - .expect("a second room self leads"); - let minted = minted_for(&m, &other); + // The member is minted for *this* room, so it would have entered its replica set — + // and therefore raised its majority — had it been adopted. Pending, no room's + // replica set moves at all, so the rooms it never meant to touch keep the quorum + // they always had. + let minted = minted_for(&m, &room); let p = peer_as(&mut r, &minted); assert!(r.deliver(p, introduces(&minted, false))); - assert_eq!( - r.membership().expect("clustered").replicas_for(&room), - m.replicas_for(&room), - "the written room's replica set is untouched", - ); + let view = r.membership().expect("clustered"); + for i in 0..32 { + let sample = format!("room-{i}").into_bytes(); + assert_eq!( + view.replicas_for(&sample), + m.replicas_for(&sample), + "no room's replica set moves", + ); + } r.record_replica_ack(followers[0].clone(), &room, seq); r.record_replica_ack(followers[1].clone(), &room, seq); @@ -441,7 +460,38 @@ fn a_member_cannot_verify_itself_into_the_ring() { let p = peer_as(&mut r, &minted); assert!(r.deliver(p, introduces(&minted, true))); - assert!(!r.membership().expect("clustered").is_adopted(&minted)); + let view = r.membership().expect("clustered"); + assert!( + !view.has_verified(&minted, &minted), + "the claim never enters the evidence", + ); + assert!(!view.is_adopted(&minted)); +} + +#[test] +fn a_claim_is_recorded_against_the_member_whose_link_carried_it() { + // The claim is worth exactly the link it arrived on, so it is attributed to that + // member and to nobody the payload names. Were it recorded against the named node + // instead, every member would be its own voucher and the bar would be no bar. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + let voucher = NodeId::from("10.0.0.1:9000"); + m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + m.merge_liveness( + &voucher, + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + assert!(m.has_verified(&voucher, &joiner), "the sender vouched"); + assert!( + !m.has_verified(&joiner, &joiner), + "and the member the payload named did not", + ); } #[test] @@ -560,44 +610,66 @@ fn a_pending_members_verification_does_not_count() { // --- the address a member is verified at is the address its id names --- #[test] -fn a_member_advertised_at_another_hosts_address_is_never_adopted() { - // The other lesser way in: the reply half of a gossip round adopts a member at - // whatever address its tuple carries, so a dialed node can introduce an id pointing - // somewhere else entirely. Such a member would be verified at one host and dialed - // at another, so no link to either vouches for it and it stays pending however many - // members claim it. - let mut m = membership_for(SELF_ADDR); +fn a_member_is_dialed_at_its_own_id_whatever_address_a_tuple_carries() { + // The other lesser way in: the reply half of a gossip round hands over members at + // whatever address their tuples carry, and a member's recorded address decided + // where every later dial went — so whoever advertised a member *first* decided + // what every node afterwards verified. Two nodes that saw it in a different order + // then verified different endpoints and placed rooms differently, forever. A node + // id is an advertise address, so the second name is dropped and the dial address is + // a function of the id alone. + let mut poisoned = membership_for(SELF_ADDR); let joiner = NodeId::from("10.9.9.9:9000"); - m.add_member(joiner.clone(), b"10.6.6.6:9000".to_vec()); - for voucher in ["10.0.0.1:9000", "10.0.0.2:9000", "10.0.0.3:9000"] { - m.merge_liveness( - &NodeId::from(voucher), - [( - joiner.clone(), - b"10.6.6.6:9000".to_vec(), - 0, - MemberState::Alive, - true, - )], - ); + poisoned.add_member(joiner.clone(), b"10.6.6.6:9000".to_vec()); + let mut clean = membership_for(SELF_ADDR); + clean.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + + let dial_of = |m: &Membership| { + m.known_members() + .into_iter() + .find(|(node, _)| node == &joiner) + .map(|(_, addr)| addr) + }; + assert_eq!(dial_of(&poisoned), Some(joiner.as_bytes().to_vec())); + assert_eq!(dial_of(&poisoned), dial_of(&clean)); + + // So the same evidence places the same rooms on both, whichever address was + // advertised first. + for m in [&mut poisoned, &mut clean] { + for voucher in ["10.0.0.1:9000", "10.0.0.2:9000"] { + m.merge_liveness( + &NodeId::from(voucher), + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + } } - assert!(!m.is_adopted(&joiner)); - // Nor does this node's own link to it vouch, for the same reason. - m.note_verified(&joiner); - assert!(!verified_by(&m, &joiner)); - assert!(!m.is_adopted(&joiner)); + assert!(poisoned.is_adopted(&joiner)); + assert_eq!(poisoned.adopted_members(), clean.adopted_members()); } #[test] fn a_member_this_view_never_learned_is_not_verified_into_existence() { // A link is evidence *about* a member; where this view holds no member there is // nothing to be evidence about, and a link that could add one would restore the - // unchecked join this closes. Learning a member stays gossip's job. + // unchecked join this closes. Learning a member stays gossip's job. Nothing is + // *stored* either, which is what keeps the evidence bounded by the roster: peer + // admission takes the id a link claims, so a certified member could otherwise open + // a link per port on its own host and grow the map without limit. let mut m = membership_for(SELF_ADDR); let stranger = NodeId::from("10.9.9.9:9000"); m.note_verified(&stranger); assert!(!m.is_member(&stranger)); assert!(!m.is_adopted(&stranger)); + assert!( + !m.has_verified(&NodeId::from(SELF_ADDR), &stranger), + "no evidence is kept about a node that is no member", + ); } // --- what a link means --- @@ -637,10 +709,13 @@ fn an_uncertified_inbound_link_does_not_verify_the_member_it_claims() { } #[test] -fn a_certified_inbound_link_verifies_the_member_it_names() { - // The other half of the promotion signal: a member dialing in behind a certificate - // that names its own host is verified by this node, exactly as this node's own dial - // to it would be. One verification is still not adoption in a cluster this size. +fn no_inbound_link_verifies_the_member_it_names_however_it_is_certified() { + // Admitting a link is not a verification, however well the certificate names the + // member. A member chooses when to dial in and how often, so a vouch earned that + // way is one the member *caused* rather than one this node independently made — and + // a certificate names a host, which mints as many node ids as it likes, so a member + // could dial in under each ground id in turn and have this node vouch for every + // one. Verification is this node's own dial and nothing else. let m = membership_for(SELF_ADDR); let room = room_self_leads(&m); let minted = minted_for(&m, &room); @@ -649,13 +724,17 @@ fn a_certified_inbound_link_verifies_the_member_it_names() { let p = peer_as(&mut r, &minted); assert!(r.deliver(p, introduces(&minted, false))); certified_peer_as(&mut r, &minted); + certified_peer_as(&mut r, &minted); let view = r.membership().expect("clustered"); - assert!(verified_by(view, &minted), "the certificate vouches"); - assert!( - !view.is_adopted(&minted), - "and one voucher is short of the bar", - ); + assert!(!verified_by(view, &minted), "no inbound link vouches"); + assert!(!view.is_adopted(&minted)); + + // This node's own dial is what does, and one voucher is still short of the bar. + r.note_peer_verified(&minted); + let view = r.membership().expect("clustered"); + assert!(verified_by(view, &minted)); + assert!(!view.is_adopted(&minted)); } // --- convergence --- @@ -733,6 +812,10 @@ fn a_reaped_member_loses_its_place_and_its_vouches() { } assert!(!m.is_member(&departing)); assert!(!m.is_adopted(&departing)); + assert!( + !m.has_verified(&departing, &joiner), + "a departed member vouches for nobody, and its entries go with it", + ); // Its vouch went with it: one further genuine claim is now the first, not the // second. @@ -831,3 +914,173 @@ fn a_plaintext_dial_still_verifies_where_no_identity_is_required() { r.note_peer_verified(&plain); assert!(verified_by(r.membership().expect("clustered"), &plain)); } + +// --- a trust unit is a host, not a node id --- + +/// A membership whose configured set puts two members on one host and one on another +/// — the shape that tells "distinct verifiers" from "distinct trust units" apart. +fn shared_host_membership() -> Membership { + Membership::from_static_config( + None, + Some(SELF_ADDR), + "10.0.0.1:9000,10.0.0.1:9001,10.0.0.2:9000", + N, + ) + .unwrap() +} + +/// `voucher`'s claim to have verified `node`, merged into `m`. +fn vouch(m: &mut Membership, voucher: &str, node: &NodeId) { + m.merge_liveness( + &NodeId::from(voucher), + [( + node.clone(), + node.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); +} + +#[test] +fn two_ids_on_one_host_are_one_voucher() { + // A certificate names a *host* (C13), and a host mints as many node ids as it + // likes — so counting distinct ids would let one machine raise the whole bar by + // itself, which is the mint one level up. Two adopted members on one host vouch + // once between them; a member on a second host is what carries it. + let mut m = shared_host_membership(); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + + vouch(&mut m, "10.0.0.1:9000", &joiner); + vouch(&mut m, "10.0.0.1:9001", &joiner); + assert!( + !m.is_adopted(&joiner), + "both vouchers are the same machine, so they are one", + ); + + vouch(&mut m, "10.0.0.2:9000", &joiner); + assert!(m.is_adopted(&joiner), "a second host carries it"); +} + +#[test] +fn a_sibling_on_a_members_own_host_does_not_vouch_for_it() { + // A member vouching for a sibling on its own host is vouching for itself: they are + // one trust unit, and the whole bar is that a member cannot place a node by itself. + let mut m = shared_host_membership(); + let sibling = NodeId::from("10.0.0.1:9002"); + m.add_member(sibling.clone(), sibling.as_bytes().to_vec()); + + vouch(&mut m, "10.0.0.1:9000", &sibling); + vouch(&mut m, "10.0.0.1:9001", &sibling); + assert!( + !m.is_adopted(&sibling), + "its own host does not vouch for it" + ); + + vouch(&mut m, "10.0.0.2:9000", &sibling); + m.note_verified(&sibling); + assert!(m.is_adopted(&sibling), "two other hosts do"); +} + +#[test] +fn a_certified_member_still_mints_ids_on_its_own_host() { + // The limit, pinned rather than implied. Placement keys on node ids and a + // certificate names a host, so a member that owns a host owns every id under it: + // it answers at each ground id, and the honest nodes that dial it are telling the + // truth when they verify one. Adoption bounds the mint to the member's own host and + // to ids the cluster can actually reach — it cannot bound it further, because every + // verification here is genuine. Closing it needs the ring to weigh trust units + // rather than ids, which is a placement change, not an evidence one. + let mut m = membership_for(SELF_ADDR); + let minted = NodeId::from("10.0.0.1:9999"); + m.add_member(minted.clone(), minted.as_bytes().to_vec()); + vouch(&mut m, "10.0.0.2:9000", &minted); + vouch(&mut m, "10.0.0.3:9000", &minted); + assert!( + m.is_adopted(&minted), + "honest nodes that reached it vouched truthfully, and it is placed", + ); +} + +// --- what a round means, and whose word counts --- + +#[test] +fn only_a_direct_round_verifies_the_member_it_reached() { + // A relay's second opinion says a *relay* reaches the target, which this node did + // not observe and cannot attribute — so it is evidence of life and of nobody's + // identity. Were it otherwise, one member confirming a target would place it. + let m = membership_for(SELF_ADDR); + let room = room_self_leads(&m); + let minted = minted_for(&m, &room); + let mut r = registry(); + let p = peer_as(&mut r, &minted); + assert!(r.deliver(p, introduces(&minted, false))); + + r.note_gossip_round(minted.clone(), GossipRoundOutcome::Relayed); + assert!(!verified_by(r.membership().expect("clustered"), &minted)); + r.note_gossip_round(minted.clone(), GossipRoundOutcome::Unreachable); + assert!(!verified_by(r.membership().expect("clustered"), &minted)); + + r.note_gossip_round(minted.clone(), GossipRoundOutcome::Direct); + assert!(verified_by(r.membership().expect("clustered"), &minted)); +} + +#[test] +fn a_plaintext_peers_vouches_do_not_count_where_identity_is_required() { + // The reply half comes from a node this one dialed, so what establishes the + // sender's identity is that dial. A deployment that requires an identified peer + // gets one from a `wss://` member's certificate and nothing at all from a plaintext + // one — and an unattributable claim must not become an adopted member's vouch. The + // liveness in the same payload still merges: reachability is nobody's identity. + let mut r = registry(); + r.set_require_peer_identity(true); + let joiner = NodeId::from("10.9.9.9:9000"); + let plaintext_member = NodeId::from("10.0.0.1:9000"); + + r.merge_gossip(&plaintext_member, advertisements(&[&joiner], true)); + let view = r.membership().expect("clustered"); + assert!(view.is_member(&joiner), "the liveness still merged"); + assert!( + !view.has_verified(&plaintext_member, &joiner), + "but the claim is attributable to nobody", + ); +} + +// --- bootstrap --- + +#[test] +fn a_node_that_knows_only_itself_places_on_what_it_has_reached() { + // The bar is a constant on every node, so two nodes never disagree about what the + // evidence must show — except for a node configured with no peers at all, which has + // no cluster to be outvoted by and would otherwise freeze forever: a second voucher + // can only ever come from a member it has already adopted. That is the single-node + // deployment, and the cluster it places on is what it has itself reached. + let mut alone = Membership::from_static_config(None, Some(SELF_ADDR), "", N).unwrap(); + let first = NodeId::from("10.9.9.9:9000"); + let second = NodeId::from("10.9.9.8:9000"); + for node in [&first, &second] { + alone.add_member((*node).clone(), node.as_bytes().to_vec()); + } + + alone.note_verified(&first); + alone.note_verified(&second); + assert!( + alone.is_adopted(&first), + "its own link is the whole verdict" + ); + assert!(alone.is_adopted(&second)); + + // The exception keys on configuration, not on the running set, so a node given a + // single seed peer is held to the constant from its first round — the seed is + // adopted from birth and there is a cluster to agree with. + let mut seeded = + Membership::from_static_config(None, Some(SELF_ADDR), "10.0.0.1:9000", N).unwrap(); + seeded.add_member(first.clone(), first.as_bytes().to_vec()); + seeded.note_verified(&first); + assert!( + !seeded.is_adopted(&first), + "one voucher is short of the bar" + ); +} From c70ad1456e8e21d8c4400e37228b5bdf11772dcc Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 20:28:58 -0300 Subject: [PATCH 04/17] test(server): isolate the trust-unit, pending-voucher and merge-path address rules The mutation sweep found three assertions that held for a reason other than the rule they name: two same-host vouchers were already one unit by the host dedup, two pending vouchers were already excluded by sharing the candidate's host, and the address rule was only exercised on the additive path. --- crates/server/tests/adoption.rs | 36 ++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index e86802e4..5b110c72 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -584,8 +584,10 @@ fn a_pending_members_verification_does_not_count() { // Two nodes minted together would otherwise vouch each other in without a single // established member ever reaching either. Only an adopted member's claim counts. let mut m = membership_for(SELF_ADDR); + // On *different* hosts, so nothing but their pending status keeps their word out: + // the trust-unit count would otherwise reach the bar on the pair alone. let first = NodeId::from("10.9.9.9:9000"); - let second = NodeId::from("10.9.9.9:9001"); + let second = NodeId::from("10.9.9.8:9000"); for node in [&first, &second] { m.add_member((*node).clone(), node.as_bytes().to_vec()); } @@ -605,6 +607,12 @@ fn a_pending_members_verification_does_not_count() { m.merge_liveness(&NodeId::from("10.0.0.1:9000"), claim(&first)); assert!(!m.is_adopted(&first)); assert!(!m.is_adopted(&second)); + + // A second *adopted* member is what carries it, so the refusal is about who + // vouched and not about the shape of the evidence. + m.merge_liveness(&NodeId::from("10.0.0.2:9000"), claim(&first)); + assert!(m.is_adopted(&first)); + assert!(!m.is_adopted(&second)); } // --- the address a member is verified at is the address its id names --- @@ -618,9 +626,24 @@ fn a_member_is_dialed_at_its_own_id_whatever_address_a_tuple_carries() { // then verified different endpoints and placed rooms differently, forever. A node // id is an advertise address, so the second name is dropped and the dial address is // a function of the id alone. - let mut poisoned = membership_for(SELF_ADDR); let joiner = NodeId::from("10.9.9.9:9000"); - poisoned.add_member(joiner.clone(), b"10.6.6.6:9000".to_vec()); + let foreign = b"10.6.6.6:9000".to_vec(); + // Poisoned through the additive path on one node and through the anti-entropy + // merge on another — both are ways a peer's tuple reaches the roster, and the rule + // has to hold on each or the two disagree. + let mut poisoned = membership_for(SELF_ADDR); + poisoned.add_member(joiner.clone(), foreign.clone()); + let mut merged = membership_for(SELF_ADDR); + merged.merge_liveness( + &NodeId::from("10.0.0.1:9000"), + [( + joiner.clone(), + foreign.clone(), + 0, + MemberState::Alive, + false, + )], + ); let mut clean = membership_for(SELF_ADDR); clean.add_member(joiner.clone(), joiner.as_bytes().to_vec()); @@ -631,6 +654,7 @@ fn a_member_is_dialed_at_its_own_id_whatever_address_a_tuple_carries() { .map(|(_, addr)| addr) }; assert_eq!(dial_of(&poisoned), Some(joiner.as_bytes().to_vec())); + assert_eq!(dial_of(&merged), Some(joiner.as_bytes().to_vec())); assert_eq!(dial_of(&poisoned), dial_of(&clean)); // So the same evidence places the same rooms on both, whichever address was @@ -972,14 +996,16 @@ fn a_sibling_on_a_members_own_host_does_not_vouch_for_it() { let sibling = NodeId::from("10.0.0.1:9002"); m.add_member(sibling.clone(), sibling.as_bytes().to_vec()); + // One voucher on the candidate's own host and one elsewhere: two members, two node + // ids, and only *one* trust unit that is not the candidate itself. vouch(&mut m, "10.0.0.1:9000", &sibling); - vouch(&mut m, "10.0.0.1:9001", &sibling); + vouch(&mut m, "10.0.0.2:9000", &sibling); assert!( !m.is_adopted(&sibling), "its own host does not vouch for it" ); - vouch(&mut m, "10.0.0.2:9000", &sibling); + // A second host does — this node, which reached it over its own link. m.note_verified(&sibling); assert!(m.is_adopted(&sibling), "two other hosts do"); } From 86702fa362116499af315f2da83de11229fb169d Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 20:41:03 -0300 Subject: [PATCH 05/17] fix(server): a trust unit is a host the certificate binding would call one, and the bootstrap bar is fixed at construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 2 broke the trust-unit count two ways. - The unit was the host as raw text while the certificate binding compares it semantically — lowercased, root label dropped, IP literals parsed as addresses. So `evil.example` beside `evil.example.`, or an IPv6 literal beside its expanded form, were one certificate and two vouchers: one machine held two units and could adopt any id anywhere, which is not the bounded residual C27 records. The unit is now reduced by that same relation. - The bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed peer fell to the single-node rule the moment that seed was reaped and began placing on one vouch while every peer still held the constant. It is decided once, at construction. Also drops `add_member`'s ignored address argument, so no caller can believe the advertised address is honored, and pins the fixpoint, the returned member's ring status and the host-aliasing relation, three rules whose tests held for another reason. Files C28 for two gossip-plane inputs review found unbounded: an unrefutable incarnation that evicts a member for good, and an unbounded roster that starves adoption. --- ARCHITECTURE.md | 6 +- DECISIONS.md | 4 +- KANBAN.md | 4 +- crates/server/src/dial.rs | 19 +++++ crates/server/src/membership.rs | 56 ++++++++----- crates/server/tests/adoption.rs | 143 ++++++++++++++++++++++++++++---- crates/server/tests/gossip.rs | 20 ++--- crates/server/tests/reaping.rs | 22 ++++- 8 files changed, 224 insertions(+), 50 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d2341f97..74b2cdde 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -970,7 +970,7 @@ The secret is a **bearer** credential, so the peer link carries it over an encry **Only this node's own dial verifies.** The dial goes to the address the member's id names and the transport authenticates the far end before a byte is written; that is an act this node chose to take. An *inbound* link never verifies the member it names, however well a certificate names it — a member chooses when to dial in and how often, so a vouch earned that way is one the member caused rather than one this node made, and a certificate names a host, which mints as many node ids as it likes, so a member could dial in under each ground id in turn and be vouched for under every one. An *indirect* (ping-req) confirmation is liveness only: it says a relay reaches the target, which this node did not observe and cannot attribute. -**A verifier is a trust unit, and a trust unit is a host.** A host is what a certificate names (§Peer Identity) and one host holds as many node ids as it likes, so the bar counts the distinct *hosts* of the adopted members that vouched, not the ids. Counting ids would let one machine holding two of them raise the whole bar by itself. A member's own host is excluded from its own count for the same reason: a member vouching for a sibling on its own host is vouching for itself. +**A verifier is a trust unit, and a trust unit is a host.** A host is what a certificate names (§Peer Identity) and one host holds as many node ids as it likes, so the bar counts the distinct *hosts* of the adopted members that vouched, not the ids. Counting ids would let one machine holding two of them raise the whole bar by itself. A member's own host is excluded from its own count for the same reason: a member vouching for a sibling on its own host is vouching for itself. **The host is reduced by the same relation the certificate binding compares by** — lowercased, root label dropped, IP literals compared as addresses — because the two must agree on when two spellings are one host: reading the host as raw text while the binding read it semantically would let one machine present as several vouchers under `evil.example` beside `evil.example.`, or an IPv6 literal beside its expanded form. **A claim is worth exactly the link that carried it.** A `verified` flag is recorded against the member the receiving link is bound to (§Peer Identity) and against nobody the payload names, so a member can assert its own verifications and no one else's; relaying another node's would make one member's word enough to place any id it liked. A tuple claiming the sender itself is dropped — a node's place in the ring is never its own to assert — and a claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. On the *reply* half of a round, the claim counts only where the dial established who answered: under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a `wss://` member's certificate does and a plaintext member's transport does not, and an unattributable claim must not become an adopted member's vouch. The cost of first-hand-only is that verifications travel by direct exchange rather than by relay: a joiner is placed once enough members have each reached it, which randomized peer selection delivers in a handful of gossip rounds rather than one. @@ -978,10 +978,14 @@ The secret is a **bearer** credential, so the peer link carries it over an encry **A member is dialed at its own node id.** A node id *is* an advertise address, so a second address for the same member is a second, unauthenticated name for one thing — and the ring turns on it, because a node dials a member in order to verify it. Keeping the advertised address made the roster first-write-wins over a field any peer may set: whoever advertised a member first decided where every later dial went, so two nodes that saw it in a different order verified different endpoints and placed rooms differently, forever. The address a gossip tuple carries is therefore dropped, which also ends the *reply* half's freedom to point the ring anywhere. +**A node converges its ring a little after its roster.** A joiner adopts the cluster's members as their vouches reach it, so for the first few gossip rounds it places rooms over the members it was configured with and holds a ring smaller than the cluster's. A client that reaches it in that window is served by a node that believes it leads rooms it does not — and the write *stalls* rather than being wrongly accepted, because majority-ack counts over the same small replica set and the real replicas refuse its `Replicate`. The window closes as every member's claims arrive, which happens on their rounds as well as its own. + **Reaping takes adoption and vouches with it.** A member removed from the roster leaves the configured set, its verifier set is dropped, and it is struck from every other member's, so the ring is recomputed without its word. A return is a fresh join and is verified again. **Adoption is only as strong as the identity beneath it, and it does not bound a host.** Under peer mTLS a verification means a certificate for that member's host answered there, so no member can manufacture another's verification. **With no certificates configured none of that holds**: a verification means only that *something* answers at the address the id names, and a secret-holder can bind a link to any member id it likes, so it can raise the whole bar itself. `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` is what makes the bar a bar. Even then one limit remains, deliberately: **placement keys on node ids and a certificate names a host**, so a member that owns a host owns every id under it — it answers at each ground id, and the honest nodes that dial it verify one truthfully. Adoption bounds the mint to the member's own host and to ids the cluster can actually reach; it cannot bound it further, because every verification involved is genuine. Closing that needs the ring to weigh trust units rather than ids — a placement change, not an evidence one. +**One further limit follows from the same gap: a certificate that names several hosts is several trust units.** §Peer Identity deliberately lets a node certificate spell its host more than one way — a DNS name beside its IP literal — and each spelling is a different host, so a machine holding such a certificate can hold an adopted member under each and vouch as two units. Until the ring weighs trust units, a deployment that wants the bar to mean two *machines* issues each node a certificate naming one host. + ## Peer Transport **A member's advertise address declares the transport its peers dial it over.** `wss://host:port` terminates TLS; `ws://host:port`, or a bare `host:port`, does not. Nothing else can decide it: whether a node terminates TLS says nothing about whether the member it is dialing does, and the advertise address is the only per-member datum the cluster already agrees on and already disseminates through gossip. Any other scheme is a configuration error rather than part of a hostname. The scheme is part of the address and so part of the node id, which is what keeps every node's view of a member's transport identical without a second field on the wire. diff --git a/DECISIONS.md b/DECISIONS.md index c4aa91c7..9211dd9d 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -31,7 +31,9 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **A claim is worth the link that carried it, on both halves of a round.** On the inbound half the link is bound to the sender (C13). On the *reply* half — a payload from a node this one dialed — the dial is what establishes the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped: nothing authenticated them, and an unattributable claim must not become an adopted member's word. That reads the member's advertised transport, which C13's pass 3 rejected for the inbound direction and which is sound here — an outbound dial runs over exactly the transport the address declares, while an inbound link is one the member dialed and its own listener's scheme describes nothing about it. The liveness in the same payload still merges: reachability is nobody's identity. -**The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. +**Cold review pass 2 overturned two more, both about what a trust unit *is*.** The unit was keyed on the host as raw text while the certificate binding compares it semantically — lowercased, root label dropped, IP literals parsed as addresses — so `evil.example` beside `evil.example.`, or an IPv6 literal beside its expanded form, were one certificate and two vouchers. One machine therefore held two units and could adopt any id anywhere, which is not the accepted residual (that bounds the mint to the member's own host; this had no bound at all). The unit is now reduced by the same relation the binding uses. And the bootstrap exception read `configured.len() <= 1` off a set that *shrinks*: a node given one seed peer fell to the single-node rule the moment that seed was reaped, and started placing on one vouch while every peer still held the constant — a ring split off an ordinary retirement. It is decided once, at construction. + +**The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. The same gap has one more edge, stated rather than hidden: §Peer Identity deliberately lets a certificate spell a host several ways, and each spelling is a different host, so a machine holding a multi-host certificate holds one trust unit per name it carries. Until C27, a deployment that wants the bar to mean two *machines* issues one host per certificate. **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. diff --git a/KANBAN.md b/KANBAN.md index e1cab53c..2e68c5bf 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,7 +30,7 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ -**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. @@ -427,6 +427,8 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C27 — a member that owns a host owns every ring position under it, because placement keys on node ids while a certificate names a host (crates/server) — READY, no dependencies. Found by cold review during C25 (#369) and pinned there as a passing test (`a_certified_member_still_mints_ids_on_its_own_host`).** C25 made rooms place on *adopted* members and made adoption the cluster's decision, which closes minting on anyone else's host, minting an id nobody can reach, and any single member manufacturing the evidence. What it cannot close is the mint on a member's **own** host: HRW scores a node id, C13 binds a link to the *host* a certificate names, and a host answers at as many ports as it likes — so a compromised member grinds `own-host:` until HRW places it on the room it wants, answers there, and every honest node that dials it verifies it **truthfully**. No evidence rule can help, because no one is lying. The consequence is the one C25 removed everywhere else: a member chooses which rooms it replicates, and from inside the replica set it reaches the leadership and durability gates. **The fix is a placement change, not an evidence one: weigh the ring by trust unit.** Score hosts rather than node ids, so a room's replica set is `n` distinct hosts and which rooms a host holds follows from a name it cannot grind; the member serving a host's position is then a local choice among that host's adopted members, and it does not matter which, since they are one trust unit already. That is also the better replication property — `n` replicas on `n` failure domains instead of `n` ids that may share a machine — and it makes the `replicas_for(room).contains(sender)` gate ask about exactly what the certificate vouches for. Costs to settle: two members on one host stop both replicating a room (a capacity change, not a correctness one), the replication factor becomes a host count, and every gate/redirect/quorum that names a member has to resolve host → member. Note this is *not* reachable without a compromised member's certificate: without peer mTLS the whole binding vouches for nothing anyway (C13's residual), and with it the mint is bounded to a host the operator already trusts to run a node. → *Cluster*. +**C28 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. → *Cluster*. + **C26 — the SDK apply seams report a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — READY, no dependencies. Split out of C19 (#367) rather than folded in.** `crdtsync_client_receive`/`WasmClient::receive` fold a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learns how many ops applied *now* and nothing else. C19 made the distinction answerable — `Op::is_admissible` is public and pure — but did not plumb it: a buffered op (which a later arrival commits) and an op no replica will ever hold are both reported as "not counted", so an SDK peered with a buggy or hostile writer sees edits vanish with no signal, and the two cases call for opposite responses (wait vs. surface a client bug). `client.rs` needs no change on this axis — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it, so nothing is missed. The server side is closed: `handle_ops` answers `ErrorCode::MalformedOp` and the ingest seams drop such a record, so this is the *offline / P2P / relay* seam, where no server sits between the writer and the fold. Wants a refused count (or a refused-op list) beside the applied one across FFI, wasm, JS, Python and Go — and is worth taking with **C23**, which has the same shape: a real refusal the SDKs render as an edit that silently did nothing. → *SDK / Ops*. **C14 — a redacted op delta leaves the same re-mint hole, and no frame can carry the ids it withholds (crates/server + crates/core) — READY, no dependencies. Found during C9 (#357), reproduced.** C9 closed the *snapshot* seam: a projected snapshot now names the recipient's own ids, so adoption never frees one the room's log holds. An **uncompacted** room serves an op delta instead, and the per-op read filter withholds the recipient's own ops on paths it may no longer read — leaving a hole in its own run with nothing to repair it from. Measured on C9's own fixture: a reader with write at `/` and read on `/a` alone writes into `/b` (seqs 0,1) then `/a` (seq 2); it restarts, its delta withholds `/b`, `next_seq` reports 0, and its next write mints seq 0 straight onto its own withheld id — the write never lands. The state encoding was the carrier for the snapshot case; an `Ops` frame has no equivalent, so closing this needs one: a frontier field on the delta, or a rule that a reader's *own* ops are never withheld from it (which trades the hole for materialising ids in a subtree it may not read — the reason C9 did not simply stop scrubbing). Note the second shape also has to answer what an op carries beyond its id: its target and content may name denied structure. → *Server / Fan-out*. diff --git a/crates/server/src/dial.rs b/crates/server/src/dial.rs index dd197908..f1b302c1 100644 --- a/crates/server/src/dial.rs +++ b/crates/server/src/dial.rs @@ -137,6 +137,25 @@ pub fn member_host(addr: &[u8]) -> Option { host_of(authority).map(|host| host.to_ascii_lowercase()) } +/// The **trust unit** an advertise address belongs to: its host reduced to the one +/// form the certificate binding compares. `None` when the address names no host. +/// +/// A host is what a certificate names, so a host is the unit that decides how many +/// *independent* parties have vouched for a member. That count is only meaningful if +/// two spellings of one host reduce to one unit — and the binding +/// ([`cert_names_member`]) already treats them as one: it lowercases, drops the root +/// label, and compares IP literals as addresses rather than as text. Reading the host +/// as raw text here while the binding read it semantically would let one machine +/// present as several: `evil.example` beside `evil.example.`, or an IPv6 literal +/// beside its expanded form, are one certificate and would have been two vouchers. +pub fn member_trust_unit(addr: &[u8]) -> Option { + let host = normalized(&member_host(addr)?); + Some(match host.parse::() { + Ok(ip) => ip.to_string(), + Err(_) => host, + }) +} + /// The host an authority names, with its port and path stripped. A bracketed IPv6 /// literal keeps its brackets off but its colons intact, so `[::1]:9000` is `::1` /// rather than everything up to the last colon. `None` when it names none. diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index 67641b1b..3507ba0c 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -63,7 +63,7 @@ use std::collections::{HashMap, HashSet}; use crdtsync_core::MemberState; -use crate::dial::member_host; +use crate::dial::member_trust_unit; use crate::placement::{Cluster, NodeId}; /// Consecutive failed direct gossip probes to a member before this node escalates @@ -220,8 +220,17 @@ pub struct Membership { reaped: HashMap, /// The members this node was *configured* with — `self` and the seed peers. The /// root of trust a cluster starts from, adopted from birth because there is no - /// earlier authority for them to be vouched for by. + /// earlier authority for them to be vouched for by. Reaping removes a member from + /// it: a configured node that departed durably is gone, and the ring is derived + /// from this set. configured: HashSet, + /// Whether this node was configured with **no peers at all** — the single-node + /// deployment, which uses the bootstrap bar ([`adoption_bar`](Self::adoption_bar)). + /// Fixed at construction rather than read off `configured`, which shrinks: a bar + /// that followed the live set would collapse to one the moment a node's last seed + /// was reaped, and that node would then place on a single vouch while every peer + /// still held the constant — a ring split off an ordinary retirement. + solitary: bool, /// The members rooms are actually placed on — the subset of the roster the /// [`Cluster`] above is built from: the configured members plus every /// gossip-learned member the evidence now carries. **Derived, never accumulated** @@ -268,6 +277,7 @@ impl Membership { // root of trust a cluster starts from, and there is no earlier authority for // it to be vouched for by. let configured: HashSet = members.iter().cloned().collect(); + let solitary = configured.len() <= 1; Self { self_id, cluster: Cluster::new(members), @@ -278,6 +288,7 @@ impl Membership { reaped: HashMap::new(), adopted: configured.clone(), configured, + solitary, verifiers: HashMap::new(), } } @@ -329,11 +340,11 @@ impl Membership { self.cluster.nodes() } - /// Learn a member — the anti-entropy union gossip applies for each `(node, addr)` - /// pair a peer advertises. The advertised `addr` is **not** what the member is - /// dialed at; see [`add_members`](Self::add_members). - pub fn add_member(&mut self, node: NodeId, addr: Vec) { - self.add_members(std::iter::once((node, addr))); + /// Learn a member — the anti-entropy union gossip applies for each node a peer + /// advertises. **The address such a tuple carries is no argument here**, because it + /// is not what the member is dialed at; see [`add_members`](Self::add_members). + pub fn add_member(&mut self, node: NodeId) { + self.add_members(std::iter::once(node)); } /// Union a batch of learned members into the roster — dialable, probable and @@ -344,7 +355,7 @@ impl Membership { /// is neither placeable nor dialable, so a malformed gossip pair cannot poison the /// set. `self` is a member from construction and is never relearned. /// - /// **A member is recorded at its own id, whatever address the pair carries.** A + /// **A member is recorded at its own id.** A /// node id *is* an advertise address, so a second address for the same member is a /// second, unauthenticated name for one thing — and the ring turns on it, because a /// node dials a member to verify it. Keeping the advertised one made the roster @@ -353,9 +364,9 @@ impl Membership { /// different order verified different endpoints and placed rooms differently, /// forever. Ignoring it makes the dial address a function of the id alone, which /// every node agrees on by construction. - pub fn add_members(&mut self, members: impl IntoIterator)>) { + pub fn add_members(&mut self, members: impl IntoIterator) { let mut added = false; - for (node, addr) in members { + for node in members { // A reaped member is never re-added by a plain re-advertise: only a live // return (an `Alive` tuple through `merge_liveness`) escapes the // tombstone, so a bare gossip of the member's address — which carries no @@ -366,7 +377,6 @@ impl Membership { { continue; } - let _ = addr; self.liveness .insert(node.clone(), MemberLiveness::new(0, MemberState::Alive)); let dial = node.as_bytes().to_vec(); @@ -599,8 +609,12 @@ impl Membership { /// which is the mint one level up. Its own host is excluded for the same reason: a /// member vouching for a sibling on its own host is vouching for itself. A member /// whose id names no host counts for nobody — there is no unit to attribute it to. + /// + /// The host is reduced by [`member_trust_unit`], the same relation the certificate + /// binding compares by, so two spellings of one host are one voucher. Reading it as + /// raw text would have let one machine present as several. fn verifier_units(&self, node: &NodeId) -> usize { - let Some(own_host) = member_host(node.as_bytes()) else { + let Some(own_unit) = member_trust_unit(node.as_bytes()) else { return 0; }; let Some(verifiers) = self.verifiers.get(node) else { @@ -609,8 +623,8 @@ impl Membership { verifiers .iter() .filter(|v| self.adopted.contains(*v)) - .filter_map(|v| member_host(v.as_bytes())) - .filter(|host| host != &own_host) + .filter_map(|v| member_trust_unit(v.as_bytes())) + .filter(|unit| unit != &own_unit) .collect::>() .len() } @@ -651,17 +665,17 @@ impl Membership { /// How many trust units must vouch for a member before it is placed: /// [`ADOPTION_VERIFIERS`], except for a node **configured with no peers at all**. /// - /// That exception is narrow and it keys on configuration, not on the running set, - /// so it is a fixed property of a node rather than a bar that moves as the ring - /// grows. A node whose config names no peer has no cluster to be outvoted by, and - /// the constant would freeze it forever — a second voucher can only ever come from - /// a member it has already adopted — so the cluster it places on is exactly the - /// members it has itself reached. That is the single-node deployment; a node + /// That exception is narrow and it is decided **once, at construction**, so it is a + /// fixed property of a node rather than a bar that moves as the cluster changes + /// under it. A node whose config names no peer has no cluster to be outvoted by, + /// and the constant would freeze it forever — a second voucher can only ever come + /// from a member it has already adopted — so the cluster it places on is exactly + /// the members it has itself reached. That is the single-node deployment; a node /// meant to join a cluster is given a seed peer, which is adopted from birth and /// lifts it past this case at boot. Configure one: a node with none takes the ring /// from whoever reaches it. fn adoption_bar(&self) -> usize { - match self.configured.len() <= 1 { + match self.solitary { true => 1, false => ADOPTION_VERIFIERS, } diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index 5b110c72..1fcb7ef2 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -476,7 +476,7 @@ fn a_claim_is_recorded_against_the_member_whose_link_carried_it() { let mut m = membership_for(SELF_ADDR); let joiner = NodeId::from("10.9.9.9:9000"); let voucher = NodeId::from("10.0.0.1:9000"); - m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + m.add_member(joiner.clone()); m.merge_liveness( &voucher, [( @@ -513,7 +513,7 @@ fn one_members_verification_does_not_place_a_node() { // place it itself. let mut m = membership_for(SELF_ADDR); let joiner = NodeId::from("10.9.9.9:9000"); - m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + m.add_member(joiner.clone()); m.merge_liveness( &NodeId::from("10.0.0.1:9000"), @@ -534,7 +534,7 @@ fn enough_members_verifications_place_a_node() { // room its id places on. let mut m = membership_for(SELF_ADDR); let joiner = NodeId::from("10.9.9.9:9000"); - m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + m.add_member(joiner.clone()); for voucher in ["10.0.0.1:9000", "10.0.0.2:9000"] { m.merge_liveness( &NodeId::from(voucher), @@ -563,7 +563,7 @@ fn the_same_members_verification_twice_is_still_one_voucher() { // re-gossips its own claim every round does not accumulate into a majority of one. let mut m = membership_for(SELF_ADDR); let joiner = NodeId::from("10.9.9.9:9000"); - m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + m.add_member(joiner.clone()); for _ in 0..8 { m.merge_liveness( &NodeId::from("10.0.0.1:9000"), @@ -589,7 +589,7 @@ fn a_pending_members_verification_does_not_count() { let first = NodeId::from("10.9.9.9:9000"); let second = NodeId::from("10.9.9.8:9000"); for node in [&first, &second] { - m.add_member((*node).clone(), node.as_bytes().to_vec()); + m.add_member((*node).clone()); } let claim = |about: &NodeId| { [( @@ -632,7 +632,7 @@ fn a_member_is_dialed_at_its_own_id_whatever_address_a_tuple_carries() { // merge on another — both are ways a peer's tuple reaches the roster, and the rule // has to hold on each or the two disagree. let mut poisoned = membership_for(SELF_ADDR); - poisoned.add_member(joiner.clone(), foreign.clone()); + poisoned.add_member(joiner.clone()); let mut merged = membership_for(SELF_ADDR); merged.merge_liveness( &NodeId::from("10.0.0.1:9000"), @@ -645,7 +645,7 @@ fn a_member_is_dialed_at_its_own_id_whatever_address_a_tuple_carries() { )], ); let mut clean = membership_for(SELF_ADDR); - clean.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + clean.add_member(joiner.clone()); let dial_of = |m: &Membership| { m.known_members() @@ -778,13 +778,13 @@ fn adoption_is_independent_of_the_order_the_evidence_arrives() { )]; let mut forward = membership_for(SELF_ADDR); - forward.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + forward.add_member(joiner.clone()); for v in vouchers { forward.merge_liveness(&NodeId::from(v), claim.clone()); } let mut backward = membership_for(SELF_ADDR); - backward.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + backward.add_member(joiner.clone()); for v in vouchers.iter().rev() { backward.merge_liveness(&NodeId::from(*v), claim.clone()); } @@ -816,7 +816,7 @@ fn a_reaped_member_loses_its_place_and_its_vouches() { let mut m = membership_for(SELF_ADDR); let departing = NodeId::from("10.0.0.1:9000"); let joiner = NodeId::from("10.9.9.9:9000"); - m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + m.add_member(joiner.clone()); m.merge_liveness( &departing, [( @@ -975,7 +975,7 @@ fn two_ids_on_one_host_are_one_voucher() { // once between them; a member on a second host is what carries it. let mut m = shared_host_membership(); let joiner = NodeId::from("10.9.9.9:9000"); - m.add_member(joiner.clone(), joiner.as_bytes().to_vec()); + m.add_member(joiner.clone()); vouch(&mut m, "10.0.0.1:9000", &joiner); vouch(&mut m, "10.0.0.1:9001", &joiner); @@ -994,7 +994,7 @@ fn a_sibling_on_a_members_own_host_does_not_vouch_for_it() { // one trust unit, and the whole bar is that a member cannot place a node by itself. let mut m = shared_host_membership(); let sibling = NodeId::from("10.0.0.1:9002"); - m.add_member(sibling.clone(), sibling.as_bytes().to_vec()); + m.add_member(sibling.clone()); // One voucher on the candidate's own host and one elsewhere: two members, two node // ids, and only *one* trust unit that is not the candidate itself. @@ -1021,7 +1021,7 @@ fn a_certified_member_still_mints_ids_on_its_own_host() { // rather than ids, which is a placement change, not an evidence one. let mut m = membership_for(SELF_ADDR); let minted = NodeId::from("10.0.0.1:9999"); - m.add_member(minted.clone(), minted.as_bytes().to_vec()); + m.add_member(minted.clone()); vouch(&mut m, "10.0.0.2:9000", &minted); vouch(&mut m, "10.0.0.3:9000", &minted); assert!( @@ -1087,7 +1087,7 @@ fn a_node_that_knows_only_itself_places_on_what_it_has_reached() { let first = NodeId::from("10.9.9.9:9000"); let second = NodeId::from("10.9.9.8:9000"); for node in [&first, &second] { - alone.add_member((*node).clone(), node.as_bytes().to_vec()); + alone.add_member((*node).clone()); } alone.note_verified(&first); @@ -1103,10 +1103,123 @@ fn a_node_that_knows_only_itself_places_on_what_it_has_reached() { // adopted from birth and there is a cluster to agree with. let mut seeded = Membership::from_static_config(None, Some(SELF_ADDR), "10.0.0.1:9000", N).unwrap(); - seeded.add_member(first.clone(), first.as_bytes().to_vec()); + seeded.add_member(first.clone()); seeded.note_verified(&first); assert!( !seeded.is_adopted(&first), "one voucher is short of the bar" ); } + +// --- a trust unit is a machine, whatever the host is spelled like --- + +#[test] +fn two_spellings_of_one_host_are_one_voucher() { + // A host is the unit because a host is what a certificate names — so the two have + // to agree on when two spellings are one host. The binding compares IP literals as + // addresses and drops the root label; reading the host as raw text here would let + // one machine present as several vouchers, which is the mint one level up. + for (a, b) in [ + ("evil.example:9000", "evil.example.:9001"), + ("EVIL.example:9000", "evil.EXAMPLE:9001"), + ( + "[2001:db8::6]:9000", + "[2001:0db8:0000:0000:0000:0000:0000:0006]:9001", + ), + ("wss://10.0.0.1:9000", "10.0.0.1:9001"), + ] { + assert_eq!( + crdtsync_server::dial::member_trust_unit(a.as_bytes()), + crdtsync_server::dial::member_trust_unit(b.as_bytes()), + "{a} and {b} are one machine", + ); + } +} + +#[test] +fn a_machine_holding_two_spellings_still_vouches_once() { + // The consequence, end to end: a member that grinds a second id under an alias of + // its own host holds two node ids and one trust unit, so it cannot carry a joiner + // into the ring on its own word. + let mut m = Membership::from_static_config( + None, + Some(SELF_ADDR), + "evil.example:9000,evil.example.:9001,10.0.0.1:9000", + N, + ) + .unwrap(); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone()); + + vouch(&mut m, "evil.example:9000", &joiner); + vouch(&mut m, "evil.example.:9001", &joiner); + assert!(!m.is_adopted(&joiner), "one machine, one voucher"); + + vouch(&mut m, "10.0.0.1:9000", &joiner); + assert!(m.is_adopted(&joiner), "a second machine carries it"); +} + +// --- the ring is a fixpoint, and the bar does not move under it --- + +#[test] +fn adopting_a_member_makes_its_own_vouch_count_in_the_same_pass() { + // The evidence is a chain: one member clears the bar and its word is then what + // carries the next. Evaluating it in a single pass would make the ring depend on + // how many times the evidence happened to be re-merged, which is the same + // history-dependence deriving the set exists to remove. + let mut m = membership_for(SELF_ADDR); + let first = NodeId::from("10.9.9.9:9000"); + let second = NodeId::from("10.9.9.8:9000"); + for node in [&first, &second] { + m.add_member((*node).clone()); + } + // `second` is vouched by one established member and by `first`, whose own word is + // worth nothing until `first` itself clears the bar. + vouch(&mut m, "10.0.0.3:9000", &second); + m.merge_liveness( + &first, + [( + second.clone(), + second.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + vouch(&mut m, "10.0.0.1:9000", &first); + assert!(!m.is_adopted(&first), "still one voucher short"); + assert!(!m.is_adopted(&second)); + + vouch(&mut m, "10.0.0.2:9000", &first); + assert!(m.is_adopted(&first)); + assert!( + m.is_adopted(&second), + "and the newly-adopted member's vouch carried the next, in the same pass", + ); +} + +#[test] +fn reaping_a_seed_does_not_lower_the_bar() { + // The bar is decided at construction, not read off a set that shrinks. A node given + // one seed peer that later departs would otherwise fall to the single-node rule and + // start placing on one vouch, while every peer still held the constant — a ring + // split off an ordinary retirement. + let mut m = Membership::from_static_config(None, Some(SELF_ADDR), "10.0.0.1:9000", N).unwrap(); + let seed = NodeId::from("10.0.0.1:9000"); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone()); + + for _ in 0..crdtsync_server::membership::DEAD_AFTER_FAILURES { + m.note_gossip_unreachable(&seed); + } + for _ in 0..crdtsync_server::membership::REAP_AFTER_DEAD_TICKS { + m.reap_dead(); + } + assert!(!m.is_member(&seed), "the seed departed"); + + m.note_verified(&joiner); + assert!( + !m.is_adopted(&joiner), + "and this node still holds the constant bar", + ); +} diff --git a/crates/server/tests/gossip.rs b/crates/server/tests/gossip.rs index 991e1582..30cd06f2 100644 --- a/crates/server/tests/gossip.rs +++ b/crates/server/tests/gossip.rs @@ -244,12 +244,12 @@ fn placement_is_independent_of_the_order_members_are_learned() { let self_addr = "10.0.0.9:9000"; // One node learns B then C; another learns C then B. let mut bc = alone(self_addr); - bc.add_member(NodeId::from_addr(B), B.as_bytes().to_vec()); - bc.add_member(NodeId::from_addr(C), C.as_bytes().to_vec()); + bc.add_member(NodeId::from_addr(B)); + bc.add_member(NodeId::from_addr(C)); let mut cb = alone(self_addr); - cb.add_member(NodeId::from_addr(C), C.as_bytes().to_vec()); - cb.add_member(NodeId::from_addr(B), B.as_bytes().to_vec()); + cb.add_member(NodeId::from_addr(C)); + cb.add_member(NodeId::from_addr(B)); assert_eq!( member_set(&bc), @@ -291,7 +291,7 @@ fn add_member_is_idempotent_for_a_known_member() { let before = member_set(&m); // Re-adding an existing member (even with a different address) is a no-op: the // set and placement are untouched. - m.add_member(NodeId::from_addr(B), b"different:1234".to_vec()); + m.add_member(NodeId::from_addr(B)); assert_eq!(member_set(&m), before); // The first-learned address wins — no churn. let b_addr = m @@ -308,7 +308,7 @@ fn a_member_with_an_empty_node_id_is_dropped() { // dialable, so it must not poison the member set. let mut m = seeded(A, B); let before = member_set(&m); - m.add_member(NodeId::from(Vec::new()), b"10.0.0.9:9000".to_vec()); + m.add_member(NodeId::from(Vec::new())); assert_eq!(member_set(&m), before, "an empty-id member is not added"); // The same guard holds through the wire merge path. merge_into( @@ -336,9 +336,9 @@ fn a_member_with_an_empty_node_id_is_dropped() { fn a_batch_add_unions_every_new_member_at_once() { let mut m = alone("10.0.0.9:9000"); m.add_members(vec![ - (NodeId::from_addr(A), A.as_bytes().to_vec()), - (NodeId::from_addr(B), B.as_bytes().to_vec()), - (NodeId::from_addr(A), A.as_bytes().to_vec()), // duplicate within the batch + NodeId::from_addr(A), + NodeId::from_addr(B), + NodeId::from_addr(A), // duplicate within the batch ]); let mut expected = vec![ NodeId::from_addr("10.0.0.9:9000"), @@ -414,7 +414,7 @@ fn a_member_learned_by_gossip_becomes_a_placement_target() { .iter() .all(|room| m.primary_for(room) == Some(NodeId::from_addr(self_addr)))); - m.add_member(newcomer_id.clone(), newcomer.as_bytes().to_vec()); + m.add_member(newcomer_id.clone()); // Learned, but pending: it is dialed and gossiped about, and placed nowhere. assert!(sample_rooms() .iter() diff --git a/crates/server/tests/reaping.rs b/crates/server/tests/reaping.rs index 1e0fd343..ca1cf22b 100644 --- a/crates/server/tests/reaping.rs +++ b/crates/server/tests/reaping.rs @@ -219,7 +219,7 @@ fn a_reaped_member_is_not_relearned_from_stale_gossip() { "stale Dead gossip does not resurrect a reaped member" ); // The additive union path is likewise blocked. - m.add_member(d.clone(), D.as_bytes().to_vec()); + m.add_member(d.clone()); assert!( !m.is_member(&d), "a plain re-advertise does not resurrect it either" @@ -277,6 +277,26 @@ fn a_returned_member_with_a_higher_incarnation_is_resurrected() { ); assert!(m.is_member(&d), "a genuinely-returned node rejoins"); assert_eq!(m.gossip_state(&d), MemberState::Alive); + // It rejoins the *roster*, not the ring: reaping struck it from the configured + // members, so it is a fresh join and the cluster verifies it again before rooms are + // placed on it (C25). + assert!( + !m.is_adopted(&d), + "a returned member is pending until verified" + ); + for voucher in [A, B] { + m.merge_liveness( + &nid(voucher), + [( + d.clone(), + D.as_bytes().to_vec(), + reap_inc + 1, + MemberState::Alive, + true, + )], + ); + } + assert!(m.is_adopted(&d), "and takes its place back once it is"); } #[test] From 2a0a5c5bbb2968ef20a05458cf6670b0059e9bd2 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 21:16:45 -0300 Subject: [PATCH 06/17] fix(server): an advertise address is an authority, in one spelling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 3 found two bypasses with one root: an address had many spellings and nothing reduced them. - The dial URL was built from the authority verbatim while the host was read as everything before the first ':'. So 'wss://a.example:1@b.example:9000' read as host 'a.example' and connected to 'b.example': a member certified for 'a.example' alone could grind such an id onto any room, bind its own certificate to it, and let every honest node verify it by dialing the honest node it embedded. A full bypass under mTLS, and not the bounded own-host residual C27 records. - Node ids were never canonicalized, so 'x:9000', 'ws://x:9000', 'WS://x:9000' and 'x:9000/anything' were four members that one honest endpoint answers for. Every peer that dialed any of them verified it truthfully, all were adopted, and none of them ever speaks — so a room whose replica set filled with them waited on acks that never came, targeted per room because HRW is public. An advertise address is now an authority and nothing else — userinfo, a path, a query and a fragment are refused — reduced to one canonical spelling, and canonicalized again at every door an id arrives through: config, a gossip tuple, and a link's claim. Also refuses a present-but-empty CRDTSYNC_CLUSTER_PEERS, which produced a clustered node running the single-node bar of one, and pins the rule the second pass introduced but never tested: an adopted member leaves the ring when a voucher is reaped, because the set is derived and not accumulated. Files C29 for two shapes in which the bar is unsatisfiable — a cluster of two trust units can never re-admit a member it reaped, and a cluster whose members share a host can never admit one. --- ARCHITECTURE.md | 4 +- DECISIONS.md | 2 + KANBAN.md | 4 +- crates/server/src/dial.rs | 139 +++++++++++++++++++++++++++----- crates/server/src/main.rs | 12 +++ crates/server/src/membership.rs | 18 +++-- crates/server/src/placement.rs | 26 ++++-- crates/server/src/registry.rs | 10 ++- crates/server/tests/adoption.rs | 137 +++++++++++++++++++++++++++++++ crates/server/tests/gossip.rs | 44 ++++++---- 10 files changed, 341 insertions(+), 55 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 74b2cdde..2cc5bc28 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -974,7 +974,9 @@ The secret is a **bearer** credential, so the peer link carries it over an encry **A claim is worth exactly the link that carried it.** A `verified` flag is recorded against the member the receiving link is bound to (§Peer Identity) and against nobody the payload names, so a member can assert its own verifications and no one else's; relaying another node's would make one member's word enough to place any id it liked. A tuple claiming the sender itself is dropped — a node's place in the ring is never its own to assert — and a claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. On the *reply* half of a round, the claim counts only where the dial established who answered: under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a `wss://` member's certificate does and a plaintext member's transport does not, and an unattributable claim must not become an adopted member's vouch. The cost of first-hand-only is that verifications travel by direct exchange rather than by relay: a joiner is placed once enough members have each reached it, which randomized peer selection delivers in a handful of gossip rounds rather than one. -**Two trust units, because one of them is the attacker.** A single compromised member would otherwise vouch for the id it ground and place it itself; requiring a second means an honest member must have independently reached that id, which also keeps an unreachable id — one that would pollute placement and stall every quorum it landed in — out of the ring entirely. The bar is a **constant**, not a fraction of the cluster, so two nodes never disagree about what the evidence has to show. **Configured members are adopted from birth**: the operator's config is the root of trust a cluster starts from, and there is no earlier authority for it to be vouched for by. The one exception is a node configured with *no peers at all*, which has no cluster to be outvoted by and which the constant would freeze forever — its ring is the members it has itself reached. That is the single-node deployment; give every node that is meant to join a cluster a seed peer, or it takes its ring from whoever reaches it. +**Two trust units, because one of them is the attacker.** A single compromised member would otherwise vouch for the id it ground and place it itself; requiring a second means an honest member must have independently reached that id, which also keeps an unreachable id — one that would pollute placement and stall every quorum it landed in — out of the ring entirely. The bar is a **constant**, not a fraction of the cluster, so two nodes never disagree about what the evidence has to show. **Configured members are adopted from birth**: the operator's config is the root of trust a cluster starts from, and there is no earlier authority for it to be vouched for by. The one exception is a node configured with *no peers at all*, which has no cluster to be outvoted by and which the constant would freeze forever — its ring is the members it has itself reached. That is the single-node deployment, and it is the peer list being **unset**: set-but-empty is refused at startup, because such a node has a peer plane and would place on its own word alone while every peer it met still held the constant. + +**An advertise address is an authority and nothing else, in one spelling.** Anything past `host:port` — userinfo, a path, a query, a fragment — is refused, and every address is reduced to a single canonical form before it becomes a node id: the redundant `ws://` dropped, the host lowercased with its root label removed, an IP literal reserialized. Both rules exist because a node id *is* an address and placement hashes it. Userinfo makes the host a reader takes out of the string differ from the host a dialer connects to (`wss://a.example:1@b.example:9000` reads as `a.example` and connects to `b.example`), so a certificate for one host would bind an id that every honest peer verifies by dialing *another* — and the holder then speaks as a member of that other node's rooms. A path, a scheme spelling, or an unnormalized host is a free alias, so one endpoint would hold unboundedly many positions in the ring: every peer that dialed any of them would verify it truthfully, all would be adopted, and only one of them ever speaks — so a room whose replica set filled with them would wait on acks that never come. **A member is dialed at its own node id.** A node id *is* an advertise address, so a second address for the same member is a second, unauthenticated name for one thing — and the ring turns on it, because a node dials a member in order to verify it. Keeping the advertised address made the roster first-write-wins over a field any peer may set: whoever advertised a member first decided where every later dial went, so two nodes that saw it in a different order verified different endpoints and placed rooms differently, forever. The address a gossip tuple carries is therefore dropped, which also ends the *reply* half's freedom to point the ring anywhere. diff --git a/DECISIONS.md b/DECISIONS.md index 9211dd9d..19eac8e6 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -33,6 +33,8 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Cold review pass 2 overturned two more, both about what a trust unit *is*.** The unit was keyed on the host as raw text while the certificate binding compares it semantically — lowercased, root label dropped, IP literals parsed as addresses — so `evil.example` beside `evil.example.`, or an IPv6 literal beside its expanded form, were one certificate and two vouchers. One machine therefore held two units and could adopt any id anywhere, which is not the accepted residual (that bounds the mint to the member's own host; this had no bound at all). The unit is now reduced by the same relation the binding uses. And the bootstrap exception read `configured.len() <= 1` off a set that *shrinks*: a node given one seed peer fell to the single-node rule the moment that seed was reaped, and started placing on one vouch while every peer still held the constant — a ring split off an ordinary retirement. It is decided once, at construction. +**Pass 3 found two more, and both are one root: an advertise address had many spellings and nothing reduced them.** `PeerEndpoint::parse` built the dial URL from the authority verbatim while `host_of` took everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as host `a.example` and *dialed* `b.example`. A member holding a certificate for `a.example` alone could therefore grind such an id onto any room, bind its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded — a full bypass under mTLS, and not the bounded own-host residual. And node ids were never canonicalized, so `x:9000`, `ws://x:9000`, `WS://x:9000` and `x:9000/anything` were four members that one honest endpoint answers for: every peer that dialed any of them verified it truthfully, all were adopted, and none of them ever speaks — so a room whose replica set filled with them waited on acks that never came, targeted per room because HRW is public. An advertise address is now an authority and nothing else (userinfo, path, query and fragment refused), reduced to one canonical spelling before it becomes a node id, and canonicalized again at every door an id arrives through — config, a gossip tuple, and a link's claim. Pass 3 also found that a *present but empty* `CRDTSYNC_CLUSTER_PEERS` produced a clustered node running the single-node bar of one; it is refused at startup, since single-node mode is the variable being unset. + **The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. The same gap has one more edge, stated rather than hidden: §Peer Identity deliberately lets a certificate spell a host several ways, and each spelling is a different host, so a machine holding a multi-host certificate holds one trust unit per name it carries. Until C27, a deployment that wants the bar to mean two *machines* issues one host per certificate. **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. diff --git a/KANBAN.md b/KANBAN.md index 2e68c5bf..509019b8 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,7 +30,7 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ -**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. @@ -427,6 +427,8 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C27 — a member that owns a host owns every ring position under it, because placement keys on node ids while a certificate names a host (crates/server) — READY, no dependencies. Found by cold review during C25 (#369) and pinned there as a passing test (`a_certified_member_still_mints_ids_on_its_own_host`).** C25 made rooms place on *adopted* members and made adoption the cluster's decision, which closes minting on anyone else's host, minting an id nobody can reach, and any single member manufacturing the evidence. What it cannot close is the mint on a member's **own** host: HRW scores a node id, C13 binds a link to the *host* a certificate names, and a host answers at as many ports as it likes — so a compromised member grinds `own-host:` until HRW places it on the room it wants, answers there, and every honest node that dials it verifies it **truthfully**. No evidence rule can help, because no one is lying. The consequence is the one C25 removed everywhere else: a member chooses which rooms it replicates, and from inside the replica set it reaches the leadership and durability gates. **The fix is a placement change, not an evidence one: weigh the ring by trust unit.** Score hosts rather than node ids, so a room's replica set is `n` distinct hosts and which rooms a host holds follows from a name it cannot grind; the member serving a host's position is then a local choice among that host's adopted members, and it does not matter which, since they are one trust unit already. That is also the better replication property — `n` replicas on `n` failure domains instead of `n` ids that may share a machine — and it makes the `replicas_for(room).contains(sender)` gate ask about exactly what the certificate vouches for. Costs to settle: two members on one host stop both replicating a room (a capacity change, not a correctness one), the replication factor becomes a host count, and every gate/redirect/quorum that names a member has to resolve host → member. Note this is *not* reachable without a compromised member's certificate: without peer mTLS the whole binding vouches for nothing anyway (C13's residual), and with it the mint is bounded to a host the operator already trusts to run a node. → *Cluster*. +**C29 — a cluster of two adopted trust units can never re-admit a member it reaped, and a cluster whose members share a host can never admit one (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Adoption asks for `ADOPTION_VERIFIERS = 2` trust units other than the candidate's own, and two shapes cannot raise them. **Re-admission:** reaping drops a member from `configured`, so a returning node is a fresh join; in a two-node cluster the only other unit is the surviving node itself, so it verifies the returner a thousand times and the count never passes one — the survivor leads every room, the returner places over both, and the split is permanent with no operator signal and no repair short of restarting the survivor. Reachable by an ordinary long partition with no attacker, and reachable deliberately with C28's unrefutable `Dead`. **Admission:** a cluster whose members all share one host has exactly one unit, so no joiner ever clears the bar — a real shape (several instances per machine or pod), failing closed and silently. Both want the same thing: a bar that is expressible in the cluster it runs in, without becoming node-local (which is what made it non-convergent before). Candidates: a cluster-wide `configured`-derived denominator, an operator-set bar, or — cleanest — C27's placement over trust units, which makes "how many units exist" a thing the ring already knows. At minimum a node should *say* when its roster holds a member the bar cannot admit, instead of quietly not placing it. → *Cluster*. + **C28 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. → *Cluster*. **C26 — the SDK apply seams report a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — READY, no dependencies. Split out of C19 (#367) rather than folded in.** `crdtsync_client_receive`/`WasmClient::receive` fold a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learns how many ops applied *now* and nothing else. C19 made the distinction answerable — `Op::is_admissible` is public and pure — but did not plumb it: a buffered op (which a later arrival commits) and an op no replica will ever hold are both reported as "not counted", so an SDK peered with a buggy or hostile writer sees edits vanish with no signal, and the two cases call for opposite responses (wait vs. surface a client bug). `client.rs` needs no change on this axis — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it, so nothing is missed. The server side is closed: `handle_ops` answers `ErrorCode::MalformedOp` and the ingest seams drop such a record, so this is the *offline / P2P / relay* seam, where no server sits between the writer and the fold. Wants a refused count (or a refused-op list) beside the applied one across FFI, wasm, JS, Python and Go — and is worth taking with **C23**, which has the same shape: a real refusal the SDKs render as an edit that silently did nothing. → *SDK / Ops*. diff --git a/crates/server/src/dial.rs b/crates/server/src/dial.rs index f1b302c1..8b0568d5 100644 --- a/crates/server/src/dial.rs +++ b/crates/server/src/dial.rs @@ -91,6 +91,17 @@ impl PeerEndpoint { if authority.is_empty() { return Err(BadPeerAddress::Empty); } + // An advertise address is an authority and nothing else. A `@`, a path, a query + // or a fragment would make the URL the dialer builds resolve somewhere other + // than the host read out of the same string: `wss://a.example:1@b.example:9000` + // reads as host `a.example` here and connects to `b.example`, so a certificate + // for `a.example` would bind an id that every peer verifies by dialing *`b`*. + // A path is refused for the second half of the same reason — it is a free + // alias, so one endpoint would answer under unboundedly many node ids, each + // separately placed and none of which ever speaks. + if authority.contains(['@', '?', '#', '/']) { + return Err(BadPeerAddress::NotAnAuthority); + } // An authority that names no host — an IPv6 literal left unbracketed, or one // opening with its port separator — is dialable by nobody and bindable to no // certificate. Refused here so every consumer inherits it: a configured member @@ -101,12 +112,8 @@ impl PeerEndpoint { return Err(BadPeerAddress::NoHost); } let scheme = transport.scheme(); - // A bare authority is dialed at the root path, as every advertise address - // in a cluster is; one that already carries a path keeps it verbatim. - let url = match authority.contains('/') { - true => format!("{scheme}://{authority}"), - false => format!("{scheme}://{authority}/"), - }; + // Every advertise address is dialed at the root path. + let url = format!("{scheme}://{authority}/"); Ok(Self { url, transport }) } @@ -137,6 +144,38 @@ pub fn member_host(addr: &[u8]) -> Option { host_of(authority).map(|host| host.to_ascii_lowercase()) } +/// The one spelling of a member's advertise address: no redundant `ws://`, the host +/// lowercased with its root label dropped, and an IP literal reserialized in its +/// canonical form. `None` when the address is not one a peer can dial. +/// +/// A node id **is** an advertise address and placement hashes it, so two spellings of +/// one endpoint are two positions in the ring that one node answers for — each +/// verified *truthfully* by every peer that dials it, each adopted, and none of which +/// ever speaks or acks. A room whose replica set filled up with them would wait on +/// acks that never come. One spelling per endpoint is what keeps the ring's positions +/// and the cluster's members the same set. +pub fn canonical_member_addr(addr: &str) -> Option { + let endpoint = PeerEndpoint::parse(addr).ok()?; + let authority = endpoint.url.split_once("://")?.1.trim_end_matches('/'); + let host = host_of(authority)?; + // Whatever follows the host — its port separator and port, or nothing. + let port = match authority.strip_prefix('[') { + Some(after) => after.split_once(']').map(|(_, rest)| rest)?, + None => authority.find(':').map_or("", |at| &authority[at..]), + }; + let host = normalized(host); + let host = match host.parse::() { + Ok(std::net::IpAddr::V6(v6)) => format!("[{v6}]"), + Ok(ip) => ip.to_string(), + Err(_) => host, + }; + let scheme = match endpoint.transport { + PeerTransport::Tls => "wss://", + PeerTransport::Plain => "", + }; + Some(format!("{scheme}{host}{port}")) +} + /// The **trust unit** an advertise address belongs to: its host reduced to the one /// form the certificate binding compares. `None` when the address names no host. /// @@ -229,6 +268,11 @@ pub enum BadPeerAddress { /// The authority names no host — an unbracketed IPv6 literal, or an address /// opening with its port separator. NoHost, + /// The address carries more than an authority — userinfo, a path, a query or a + /// fragment. Each is a way for the host a reader takes from the string to differ + /// from the host a dialer connects to, or for one endpoint to answer under many + /// node ids. + NotAnAuthority, /// The address carries a scheme that is not `ws` or `wss`. UnknownScheme(String), } @@ -242,6 +286,11 @@ impl std::fmt::Display for BadPeerAddress { "the address names no host, so no peer can dial it — use `host:port`, \ bracketing an IPv6 literal" ), + BadPeerAddress::NotAnAuthority => write!( + f, + "the address carries more than a host and port — use `host:port`, with \ + no user, path, query or fragment" + ), BadPeerAddress::UnknownScheme(scheme) => write!( f, "`{scheme}://` is not a peer transport — use `wss://host:port`, \ @@ -442,10 +491,67 @@ mod tests { } #[test] - fn an_address_that_carries_a_path_keeps_it() { - assert_eq!( - PeerEndpoint::parse("wss://10.0.0.1:9000/sync").unwrap().url, - "wss://10.0.0.1:9000/sync" + fn an_address_that_carries_more_than_an_authority_is_refused() { + // An advertise address is a host and a port. Userinfo would make the host read + // out of the string differ from the host the dial connects to; a path, a query + // and a fragment are free aliases, so one endpoint would answer under + // unboundedly many node ids. + for addr in [ + "wss://10.0.0.1:9000/sync", + "wss://a.example:1@b.example:9000", + "a.example@b.example:9000", + "10.0.0.1:9000?x=1", + "10.0.0.1:9000#f", + ] { + assert_eq!( + PeerEndpoint::parse(addr), + Err(BadPeerAddress::NotAnAuthority), + "{addr}" + ); + assert_eq!(member_host(addr.as_bytes()), None, "{addr}"); + assert_eq!(canonical_member_addr(addr), None, "{addr}"); + } + } + + #[test] + fn a_dialed_url_names_the_same_host_the_binding_reads() { + // The one invariant the refusal above exists for: whatever a reader takes out + // of an address, the dialer connects to the same host. + for addr in ["10.0.0.1:9000", "wss://node-a.example:9000", "[::1]:9000"] { + let url = PeerEndpoint::parse(addr).unwrap().url; + let authority = url.split_once("://").unwrap().1.trim_end_matches('/'); + assert_eq!( + host_of(authority).map(str::to_ascii_lowercase), + member_host(addr.as_bytes()), + "{addr}", + ); + } + } + + #[test] + fn one_endpoint_has_one_canonical_address() { + // Two spellings of one endpoint would be two node ids, so two positions in the + // ring that one node answers for and only one of which ever speaks. + for (a, b) in [ + ("10.0.0.1:9000", "ws://10.0.0.1:9000"), + ("WS://Node-A.Example.:9000", "node-a.example:9000"), + ( + "[2001:0db8:0000:0000:0000:0000:0000:0006]:9000", + "[2001:db8::6]:9000", + ), + (" wss://Node-A.Example:9000 ", "wss://node-a.example:9000"), + ] { + assert_eq!( + canonical_member_addr(a), + canonical_member_addr(b), + "{a}/{b}" + ); + assert!(canonical_member_addr(a).is_some(), "{a}"); + } + // A member's transport is part of its identity, so it is not folded away. + assert_ne!( + canonical_member_addr("wss://node-a.example:9000"), + canonical_member_addr("node-a.example:9000"), ); } @@ -462,7 +568,7 @@ mod tests { // Dialable by nobody and bindable to no certificate, so it fails at the parse // every consumer shares: a configured member at startup, a gossiped one as a // *permanent* dial failure rather than one redialed forever. - for addr in ["::1:9000", ":9000", "[]:9000", "ws:///path", "wss://:9000"] { + for addr in ["::1:9000", ":9000", "[]:9000", "wss://:9000"] { assert_eq!( PeerEndpoint::parse(addr), Err(BadPeerAddress::NoHost), @@ -476,12 +582,7 @@ mod tests { #[test] fn a_bracketed_literal_and_a_bare_host_still_parse() { // The refusal must not catch a legitimate address on its way past. - for addr in [ - "[::1]:9000", - "wss://[2001:db8::1]:9000", - "node-a", - "node-a:9000/x", - ] { + for addr in ["[::1]:9000", "wss://[2001:db8::1]:9000", "node-a"] { assert!(PeerEndpoint::parse(addr).is_ok(), "{addr}"); assert!(member_host(addr.as_bytes()).is_some(), "{addr}"); } @@ -570,10 +671,6 @@ mod tests { member_host(b"wss://node-a.internal:9000").as_deref(), Some("node-a.internal") ); - assert_eq!( - member_host(b"ws://node-a.internal:9000/sync").as_deref(), - Some("node-a.internal") - ); assert_eq!(member_host(b"10.0.0.1:9000").as_deref(), Some("10.0.0.1")); } diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index e78185c8..927bbb91 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -186,6 +186,18 @@ fn membership() -> std::io::Result> { let Some(peers) = path_var("CRDTSYNC_CLUSTER_PEERS")? else { return Ok(None); }; + // Present but empty is refused rather than read as either shape. It is neither + // single-node mode (which is the variable being *unset*) nor a cluster, and taking + // it as a cluster of one gives the node a peer plane while leaving it with nobody + // to be outvoted by — so it would adopt a member into the ring on its own word + // while every peer it later meets still holds the cluster's bar (§Member Adoption). + if peers.trim().is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "CRDTSYNC_CLUSTER_PEERS is set but empty: list this node's seed peers, or \ + unset it for single-node mode", + )); + } let node_id = path_var("CRDTSYNC_NODE_ID")?; let advertise = path_var("CRDTSYNC_ADVERTISE_ADDR")?; let factor = match path_var("CRDTSYNC_REPLICATION_FACTOR")? { diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index 3507ba0c..172f5a5d 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -367,14 +367,17 @@ impl Membership { pub fn add_members(&mut self, members: impl IntoIterator) { let mut added = false; for node in members { + // An address is taken in its one canonical spelling, or not at all: a + // second spelling of one endpoint would be a second member, separately + // placed, that the same node answers for and never speaks as. + let Some(node) = NodeId::canonical(node.as_bytes()) else { + continue; + }; // A reaped member is never re-added by a plain re-advertise: only a live // return (an `Alive` tuple through `merge_liveness`) escapes the // tombstone, so a bare gossip of the member's address — which carries no // liveness — cannot resurrect it. - if node.as_bytes().is_empty() - || self.addrs.contains_key(&node) - || self.reaped.contains_key(&node) - { + if self.addrs.contains_key(&node) || self.reaped.contains_key(&node) { continue; } self.liveness @@ -829,9 +832,12 @@ impl Membership { let mut rebuilt = false; let mut claimed = Vec::new(); for (node, addr, incarnation, state, verified) in payload { - if node.as_bytes().is_empty() { + // One spelling per endpoint, or the tuple names no member at all — the + // same bar the additive path applies, and the reason a malformed id is + // dropped rather than placed. + let Some(node) = NodeId::canonical(node.as_bytes()) else { continue; - } + }; if verified && &node != sender && !self.is_self(&node) { claimed.push(node.clone()); } diff --git a/crates/server/src/placement.rs b/crates/server/src/placement.rs index af5b44e3..4ff2b9fd 100644 --- a/crates/server/src/placement.rs +++ b/crates/server/src/placement.rs @@ -22,13 +22,27 @@ impl NodeId { &self.0 } - /// Derive a node's id from its advertise address. The id is the trimmed - /// address bytes verbatim — a pure function of the address, so every node - /// configured with the same peer address string derives the identical id and - /// the cluster agrees on placement. Trimming absorbs padding from the - /// comma-separated peer-list carrier. + /// Derive a node's id from its advertise address: the address in its one + /// canonical spelling ([`canonical_member_addr`](crate::dial::canonical_member_addr)), + /// so every node configured with any spelling of the same endpoint derives the + /// identical id and the cluster agrees on placement — and one endpoint never holds + /// two positions in the ring. An address that is not one a peer can dial has no + /// canonical form and is kept verbatim (trimmed), so it stays distinguishable for + /// the startup refusal that reports it rather than colliding with another. pub fn from_addr(addr: &str) -> Self { - Self(addr.trim().as_bytes().to_vec()) + match crate::dial::canonical_member_addr(addr) { + Some(canonical) => Self(canonical.into_bytes()), + None => Self(addr.trim().as_bytes().to_vec()), + } + } + + /// The canonical id for an advertise address arriving from outside — a gossip + /// tuple, or the claim a peer link binds to. `None` when the address is not one a + /// peer can dial, so a member nobody could reach never enters a roster or a ring, + /// and a second spelling of a member already known never becomes a second member. + pub fn canonical(addr: &[u8]) -> Option { + let addr = std::str::from_utf8(addr).ok()?; + Some(Self(crate::dial::canonical_member_addr(addr)?.into_bytes())) } } diff --git a/crates/server/src/registry.rs b/crates/server/src/registry.rs index 99b56674..dff93abc 100644 --- a/crates/server/src/registry.rs +++ b/crates/server/src/registry.rs @@ -587,10 +587,12 @@ impl Registry { return false; } // A link that names no member has no identity to gate on, so there is nothing - // to admit it as. - if claimed.is_empty() { + // to admit it as — and it must name it in the one spelling the roster and the + // ring use, or the identity every gate decides against would be a member of + // neither. + let Some(member) = NodeId::canonical(claimed) else { return false; - } + }; let Some(conn) = self.conns.get_mut(&id) else { return false; }; @@ -615,7 +617,7 @@ impl Registry { } Some(_) => {} } - conn.peer = Some(NodeId::from(claimed.to_vec())); + conn.peer = Some(member); // Admitting a link is **not** a verification of the member it names, however // well the certificate names it. A member chooses when to dial in and how // often, so a vouch earned that way is one the member caused rather than one diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index 1fcb7ef2..1cd5688b 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -1223,3 +1223,140 @@ fn reaping_a_seed_does_not_lower_the_bar() { "and this node still holds the constant bar", ); } + +// --- an advertise address names one endpoint, and one endpoint has one id --- + +#[test] +fn an_address_that_carries_more_than_an_authority_is_no_address() { + // The host a reader takes out of the string and the host a dialer connects to must + // be the same one. A URL's userinfo splits them: `a.example:1@b.example:9000` reads + // as `a.example` and connects to `b.example`, so a certificate for `a.example` + // would bind an id that every honest peer verifies by dialing *`b`* — and the + // attacker then speaks as a member of `b`'s rooms. A path splits nothing but is a + // free alias, which is the other half of the same defect. + for addr in [ + "wss://evil.example:1@10.0.0.1:9000", + "evil.example@10.0.0.1:9000", + "10.0.0.1:9000/x", + "ws://10.0.0.1:9000/a/b", + "10.0.0.1:9000?x=1", + "10.0.0.1:9000#f", + ] { + assert!( + crdtsync_server::dial::PeerEndpoint::parse(addr).is_err(), + "{addr} is not an advertise address", + ); + assert!( + crdtsync_server::dial::canonical_member_addr(addr).is_none(), + "{addr} has no canonical form", + ); + } +} + +#[test] +fn an_id_that_embeds_another_host_reaches_no_roster_and_no_link() { + // End to end: the ground id is refused at every door. Gossip does not learn it, so + // no node ever dials the honest host it embeds and verifies it; and no link can + // bind to it, so it speaks as nobody even if some peer had. + let minted = NodeId::from("wss://evil.example:1@10.0.0.1:9000"); + let mut r = registry(); + r.merge_gossip( + &NodeId::from("10.0.0.1:9000"), + advertisements(&[&minted], false), + ); + let view = r.membership().expect("clustered"); + assert!(!view.is_member(&minted), "it is no member"); + assert!(!view.is_adopted(&minted)); + + let id = r.connect(); + assert!( + !r.deliver( + id, + Message::PeerAuth { + node: minted.as_bytes().to_vec(), + secret: SECRET.to_vec(), + }, + ), + "and no link binds to it", + ); +} + +#[test] +fn one_endpoint_holds_one_node_id() { + // A node id *is* an advertise address and placement hashes it, so two spellings of + // one endpoint would be two positions in the ring that one node answers for. Every + // peer that dialed either would verify it truthfully and both would be adopted — + // and only one of them ever speaks, so a room that placed on the other waits on an + // ack that never comes. The spellings collapse before the roster sees them. + let canonical = NodeId::from("10.9.9.9:9000"); + let spellings = [ + "10.9.9.9:9000", + "ws://10.9.9.9:9000", + "WS://10.9.9.9:9000", + " 10.9.9.9:9000 ", + ]; + for spelling in spellings { + assert_eq!(NodeId::from_addr(spelling), canonical, "{spelling}"); + } + + let mut m = membership_for(SELF_ADDR); + let before = m.members().len(); + m.add_members(spellings.iter().map(|s| NodeId::from(*s))); + assert_eq!( + m.members().len(), + before + 1, + "every spelling is the same one member", + ); + assert!(m.is_member(&canonical)); +} + +#[test] +fn a_tls_member_keeps_its_scheme_and_a_plain_one_drops_it() { + // The canonical form is not "strip the scheme": a member's transport is part of its + // identity (§Peer Transport), and dropping `wss://` would make a TLS member and a + // plaintext one on the same authority one id. + assert_eq!( + NodeId::from_addr("WSS://Node-A.Example.:9000"), + NodeId::from("wss://node-a.example:9000"), + ); + assert_ne!( + NodeId::from_addr("wss://node-a.example:9000"), + NodeId::from_addr("ws://node-a.example:9000"), + ); + assert_eq!( + NodeId::from_addr("[2001:0DB8:0000:0000:0000:0000:0000:0006]:9000"), + NodeId::from_addr("[2001:db8::6]:9000"), + ); +} + +#[test] +fn an_adopted_member_is_un_adopted_when_a_voucher_is_reaped() { + // The ring is *derived* from the evidence, not accumulated as it goes past. So + // evidence that goes away takes the placement with it — which is the whole point: + // a node that kept the member placed because it happened to see the vouches before + // the reap would disagree, permanently, with one that saw them after. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + let voucher = NodeId::from("10.0.0.1:9000"); + m.add_member(joiner.clone()); + vouch(&mut m, "10.0.0.1:9000", &joiner); + vouch(&mut m, "10.0.0.2:9000", &joiner); + assert!(m.is_adopted(&joiner)); + + for _ in 0..crdtsync_server::membership::DEAD_AFTER_FAILURES { + m.note_gossip_unreachable(&voucher); + } + for _ in 0..crdtsync_server::membership::REAP_AFTER_DEAD_TICKS { + m.reap_dead(); + } + assert!(!m.is_member(&voucher), "the voucher departed"); + assert!( + !m.is_adopted(&joiner), + "and the member it carried leaves the ring with it", + ); + + // A replacement voucher puts it back, so this is the evidence speaking and not a + // one-way door. + vouch(&mut m, "10.0.0.3:9000", &joiner); + assert!(m.is_adopted(&joiner)); +} diff --git a/crates/server/tests/gossip.rs b/crates/server/tests/gossip.rs index 30cd06f2..4870e731 100644 --- a/crates/server/tests/gossip.rs +++ b/crates/server/tests/gossip.rs @@ -400,28 +400,40 @@ fn a_single_node_registry_knows_no_members_and_ignores_gossip() { #[test] fn a_member_learned_by_gossip_becomes_a_placement_target() { - // Start knowing only self, which leads every room. Learn a peer by gossip, then - // some rooms place on — and are led by — the newcomer, so it is a live redirect - // and replica target (effective_primary_for elects it; the redirect uses that - // leader's address). + // Learn a peer by gossip and have the cluster verify it; then some rooms place on + // — and are led by — the newcomer, so it is a live redirect and replica target + // (effective_primary_for elects it; the redirect uses that leader's address). let self_addr = "10.0.0.9:9000"; - let mut m = alone(self_addr); + // Seeded with peers, so the newcomer clears the cluster's bar rather than the + // single-node exception: two adopted trust units have to have reached it. + let mut m = seeded(self_addr, &format!("{A},{B}")); let newcomer = "10.0.0.7:9000"; let newcomer_id = NodeId::from_addr(newcomer); + let places_on_newcomer = |m: &Membership| { + sample_rooms() + .iter() + .any(|room| m.replicas_for(room).contains(&newcomer_id)) + }; - // Before learning it, the newcomer leads nothing here. - assert!(sample_rooms() - .iter() - .all(|room| m.primary_for(room) == Some(NodeId::from_addr(self_addr)))); - + assert!(!places_on_newcomer(&m), "unknown, so it holds nothing"); m.add_member(newcomer_id.clone()); - // Learned, but pending: it is dialed and gossiped about, and placed nowhere. - assert!(sample_rooms() - .iter() - .all(|room| m.primary_for(room) == Some(NodeId::from_addr(self_addr)))); - // This node then reaches it over an identity-checked link, which is the whole - // cluster's verdict here — there is nobody else to raise. + assert!( + !places_on_newcomer(&m), + "learned but pending: dialed and gossiped about, placed nowhere", + ); + // This node reaches it over an identity-checked link, and one of its peers reports + // having done the same — two trust units, which is the cluster's bar. m.note_verified(&newcomer_id); + m.merge_liveness( + &NodeId::from_addr(A), + [( + newcomer_id.clone(), + newcomer.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); let room = sample_rooms() .into_iter() From 5e1cb5ba4f3a843e38fa549504626c177fa84d0b Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 21:51:11 -0300 Subject: [PATCH 07/17] fix(server): the id a node is configured with is the id the cluster spells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 4, after pass 3 had approved the address story. All three are in how an id is derived from *configuration* rather than in the evidence machinery. - The bootstrap exception read `configured.len() <= 1` — a set, after canonicalization and de-duplication — so a peer list whose every entry spelled this node's own address collapsed to one member and dropped the bar to a single vouch. That is the state the empty-list refusal had just been added to catch, reached without the variable being empty, and a templated peer list on a one-replica deployment reaches it. The bar now reads the list for a peer other than self, and such a list is refused. - `CRDTSYNC_NODE_ID` was the one door that did not canonicalize, a regression from the plain trim it replaced. A node written in a non-canonical spelling kept it while its peers derived the canonical one, so it carried a doppelganger of itself: adopted on two honest vouchers, placed on rooms it never answers for, unable to refute a suspicion of itself, and dropping every follower-head report because the link was bound to the other spelling. - The canonical form reduced the host but not the port, so `:9000`, `:09000` and `:009000` were three node ids on one listener — one certificate, one trust unit, unbounded ring positions. That is the targeted write-stall the previous pass was meant to have closed, and an operator reaches it with no attacker by writing a leading zero. An address with no canonical form is now refused where it is written rather than joined under, a frame on a link claiming this node's own id makes it vouch for nobody, and an authority may carry only the characters a host and a port are written with — so a malformed address is a permanent failure rather than one redialed forever. Two tests held for the wrong reason and are replaced: the two-spellings vouch test could not fail (config had already canonicalized them), and the one-endpoint-one-id test had no port spellings, which is what hid the port. --- ARCHITECTURE.md | 6 +- DECISIONS.md | 2 + KANBAN.md | 2 +- crates/server/src/dial.rs | 28 +++++- crates/server/src/main.rs | 10 ++- crates/server/src/membership.rs | 64 ++++++++++++-- crates/server/tests/adoption.rs | 127 +++++++++++++++++++++++---- crates/server/tests/peer_identity.rs | 28 +++++- crates/server/tests/reaping.rs | 4 +- 9 files changed, 234 insertions(+), 37 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2cc5bc28..3ba7baf7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -972,11 +972,11 @@ The secret is a **bearer** credential, so the peer link carries it over an encry **A verifier is a trust unit, and a trust unit is a host.** A host is what a certificate names (§Peer Identity) and one host holds as many node ids as it likes, so the bar counts the distinct *hosts* of the adopted members that vouched, not the ids. Counting ids would let one machine holding two of them raise the whole bar by itself. A member's own host is excluded from its own count for the same reason: a member vouching for a sibling on its own host is vouching for itself. **The host is reduced by the same relation the certificate binding compares by** — lowercased, root label dropped, IP literals compared as addresses — because the two must agree on when two spellings are one host: reading the host as raw text while the binding read it semantically would let one machine present as several vouchers under `evil.example` beside `evil.example.`, or an IPv6 literal beside its expanded form. -**A claim is worth exactly the link that carried it.** A `verified` flag is recorded against the member the receiving link is bound to (§Peer Identity) and against nobody the payload names, so a member can assert its own verifications and no one else's; relaying another node's would make one member's word enough to place any id it liked. A tuple claiming the sender itself is dropped — a node's place in the ring is never its own to assert — and a claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. On the *reply* half of a round, the claim counts only where the dial established who answered: under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a `wss://` member's certificate does and a plaintext member's transport does not, and an unattributable claim must not become an adopted member's vouch. The cost of first-hand-only is that verifications travel by direct exchange rather than by relay: a joiner is placed once enough members have each reached it, which randomized peer selection delivers in a handful of gossip rounds rather than one. +**A claim is worth exactly the link that carried it.** A `verified` flag is recorded against the member the receiving link is bound to (§Peer Identity) and against nobody the payload names — and never against *this* node, so a frame arriving on a link that claims this node's own id cannot make it vouch for a member it never dialed — so a member can assert its own verifications and no one else's; relaying another node's would make one member's word enough to place any id it liked. A tuple claiming the sender itself is dropped — a node's place in the ring is never its own to assert — and a claim by a member that is not itself adopted does not count, so two nodes minted together cannot vouch each other in. On the *reply* half of a round, the claim counts only where the dial established who answered: under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a `wss://` member's certificate does and a plaintext member's transport does not, and an unattributable claim must not become an adopted member's vouch. The cost of first-hand-only is that verifications travel by direct exchange rather than by relay: a joiner is placed once enough members have each reached it, which randomized peer selection delivers in a handful of gossip rounds rather than one. -**Two trust units, because one of them is the attacker.** A single compromised member would otherwise vouch for the id it ground and place it itself; requiring a second means an honest member must have independently reached that id, which also keeps an unreachable id — one that would pollute placement and stall every quorum it landed in — out of the ring entirely. The bar is a **constant**, not a fraction of the cluster, so two nodes never disagree about what the evidence has to show. **Configured members are adopted from birth**: the operator's config is the root of trust a cluster starts from, and there is no earlier authority for it to be vouched for by. The one exception is a node configured with *no peers at all*, which has no cluster to be outvoted by and which the constant would freeze forever — its ring is the members it has itself reached. That is the single-node deployment, and it is the peer list being **unset**: set-but-empty is refused at startup, because such a node has a peer plane and would place on its own word alone while every peer it met still held the constant. +**Two trust units, because one of them is the attacker.** A single compromised member would otherwise vouch for the id it ground and place it itself; requiring a second means an honest member must have independently reached that id, which also keeps an unreachable id — one that would pollute placement and stall every quorum it landed in — out of the ring entirely. The bar is a **constant**, not a fraction of the cluster, so two nodes never disagree about what the evidence has to show. **Configured members are adopted from birth**: the operator's config is the root of trust a cluster starts from, and there is no earlier authority for it to be vouched for by. The one exception is a node configured with *no peers at all*, which has no cluster to be outvoted by and which the constant would freeze forever — its ring is the members it has itself reached. That is the single-node deployment, and it is the peer list being **unset**. A peer list that is set but names nobody other than this node — empty, or every entry a spelling of this node's own address — is refused at startup: such a node has a peer plane and a cluster secret, and would place a member on rooms on its own word alone while every peer it met still held the constant. The exception is decided from the *list*, not from the member count it collapses to, so de-duplication cannot lower the bar. -**An advertise address is an authority and nothing else, in one spelling.** Anything past `host:port` — userinfo, a path, a query, a fragment — is refused, and every address is reduced to a single canonical form before it becomes a node id: the redundant `ws://` dropped, the host lowercased with its root label removed, an IP literal reserialized. Both rules exist because a node id *is* an address and placement hashes it. Userinfo makes the host a reader takes out of the string differ from the host a dialer connects to (`wss://a.example:1@b.example:9000` reads as `a.example` and connects to `b.example`), so a certificate for one host would bind an id that every honest peer verifies by dialing *another* — and the holder then speaks as a member of that other node's rooms. A path, a scheme spelling, or an unnormalized host is a free alias, so one endpoint would hold unboundedly many positions in the ring: every peer that dialed any of them would verify it truthfully, all would be adopted, and only one of them ever speaks — so a room whose replica set filled with them would wait on acks that never come. +**An advertise address is an authority and nothing else, in one spelling.** Anything past `host:port` is refused — userinfo, a path, a query, a fragment, and any character a host and a port are not written with — and every address is reduced to a single canonical form before it becomes a node id: the redundant `ws://` dropped, the host lowercased with its root label removed, an IP literal reserialized, and **the port read as a number** so `:9000` and `:09000` are one port on one listener. An address with no canonical form is refused where it is written — in the peer list, in `CRDTSYNC_NODE_ID`, and at every door an id arrives through — rather than joined under, because an id nobody can dial names a member the cluster can never verify and its own node can never be recognised as. Both rules exist because a node id *is* an address and placement hashes it. Userinfo makes the host a reader takes out of the string differ from the host a dialer connects to (`wss://a.example:1@b.example:9000` reads as `a.example` and connects to `b.example`), so a certificate for one host would bind an id that every honest peer verifies by dialing *another* — and the holder then speaks as a member of that other node's rooms. A path, a scheme spelling, or an unnormalized host is a free alias, so one endpoint would hold unboundedly many positions in the ring: every peer that dialed any of them would verify it truthfully, all would be adopted, and only one of them ever speaks — so a room whose replica set filled with them would wait on acks that never come. **A member is dialed at its own node id.** A node id *is* an advertise address, so a second address for the same member is a second, unauthenticated name for one thing — and the ring turns on it, because a node dials a member in order to verify it. Keeping the advertised address made the roster first-write-wins over a field any peer may set: whoever advertised a member first decided where every later dial went, so two nodes that saw it in a different order verified different endpoints and placed rooms differently, forever. The address a gossip tuple carries is therefore dropped, which also ends the *reply* half's freedom to point the ring anywhere. diff --git a/DECISIONS.md b/DECISIONS.md index 19eac8e6..bb968e33 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -35,6 +35,8 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Pass 3 found two more, and both are one root: an advertise address had many spellings and nothing reduced them.** `PeerEndpoint::parse` built the dial URL from the authority verbatim while `host_of` took everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as host `a.example` and *dialed* `b.example`. A member holding a certificate for `a.example` alone could therefore grind such an id onto any room, bind its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded — a full bypass under mTLS, and not the bounded own-host residual. And node ids were never canonicalized, so `x:9000`, `ws://x:9000`, `WS://x:9000` and `x:9000/anything` were four members that one honest endpoint answers for: every peer that dialed any of them verified it truthfully, all were adopted, and none of them ever speaks — so a room whose replica set filled with them waited on acks that never came, targeted per room because HRW is public. An advertise address is now an authority and nothing else (userinfo, path, query and fragment refused), reduced to one canonical spelling before it becomes a node id, and canonicalized again at every door an id arrives through — config, a gossip tuple, and a link's claim. Pass 3 also found that a *present but empty* `CRDTSYNC_CLUSTER_PEERS` produced a clustered node running the single-node bar of one; it is refused at startup, since single-node mode is the variable being unset. +**Pass 4 found three more, all in how an id is derived from *configuration* rather than in the evidence machinery.** The bootstrap exception read `configured.len() <= 1` — a set, after canonicalization and de-duplication — so a peer list whose every entry spelled this node's own address collapsed to one member and dropped the bar to a single vouch, which is exactly the state the empty-list refusal had just been added to catch, reached without the variable being empty; it now reads the *list* for a peer other than self, and such a list is refused at startup. `CRDTSYNC_NODE_ID` was the one door that did not canonicalize, a regression from the plain `trim()` it replaced: a node written `wss://Node-A.Example:9000` kept that spelling as its own id while its peers derived the canonical one, so it carried a **doppelgänger of itself** in its roster — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, and dropping every follower-head report because the link was bound to the other spelling. And the canonical form canonicalized the host but not the **port**, so `:9000`, `:09000` and `:009000` were three node ids on one listener — one certificate, one trust unit, unbounded ring positions, which is the targeted write-stall pass 3 was supposed to have closed, and which an operator reaches with no attacker at all by writing a leading zero in one node's peer list. The port is now read as a number, and an address that has no canonical form is refused where it is written. + **The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. The same gap has one more edge, stated rather than hidden: §Peer Identity deliberately lets a certificate spell a host several ways, and each spelling is a different host, so a machine holding a multi-host certificate holds one trust unit per name it carries. Until C27, a deployment that wants the bar to mean two *machines* issues one host per certificate. **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. diff --git a/KANBAN.md b/KANBAN.md index 509019b8..f4d3b532 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,7 +30,7 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ -**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Pass 4 found three more, all in how an id is derived from configuration:** the bootstrap exception read a *de-duplicated set*, so a peer list of only this node's own address collapsed to one member and dropped the bar to a single vouch — the very state the empty-list refusal had been added to catch; `CRDTSYNC_NODE_ID` was the one door that did not canonicalize (a regression from the plain `trim()` it replaced), so a node written in a non-canonical spelling carried a **doppelgänger of itself** — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, dropping every follower-head report; and the port was not canonicalized, so `:9000`/`:09000`/`:009000` were three ring positions on one listener, which is the targeted write-stall pass 3 was supposed to have closed and which an operator reaches with no attacker by writing a leading zero. The bar now reads the list rather than the set, every door canonicalizes, the port is read as a number, and an address with no canonical form is refused where it is written. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. diff --git a/crates/server/src/dial.rs b/crates/server/src/dial.rs index 8b0568d5..01198845 100644 --- a/crates/server/src/dial.rs +++ b/crates/server/src/dial.rs @@ -99,7 +99,16 @@ impl PeerEndpoint { // A path is refused for the second half of the same reason — it is a free // alias, so one endpoint would answer under unboundedly many node ids, each // separately placed and none of which ever speaks. - if authority.contains(['@', '?', '#', '/']) { + // Only the characters a host and a port are written with. `@`, `/`, `?` and + // `#` are the ones that change where a dial lands or hand one endpoint a + // second id; the rest of the refusal is what keeps an unreachable address from + // being *retried forever* — a control character or a space builds a URL the + // dialer rejects at send time, which reads as "unreachable" and is redialed on + // the fast cadence rather than classified a permanent address error. + if !authority + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b':' | b'[' | b']')) + { return Err(BadPeerAddress::NotAnAuthority); } // An authority that names no host — an IPv6 literal left unbracketed, or one @@ -159,10 +168,20 @@ pub fn canonical_member_addr(addr: &str) -> Option { let authority = endpoint.url.split_once("://")?.1.trim_end_matches('/'); let host = host_of(authority)?; // Whatever follows the host — its port separator and port, or nothing. - let port = match authority.strip_prefix('[') { + let after_host = match authority.strip_prefix('[') { Some(after) => after.split_once(']').map(|(_, rest)| rest)?, None => authority.find(':').map_or("", |at| &authority[at..]), }; + // The port is a *number*, so it is canonicalized as one: `:9000`, `:09000` and + // `:+9000` are one port on one listener, and leaving them as text would give one + // endpoint unboundedly many node ids — separately placed, and only one of them + // ever answering. An absent port and an empty one are the same absence. A port + // that is no port has no canonical form and the address is refused. + let port = match after_host.strip_prefix(':') { + None => None, + Some("") => None, + Some(digits) => Some(digits.parse::().ok()?), + }; let host = normalized(host); let host = match host.parse::() { Ok(std::net::IpAddr::V6(v6)) => format!("[{v6}]"), @@ -173,7 +192,10 @@ pub fn canonical_member_addr(addr: &str) -> Option { PeerTransport::Tls => "wss://", PeerTransport::Plain => "", }; - Some(format!("{scheme}{host}{port}")) + Some(match port { + Some(port) => format!("{scheme}{host}:{port}"), + None => format!("{scheme}{host}"), + }) } /// The **trust unit** an advertise address belongs to: its host reduced to the one diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 927bbb91..ba7d2ea7 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -215,10 +215,14 @@ fn membership() -> std::io::Result> { let m = Membership::from_static_config(node_id.as_deref(), advertise.as_deref(), &peers, factor) .map_err(|e| { + // Every membership config error is the same class of operator mistake + // and every one of them refuses the start. The match stays exhaustive + // with no catch-all, so a new one has to be classified here. let kind = match e { - MembershipConfigError::EmptyPeer | MembershipConfigError::MissingSelfId => { - std::io::ErrorKind::InvalidInput - } + MembershipConfigError::EmptyPeer + | MembershipConfigError::MissingSelfId + | MembershipConfigError::NotAnAddress(_) + | MembershipConfigError::NoPeerButSelf => std::io::ErrorKind::InvalidInput, }; std::io::Error::new(kind, e) })?; diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index 172f5a5d..ee1e983a 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -152,6 +152,16 @@ pub enum MembershipConfigError { /// Peers were configured but the node has no advertise address or node id, so /// it cannot place itself in its own cluster. MissingSelfId, + /// A configured address is not one a peer could dial — it carries more than a + /// host and a port, names no host, or names a port that is no port. Refused + /// rather than joined under, because an id nobody can reach is a member the + /// cluster can never verify and this node can never be recognised as. + NotAnAddress(String), + /// A peer list was configured but named nobody except this node. Such a node has + /// a peer plane and a cluster secret and yet no cluster to be outvoted by, so its + /// adoption bar would fall to a single vouch — it would place a member on rooms + /// on its own word while every peer it met still held the cluster's bar. + NoPeerButSelf, } impl std::fmt::Display for MembershipConfigError { @@ -164,6 +174,16 @@ impl std::fmt::Display for MembershipConfigError { f, "cluster peers configured but no node id or advertise address for self" ), + MembershipConfigError::NotAnAddress(addr) => write!( + f, + "`{addr}` is not an address a peer can dial — use `host:port`, with no \ + user, path, query or fragment, and bracket an IPv6 literal" + ), + MembershipConfigError::NoPeerButSelf => write!( + f, + "the cluster peer list names nobody but this node: list this node's \ + seed peers, or unset it for single-node mode" + ), } } } @@ -258,6 +278,14 @@ impl Membership { peers: impl IntoIterator, replication_factor: usize, ) -> Self { + let peers: Vec = peers.into_iter().collect(); + // Whether the configuration named a peer *other than this node* — not how many + // distinct members it collapsed to. A peer list whose every entry canonicalizes + // to this node's own id de-duplicates down to one member, and a bar read off + // that count would fall to the single-node rule on a node that has a peer plane + // and a cluster secret. That is the shape the startup refusal exists to catch, + // reached without the variable being empty. + let solitary = !peers.iter().any(|peer| peer != &self_id); let members: Vec = std::iter::once(self_id.clone()).chain(peers).collect(); // Every seeded member dials at its node id — the identity each is derived // from (`NodeId::from_addr`), so the id and the dial address are the same @@ -277,7 +305,6 @@ impl Membership { // root of trust a cluster starts from, and there is no earlier authority for // it to be vouched for by. let configured: HashSet = members.iter().cloned().collect(); - let solitary = configured.len() <= 1; Self { self_id, cluster: Cluster::new(members), @@ -311,12 +338,26 @@ impl Membership { // derives the same id every peer's trimmed `from_addr` does. let node_id = node_id.map(str::trim).filter(|s| !s.is_empty()); let advertise_addr = advertise_addr.map(str::trim).filter(|s| !s.is_empty()); - let self_id = match (node_id, advertise_addr) { - (Some(id), _) => NodeId::from(id), - (None, Some(addr)) => NodeId::from_addr(addr), + let written = match (node_id, advertise_addr) { + (Some(id), _) => id, + (None, Some(addr)) => addr, (None, None) => return Err(MembershipConfigError::MissingSelfId), }; + // The explicit id goes through the same canonicalization every other door + // applies. Left verbatim it would name a member the cluster spells differently: + // this node's peers would learn its canonical id, this node would not recognise + // that id as itself, and it would carry a *doppelgänger* of itself in its own + // roster — adopted on two honest vouchers, placed on rooms it never answers + // for, unable to refute a suspicion of itself, and dropping every follower-head + // report because the link is bound to the other spelling. + let self_id = NodeId::from_addr(written); + if crate::dial::canonical_member_addr(written).is_none() { + return Err(MembershipConfigError::NotAnAddress(written.to_string())); + } let peers = parse_peers(peers)?; + if !peers.is_empty() && !peers.iter().any(|peer| peer != &self_id) { + return Err(MembershipConfigError::NoPeerButSelf); + } Ok(Self::new(self_id, peers, replication_factor)) } @@ -838,7 +879,9 @@ impl Membership { let Some(node) = NodeId::canonical(node.as_bytes()) else { continue; }; - if verified && &node != sender && !self.is_self(&node) { + // A claim is the *sender's*, so a frame that arrives on a link bound to this + // node's own id cannot make this node vouch for a member it never dialed. + if verified && &node != sender && !self.is_self(&node) && !self.is_self(sender) { claimed.push(node.clone()); } if self.is_self(&node) { @@ -949,9 +992,14 @@ fn parse_peers(list: &str) -> Result, MembershipConfigError> { .map(|entry| { let entry = entry.trim(); if entry.is_empty() { - Err(MembershipConfigError::EmptyPeer) - } else { - Ok(NodeId::from_addr(entry)) + return Err(MembershipConfigError::EmptyPeer); + } + // A peer with no canonical form is one no member of this cluster could + // dial or agree with, so it is refused where it is written rather than + // seeded verbatim into a ring nobody else computes the same way. + match crate::dial::canonical_member_addr(entry) { + Some(_) => Ok(NodeId::from_addr(entry)), + None => Err(MembershipConfigError::NotAnAddress(entry.to_string())), } }) .collect() diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index 1cd5688b..b99679da 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -1137,26 +1137,24 @@ fn two_spellings_of_one_host_are_one_voucher() { } #[test] -fn a_machine_holding_two_spellings_still_vouches_once() { - // The consequence, end to end: a member that grinds a second id under an alias of - // its own host holds two node ids and one trust unit, so it cannot carry a joiner - // into the ring on its own word. - let mut m = Membership::from_static_config( +fn a_machine_holding_two_spellings_holds_one_member() { + // Canonicalization is what makes the trust unit hard to game: two spellings of one + // machine are not two vouchers because they are not two *members*. The reduction in + // the unit count is the second line of the same defence, for an id that ever + // reached the roster unreduced. + let m = Membership::from_static_config( None, Some(SELF_ADDR), - "evil.example:9000,evil.example.:9001,10.0.0.1:9000", + "evil.example:9000,EVIL.example.:9000,ws://evil.example:09000", N, ) .unwrap(); - let joiner = NodeId::from("10.9.9.9:9000"); - m.add_member(joiner.clone()); - - vouch(&mut m, "evil.example:9000", &joiner); - vouch(&mut m, "evil.example.:9001", &joiner); - assert!(!m.is_adopted(&joiner), "one machine, one voucher"); - - vouch(&mut m, "10.0.0.1:9000", &joiner); - assert!(m.is_adopted(&joiner), "a second machine carries it"); + let members: Vec<_> = m + .members() + .into_iter() + .filter(|node| node != &NodeId::from(SELF_ADDR)) + .collect(); + assert_eq!(members, vec![NodeId::from("evil.example:9000")]); } // --- the ring is a fixpoint, and the bar does not move under it --- @@ -1294,6 +1292,9 @@ fn one_endpoint_holds_one_node_id() { "ws://10.9.9.9:9000", "WS://10.9.9.9:9000", " 10.9.9.9:9000 ", + // A port is a number, not text: one listener, however it is written. + "10.9.9.9:09000", + "10.9.9.9:009000", ]; for spelling in spellings { assert_eq!(NodeId::from_addr(spelling), canonical, "{spelling}"); @@ -1360,3 +1361,99 @@ fn an_adopted_member_is_un_adopted_when_a_voucher_is_reaped() { vouch(&mut m, "10.0.0.3:9000", &joiner); assert!(m.is_adopted(&joiner)); } + +// --- the id a node is configured with is the id the cluster spells --- + +#[test] +fn a_peer_list_naming_nobody_but_this_node_is_refused() { + // Such a node has a peer plane, a cluster secret and gossip, and yet no cluster to + // be outvoted by — its bar would fall to a single vouch and it would place a member + // on rooms on its own word while every peer it met still held the cluster's bar. + // The refusal reads the list, not the member count it collapses to: every spelling + // below canonicalizes to this node's own id and de-duplicates away. + for peers in [ + SELF_ADDR, + "WS://10.0.0.6:9000", + " 10.0.0.6:09000 ", + "10.0.0.6:9000,ws://10.0.0.6:9000", + ] { + let e = Membership::from_static_config(None, Some(SELF_ADDR), peers, N) + .expect_err("a peer list of only self is refused"); + assert!( + e.to_string().contains("names nobody but this node"), + "{peers}" + ); + } + // A list that names one real peer is a cluster, and is accepted. + assert!(Membership::from_static_config(None, Some(SELF_ADDR), "10.0.0.1:9000", N).is_ok()); +} + +#[test] +fn an_explicit_node_id_is_the_id_the_cluster_spells() { + // The explicit-id door canonicalizes like every other. Left verbatim, a node would + // carry a *doppelgänger* of itself: its peers learn its canonical id, it does not + // recognise that id as itself, two honest members vouch for it truthfully, and it + // is adopted — rooms placed on a member that never answers, a suspicion of itself + // it can never refute, and every follower-head report dropped because the link is + // bound to the other spelling. + let written = "WSS://Node-A.Example.:09000"; + let m = Membership::from_static_config(Some(written), None, "wss://node-b.example:9000", N) + .unwrap(); + assert_eq!(m.self_id(), &NodeId::from("wss://node-a.example:9000")); + assert_eq!(m.self_id(), &NodeId::from_addr(written)); + + // So the canonical id arriving by gossip is recognised as itself and never learned + // as a second member. + let mut m = m; + let before = m.members().len(); + m.add_member(NodeId::from_addr(written)); + m.add_member(NodeId::from("wss://node-a.example:9000")); + assert_eq!(m.members().len(), before, "no doppelgänger"); +} + +#[test] +fn a_configured_address_no_peer_could_dial_is_refused() { + // An id with no canonical form names a member the cluster could never verify and + // this node could never be recognised as, so it is refused where it is written + // rather than joined under. + for id in [ + "::1:9000", + "wss://a.example:1@b.example:9000", + "10.0.0.1:9000/sync", + "10.0.0.1:99999", + "10.0.0.1:nine", + "10.0.0.1:+9000", + "10.0.0.1 :9000", + ] { + assert!( + Membership::from_static_config(Some(id), None, "10.0.0.1:9000", N).is_err(), + "{id}", + ); + } +} + +#[test] +fn a_link_bound_to_this_nodes_own_id_makes_it_vouch_for_nobody() { + // A claim is the sender's. A frame arriving on a link that names *this* node would + // otherwise insert this node into a member's verifier set — a whole trust unit, + // for a member this node never dialed. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone()); + let me = NodeId::from(SELF_ADDR); + m.merge_liveness( + &me, + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + assert!( + !m.has_verified(&me, &joiner), + "this node vouched for nobody" + ); + assert!(!m.is_adopted(&joiner)); +} diff --git a/crates/server/tests/peer_identity.rs b/crates/server/tests/peer_identity.rs index 1298332a..7e6316cc 100644 --- a/crates/server/tests/peer_identity.rs +++ b/crates/server/tests/peer_identity.rs @@ -954,6 +954,14 @@ mod live { Membership::from_static_config(Some(me), None, other, 2).unwrap() } + /// The same two-member view, built **without** the config validation. C25 refuses an + /// address with no canonical form where it is written, so the startup refusals below + /// — which are about a member that reaches the *serve* path naming no host — need a + /// membership that did not go through that door to still have something to refuse. + fn unvalidated_two_node_membership(me: &str, other: &str) -> Membership { + Membership::new(NodeId::from(me), [NodeId::from(other)], 2) + } + /// A node that terminates mTLS, dials its peers with an identity of its own, and /// refuses any peer link carrying none — the whole declared posture. /// @@ -1411,12 +1419,23 @@ mod live { // No peer can dial it and no certificate could ever be bound to it, so it is // the address that is wrong — refused however the deployment is configured, // with the message for the problem it actually is rather than as a certificate - // naming the wrong host. + // naming the wrong host. The refusal is now at the membership config itself + // (C25): an id with no canonical form names a member the cluster could never + // verify and this node could never be recognised as, so it never reaches a + // listener at all. + let e = Membership::from_static_config(Some("::1:9000"), None, "10.0.0.2:9000", 2) + .expect_err("an unbracketed IPv6 literal names no host"); + assert!( + e.to_string().contains("is not an address a peer can dial"), + "{e}" + ); + // And a member *learned* at such an address is still refused where it is read, + // which is what keeps it a permanent dial failure rather than a forever-retry. let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let e = startup_error( listener, ServeConfig { - membership: Some(two_node_membership("::1:9000", "10.0.0.2:9000")), + membership: Some(unvalidated_two_node_membership("10.0.0.1:9000", "::1:9000")), cluster_secret: Some(SECRET.to_vec()), ..ServeConfig::default() }, @@ -1439,7 +1458,10 @@ mod live { let e = startup_error( listener, ServeConfig { - membership: Some(two_node_membership("wss://10.0.0.1:9000", "wss://::1:9000")), + membership: Some(unvalidated_two_node_membership( + "wss://10.0.0.1:9000", + "wss://::1:9000", + )), cluster_secret: Some(SECRET.to_vec()), require_peer_identity: true, client_cert_verification: true, diff --git a/crates/server/tests/reaping.rs b/crates/server/tests/reaping.rs index ca1cf22b..e54a6ee2 100644 --- a/crates/server/tests/reaping.rs +++ b/crates/server/tests/reaping.rs @@ -284,7 +284,9 @@ fn a_returned_member_with_a_higher_incarnation_is_resurrected() { !m.is_adopted(&d), "a returned member is pending until verified" ); - for voucher in [A, B] { + // Two members other than this node: a claim carried on a link bound to this node's + // own id is not this node vouching, so `A` (self here) is not one of them. + for voucher in [B, C] { m.merge_liveness( &nid(voucher), [( From 2c58b6db65ca6c09b6069e8bf1a9dc3a4d0ecf7b Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 22:03:40 -0300 Subject: [PATCH 08/17] test(server): the unparseable-member startup refusal needs a membership that skipped the config door C25 refuses an address with no canonical form where it is written, so a config-built membership can no longer carry one as far as the serve path. The startup refusal is still the right gate for a membership assembled programmatically, so the test asserts both: the config door refuses it first, and the dialer's validation refuses it for a view that did not pass through that door. --- crates/server/tests/peer_tls.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/server/tests/peer_tls.rs b/crates/server/tests/peer_tls.rs index 0b5f61db..9e6e571a 100644 --- a/crates/server/tests/peer_tls.rs +++ b/crates/server/tests/peer_tls.rs @@ -178,6 +178,14 @@ fn two_node_membership(me: &str, other: &str) -> Membership { Membership::from_static_config(Some(me), None, other, 2).unwrap() } +/// The same view built **without** the config validation. C25 refuses an address with +/// no canonical form where it is written, so a startup refusal about a member that +/// reaches the *serve* path with an address no dial can honor needs a membership that +/// did not come through that door to still have something to refuse. +fn unvalidated_two_node_membership(me: &str, other: &str) -> Membership { + Membership::new(NodeId::from(me), [NodeId::from(other)], 2) +} + fn clustered(me: &str, other: &str) -> ServeConfig { ServeConfig { membership: Some(two_node_membership(me, other)), @@ -953,12 +961,28 @@ async fn peer_tls_without_a_cluster_refuses_to_start() { async fn an_unparseable_member_address_refuses_to_start() { // A scheme the dial cannot honor is a typo, not a hostname: folding it into one // would produce `ws://http://host/` and a dial that fails forever. + // Refused at the membership config first: an address with no canonical form names + // a member the cluster could never verify (C25). + let e = Membership::from_static_config(Some("10.0.0.1:9000"), None, "http://10.0.0.9:9000", 2) + .expect_err("an unknown scheme is not an address"); + assert!( + e.to_string().contains("is not an address a peer can dial"), + "{e}" + ); + + // And still refused where the dialer reads it, for a member that reached the serve + // path without passing that door. let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap().to_string(); let err = startup_error( listener, ServeConfig { - ..clustered(&addr, "http://10.0.0.9:9000") + membership: Some(unvalidated_two_node_membership( + &addr, + "http://10.0.0.9:9000", + )), + cluster_secret: Some(SECRET.to_vec()), + ..ServeConfig::default() }, ) .await; From ca75112e86b9df5e0065100505b90d3358cce280 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 22:23:06 -0300 Subject: [PATCH 09/17] fix(server): a reduction that is not a fixpoint is not a reduction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 5, after pass 4 had approved the address story. `normalized` stripped one trailing dot, so `a.example..` reduced to `a.example.` — a *different* node id, which DNS resolves to the same host and which rustls accepts against the same certificate. A compromised member had only to answer one gossip round with that spelling: the victim learned a second id for an *honest* member, dialed it, verified it truthfully, and adopted it on the attacker's single vouch plus its own. Measured on a five-member view: 47 of 90 sample rooms took the ghost into their replica set and 13 believed it their leader, none of which the honest node ever speaks as — so those rooms' majority-ack never releases. Not the own-host residual: the minted id is on a host the attacker does not own, and it works under the strongest posture. Every trailing dot now folds, a host with an empty label is refused, and a bracketed host must be followed by its port and nothing else — a *wrong* canonical form is worse than none, because it is accepted. A certificate name carrying a run of root labels binds nothing, refused as malformed rather than folded, which keeps C13's narrow binding and the fixpoint at once. An underscore is allowed: container runtimes hand those names out and they carry no second reading. Two rules the sweep proved untested are now pinned — that a node advertises only the verifications it made itself, and that the peer-list door refuses an undialable address — plus the fixpoint itself and the ghost-id shape end to end. The wildcard SAN is recorded on C27: the inbound binding refuses one, the outbound dial is plain TLS and honours it, so it is an unbounded trust-unit generator. --- ARCHITECTURE.md | 4 +- DECISIONS.md | 2 + KANBAN.md | 2 +- crates/server/src/dial.rs | 101 ++++++++++++++++++++++++++++++-- crates/server/tests/adoption.rs | 80 +++++++++++++++++++++++++ 5 files changed, 181 insertions(+), 8 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3ba7baf7..8dc9c68c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -986,7 +986,9 @@ The secret is a **bearer** credential, so the peer link carries it over an encry **Adoption is only as strong as the identity beneath it, and it does not bound a host.** Under peer mTLS a verification means a certificate for that member's host answered there, so no member can manufacture another's verification. **With no certificates configured none of that holds**: a verification means only that *something* answers at the address the id names, and a secret-holder can bind a link to any member id it likes, so it can raise the whole bar itself. `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` is what makes the bar a bar. Even then one limit remains, deliberately: **placement keys on node ids and a certificate names a host**, so a member that owns a host owns every id under it — it answers at each ground id, and the honest nodes that dial it verify one truthfully. Adoption bounds the mint to the member's own host and to ids the cluster can actually reach; it cannot bound it further, because every verification involved is genuine. Closing that needs the ring to weigh trust units rather than ids — a placement change, not an evidence one. -**One further limit follows from the same gap: a certificate that names several hosts is several trust units.** §Peer Identity deliberately lets a node certificate spell its host more than one way — a DNS name beside its IP literal — and each spelling is a different host, so a machine holding such a certificate can hold an adopted member under each and vouch as two units. Until the ring weighs trust units, a deployment that wants the bar to mean two *machines* issues each node a certificate naming one host. +**A reduction that is not a fixpoint is not a reduction.** Every spelling must reduce to the same id in one step, or a peer sends the un-reduced form, a door reduces it once, and the roster holds a *second* id for an endpoint it already has — one that resolves to the same host and that TLS accepts against the same certificate, so every honest node verifies it truthfully. So all trailing dots fold, not one; a host with an empty label is refused; and a bracketed host must be followed by its port and nothing else, since a *wrong* canonical form is worse than none because it is accepted. A certificate name carrying a run of root labels binds nothing — refused as malformed rather than folded, which keeps the binding narrow without giving up the fixpoint. + +**One further limit follows from the same gap: a certificate that names several hosts is several trust units.** §Peer Identity deliberately lets a node certificate spell its host more than one way — a DNS name beside its IP literal — and each spelling is a different host, so a machine holding such a certificate can hold an adopted member under each and vouch as two units. Until the ring weighs trust units, a deployment that wants the bar to mean two *machines* issues each node a certificate naming one host — and a **wildcard** SAN is the unbounded case of the same thing: the inbound binding refuses a wildcard deliberately, but the outbound dial is plain TLS, which honours one, so a member holding `*.x.example` answers truthfully at every name under it and each reads as a different trust unit. Issue per-host certificates. ## Peer Transport diff --git a/DECISIONS.md b/DECISIONS.md index bb968e33..476c5254 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -37,6 +37,8 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Pass 4 found three more, all in how an id is derived from *configuration* rather than in the evidence machinery.** The bootstrap exception read `configured.len() <= 1` — a set, after canonicalization and de-duplication — so a peer list whose every entry spelled this node's own address collapsed to one member and dropped the bar to a single vouch, which is exactly the state the empty-list refusal had just been added to catch, reached without the variable being empty; it now reads the *list* for a peer other than self, and such a list is refused at startup. `CRDTSYNC_NODE_ID` was the one door that did not canonicalize, a regression from the plain `trim()` it replaced: a node written `wss://Node-A.Example:9000` kept that spelling as its own id while its peers derived the canonical one, so it carried a **doppelgänger of itself** in its roster — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, and dropping every follower-head report because the link was bound to the other spelling. And the canonical form canonicalized the host but not the **port**, so `:9000`, `:09000` and `:009000` were three node ids on one listener — one certificate, one trust unit, unbounded ring positions, which is the targeted write-stall pass 3 was supposed to have closed, and which an operator reaches with no attacker at all by writing a leading zero in one node's peer list. The port is now read as a number, and an address that has no canonical form is refused where it is written. +**Pass 5 found the reduction was not a fixpoint.** `normalized` stripped one trailing dot, so `a.example..` reduced to `a.example.` — a *different* node id, which DNS resolves to the same host and which rustls accepts against the same certificate. A compromised member had only to answer one gossip round with that spelling: the victim learned a second id for an *honest* member, dialed it, verified it truthfully, and adopted it on the attacker's single vouch plus its own — 47 of 90 sample rooms took the ghost into their replica set and 13 believed it their leader, none of which the honest node ever speaks as. That is not the own-host residual: the minted id is on a host the attacker does not own, and it works under the strongest posture. All trailing dots now fold, a host with an empty label is refused, and a certificate name carrying a run of root labels binds nothing — refused as malformed rather than folded, which keeps C13's narrow binding *and* the fixpoint. The same pass found a bracketed host with trailing junk being given a *wrong* canonical form (the port silently dropped) rather than refused, which is worse than none because it is accepted. + **The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. The same gap has one more edge, stated rather than hidden: §Peer Identity deliberately lets a certificate spell a host several ways, and each spelling is a different host, so a machine holding a multi-host certificate holds one trust unit per name it carries. Until C27, a deployment that wants the bar to mean two *machines* issues one host per certificate. **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. diff --git a/KANBAN.md b/KANBAN.md index f4d3b532..aaa6512b 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -427,7 +427,7 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C27 — a member that owns a host owns every ring position under it, because placement keys on node ids while a certificate names a host (crates/server) — READY, no dependencies. Found by cold review during C25 (#369) and pinned there as a passing test (`a_certified_member_still_mints_ids_on_its_own_host`).** C25 made rooms place on *adopted* members and made adoption the cluster's decision, which closes minting on anyone else's host, minting an id nobody can reach, and any single member manufacturing the evidence. What it cannot close is the mint on a member's **own** host: HRW scores a node id, C13 binds a link to the *host* a certificate names, and a host answers at as many ports as it likes — so a compromised member grinds `own-host:` until HRW places it on the room it wants, answers there, and every honest node that dials it verifies it **truthfully**. No evidence rule can help, because no one is lying. The consequence is the one C25 removed everywhere else: a member chooses which rooms it replicates, and from inside the replica set it reaches the leadership and durability gates. **The fix is a placement change, not an evidence one: weigh the ring by trust unit.** Score hosts rather than node ids, so a room's replica set is `n` distinct hosts and which rooms a host holds follows from a name it cannot grind; the member serving a host's position is then a local choice among that host's adopted members, and it does not matter which, since they are one trust unit already. That is also the better replication property — `n` replicas on `n` failure domains instead of `n` ids that may share a machine — and it makes the `replicas_for(room).contains(sender)` gate ask about exactly what the certificate vouches for. Costs to settle: two members on one host stop both replicating a room (a capacity change, not a correctness one), the replication factor becomes a host count, and every gate/redirect/quorum that names a member has to resolve host → member. Note this is *not* reachable without a compromised member's certificate: without peer mTLS the whole binding vouches for nothing anyway (C13's residual), and with it the mint is bounded to a host the operator already trusts to run a node. → *Cluster*. -**C29 — a cluster of two adopted trust units can never re-admit a member it reaped, and a cluster whose members share a host can never admit one (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Adoption asks for `ADOPTION_VERIFIERS = 2` trust units other than the candidate's own, and two shapes cannot raise them. **Re-admission:** reaping drops a member from `configured`, so a returning node is a fresh join; in a two-node cluster the only other unit is the surviving node itself, so it verifies the returner a thousand times and the count never passes one — the survivor leads every room, the returner places over both, and the split is permanent with no operator signal and no repair short of restarting the survivor. Reachable by an ordinary long partition with no attacker, and reachable deliberately with C28's unrefutable `Dead`. **Admission:** a cluster whose members all share one host has exactly one unit, so no joiner ever clears the bar — a real shape (several instances per machine or pod), failing closed and silently. Both want the same thing: a bar that is expressible in the cluster it runs in, without becoming node-local (which is what made it non-convergent before). Candidates: a cluster-wide `configured`-derived denominator, an operator-set bar, or — cleanest — C27's placement over trust units, which makes "how many units exist" a thing the ring already knows. At minimum a node should *say* when its roster holds a member the bar cannot admit, instead of quietly not placing it. → *Cluster*. +**C29 — a cluster of two adopted trust units can never re-admit a member it reaped, and a cluster whose members share a host can never admit one (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Adoption asks for `ADOPTION_VERIFIERS = 2` trust units other than the candidate's own, and two shapes cannot raise them. **Re-admission:** reaping drops a member from `configured`, so a returning node is a fresh join; in a two-node cluster the only other unit is the surviving node itself, so it verifies the returner a thousand times and the count never passes one — the survivor leads every room, the returner places over both, and the split is permanent with no operator signal and no repair short of restarting the survivor. Reachable by an ordinary long partition with no attacker, and reachable deliberately with C28's unrefutable `Dead`. **Admission:** a cluster whose members all share one host has exactly one unit, so no joiner ever clears the bar — a real shape (several instances per machine or pod), failing closed and silently. Both want the same thing: a bar that is expressible in the cluster it runs in, without becoming node-local (which is what made it non-convergent before). Candidates: a cluster-wide `configured`-derived denominator, an operator-set bar, or — cleanest — C27's placement over trust units, which makes "how many units exist" a thing the ring already knows. At minimum a node should *say* when its roster holds a member the bar cannot admit, instead of quietly not placing it. Also unbounded, and the same root: a **wildcard** SAN. The inbound binding refuses a wildcard deliberately (`cert_names_member` compares whole names), but the only thing that mints verification evidence is this node's *outbound* dial, which is plain TLS and honours one — so a member holding `*.x.example` answers truthfully at every name under it, each of which reads as a distinct trust unit, and each of which is then adoptable. Whatever C27 does about trust units has to close the outbound side too, or say that per-host certificates are a deployment requirement. → *Cluster*. **C28 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. → *Cluster*. diff --git a/crates/server/src/dial.rs b/crates/server/src/dial.rs index 01198845..44716251 100644 --- a/crates/server/src/dial.rs +++ b/crates/server/src/dial.rs @@ -105,12 +105,31 @@ impl PeerEndpoint { // being *retried forever* — a control character or a space builds a URL the // dialer rejects at send time, which reads as "unreachable" and is redialed on // the fast cadence rather than classified a permanent address error. - if !authority - .bytes() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b':' | b'[' | b']')) - { + if !authority.bytes().all(|b| { + b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b':' | b'[' | b']' | b'_') + }) { return Err(BadPeerAddress::NotAnAuthority); } + // A bracketed host must be followed by its port and nothing else. Trailing junk + // after `]` is read as part of the authority by the dialer and dropped by the + // canonical form, so the two would disagree about which endpoint the string + // names — and a *wrong* canonical form is worse than none, because it is + // accepted. + if let Some(after) = authority.strip_prefix('[') { + match after.split_once(']') { + Some((_, rest)) if rest.is_empty() || rest.starts_with(':') => {} + _ => return Err(BadPeerAddress::NotAnAuthority), + } + } + // A host with an empty label — `a..example`, `.example`, a bare `..` — is no + // host any resolver accepts, and it is the shape that survives a one-dot + // reduction as a second spelling of a real one. + if let Some(host) = host_of(authority) { + let host = host.trim_end_matches('.'); + if !host.starts_with('[') && (host.is_empty() || host.split('.').any(str::is_empty)) { + return Err(BadPeerAddress::NotAnAuthority); + } + } // An authority that names no host — an IPv6 literal left unbracketed, or one // opening with its port separator — is dialable by nobody and bindable to no // certificate. Refused here so every consumer inherits it: a configured member @@ -246,6 +265,15 @@ pub fn cert_names_member(name: &[u8], addr: &[u8]) -> bool { let Ok(name) = std::str::from_utf8(name) else { return false; }; + // A name carries at most one root label. A run of them, or any other empty label, + // is not a name a resolver accepts, so it binds nothing — checked before the fold + // rather than by folding one dot, because folding one is not a fixpoint and a + // reduction that is not a fixpoint hands one endpoint a second identity. + let trimmed = name.trim(); + let without_root = trimmed.strip_suffix('.').unwrap_or(trimmed); + if without_root.split('.').any(str::is_empty) { + return false; + } let name = normalized(name); if name.is_empty() { return false; @@ -267,9 +295,14 @@ pub fn cert_names_member(name: &[u8], addr: &[u8]) -> bool { /// trimmed, lowercased as DNS names are, and without its root label — `node-a.` and /// `node-a` name one host, and a certificate and an advertise address need not agree on /// which spelling to use. +/// +/// **Every** trailing dot goes, not one. Stripping a single dot is not a fixpoint, and a +/// reduction that is not a fixpoint is not a reduction: `a.example..` would reduce to +/// `a.example.`, a *different* node id that DNS resolves to the same host and that TLS +/// accepts against the same certificate — so one honest endpoint would hold a second +/// ring position that it never speaks as. fn normalized(name: &str) -> String { - let name = name.trim(); - name.strip_suffix('.').unwrap_or(name).to_ascii_lowercase() + name.trim().trim_end_matches('.').to_ascii_lowercase() } /// Whether `name` is an IP address rather than a host name, judged after the same @@ -550,6 +583,62 @@ mod tests { } } + #[test] + fn the_canonical_form_is_a_fixpoint() { + // A reduction that is not a fixpoint is not a reduction: if `canonical(x)` can + // still be reduced, then a peer sends the un-reduced spelling, one door reduces + // it once, and the roster holds a *second* id for an endpoint it already has — + // one that resolves to the same host and that TLS accepts against the same + // certificate, so every honest node verifies it truthfully. + for addr in [ + "a.example:9000", + "a.example.:9000", + "a.example..:9000", + "a.example...:9000", + "wss://A.Example..:09000", + "10.0.0.1..:9000", + "[2001:db8::6]:9000", + "node_a.internal:9000", + "node-a", + ] { + let once = canonical_member_addr(addr); + if let Some(once) = &once { + assert_eq!( + canonical_member_addr(once).as_ref(), + Some(once), + "{addr} -> {once} is not a fixpoint", + ); + } + } + // And every spelling of one host reduces to the *same* id, not to a chain of + // distinct ones. + let canonical = canonical_member_addr("a.example:9000"); + for addr in ["a.example.:9000", "a.example..:9000", "A.EXAMPLE...:9000"] { + assert_eq!(canonical_member_addr(addr), canonical, "{addr}"); + } + } + + #[test] + fn an_authority_with_no_single_reading_is_refused() { + // Each of these is read one way by the dialer and another by the canonical + // form, or is no host at all. A *wrong* canonical form is worse than none, + // because it is accepted. + for addr in [ + "[::1]junk:9000", + "[2001:db8::1]xy:8080", + "a..example:9000", + ".example:9000", + "..:9000", + "[::1", + ] { + assert!(PeerEndpoint::parse(addr).is_err(), "{addr}"); + assert_eq!(canonical_member_addr(addr), None, "{addr}"); + } + // An underscore is legal in the names container runtimes hand out, and carries + // no second reading, so it is not swept up by the refusal. + assert!(canonical_member_addr("node_a.internal:9000").is_some()); + } + #[test] fn one_endpoint_has_one_canonical_address() { // Two spellings of one endpoint would be two node ids, so two positions in the diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index b99679da..ca380b20 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -1457,3 +1457,83 @@ fn a_link_bound_to_this_nodes_own_id_makes_it_vouch_for_nobody() { ); assert!(!m.is_adopted(&joiner)); } + +#[test] +fn a_node_advertises_only_the_verifications_it_made_itself() { + // The flag is *first-hand*. Relaying what this node merely heard would make one + // member's word enough to place any id it liked: a compromised member's claim + // would be re-asserted by every honest node that received it, and the bar would be + // met by one attacker and one echo. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone()); + vouch(&mut m, "10.0.0.1:9000", &joiner); + assert!( + m.has_verified(&NodeId::from("10.0.0.1:9000"), &joiner), + "the claim was recorded", + ); + assert!( + !verified_by(&m, &joiner), + "and this node advertises nothing it did not do itself", + ); + + // Its own link is what puts the flag on the wire. + m.note_verified(&joiner); + assert!(verified_by(&m, &joiner)); +} + +#[test] +fn a_peer_entry_no_peer_could_dial_is_refused_where_it_is_written() { + // The peer-list door, not just the node-id door. A configured member is adopted + // from birth, so an undialable id seeded here would be placed on rooms it can never + // answer for — the write-stall, reached with no attacker at all. + for peers in [ + "10.0.0.1:9000/sync", + "10.0.0.1:9000,::1:9000", + "wss://a.example:1@b.example:9000", + "10.0.0.1:99999", + "a..example:9000", + ] { + let e = Membership::from_static_config(None, Some(SELF_ADDR), peers, N) + .expect_err("an undialable peer is refused"); + assert!( + e.to_string().contains("is not an address a peer can dial"), + "{peers}" + ); + } +} + +#[test] +fn a_second_spelling_of_an_honest_member_is_not_a_second_member() { + // The shape a compromised member reaches for once it cannot mint on another host: + // a spelling of an *honest* member's address that reduces to a different id. It + // would resolve to the honest host, answer with the honest certificate, and be + // verified truthfully by everyone — a ring position the honest node never speaks + // as, and a room whose quorum can never be met. Every such spelling either reduces + // to the same id or is no address at all. + let honest = NodeId::from("wss://a.example:9000"); + let mut m = Membership::from_static_config( + None, + Some("wss://victim.example:9000"), + "wss://a.example:9000,wss://b.example:9000", + N, + ) + .unwrap(); + let before = m.members().len(); + for spelling in [ + "wss://a.example.:9000", + "wss://a.example..:9000", + "wss://a.example...:9000", + "wss://A.Example.:09000", + ] { + m.add_member(NodeId::from(spelling)); + assert!( + matches!( + crdtsync_server::dial::canonical_member_addr(spelling), + None | Some(_) + ) && NodeId::canonical(spelling.as_bytes()).is_none_or(|id| id == honest), + "{spelling} is either no address or the honest member", + ); + } + assert_eq!(m.members().len(), before, "no ghost joined the roster"); +} From 664e396c24d33b05cff77a0b31e5af463f1ed4f6 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 23:37:22 -0300 Subject: [PATCH 10/17] fix(server): brackets mean an IPv6 literal, and a claim's sender clears the same bar as its subject MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 6. `PeerEndpoint::parse` required a bracketed host to be followed by a port and nothing else, but never that the brackets *contain* an IPv6 literal. `host_of` then returned everything between them with its colons intact and the canonical form emitted that verbatim with the port appended, so `[10.0.0.1:9000]:9443` became the node id `10.0.0.1:9000:9443`. That id is not refused, it is accepted: it parses, so startup validation and the peer-identity member loop both pass, and written in configuration it is adopted from birth and placed on every room HRW gives it — while the URL the dialer builds from it is rejected as an invalid authority at send time, which reads as unreachable rather than as a permanent address error and is redialed forever. Those rooms reach no quorum. Written as `CRDTSYNC_NODE_ID` it has no canonical form at all, so peers drop it from gossip and refuse its `PeerAuth`, and the node joins nothing while believing it has. An operator typo reaches both. The root cause is that pass 5's fixpoint property was asserted over a list of spellings rather than established by construction, and the bracketed family was not on the list. The list now carries it, and brackets carry an IPv6 literal. Separately, `merge_liveness` gated the node a claim was *about* but stored it under a `sender` it never gated — the hazard `note_verified`'s own contract states, applied to only one side. Peer admission takes the id a link claims, so one certified host opens a link per port and banks a permanent verifier entry per link against every member, keyed on ids no reap ever strikes because reaping strikes members. They can never count, only grow. Both sides of a claim now clear one predicate. Also: `gossip::exchange` let an inbound push introduce a third member, which production's `apply_gossip` refuses, so the convergence tests were measuring a push the cluster rejects. Tests: the `can_be_verified` gate had no test (the one mutation that survived pass 6's sweep) and now has one; a tautological `matches!(_, None | Some(_))` conjunct is replaced by the bracketed spellings it should have covered. C30 (the canonical id is finer-grained than what the resolver dials) and C31 (a member adopted after startup has no replication link) filed. --- DECISIONS.md | 2 + KANBAN.md | 6 +- crates/server/src/dial.rs | 63 ++++++++++++++++- crates/server/src/gossip.rs | 14 +++- crates/server/src/membership.rs | 24 +++++-- crates/server/tests/adoption.rs | 119 ++++++++++++++++++++++++++++++-- 6 files changed, 214 insertions(+), 14 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index 476c5254..aa4cd996 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -39,6 +39,8 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Pass 5 found the reduction was not a fixpoint.** `normalized` stripped one trailing dot, so `a.example..` reduced to `a.example.` — a *different* node id, which DNS resolves to the same host and which rustls accepts against the same certificate. A compromised member had only to answer one gossip round with that spelling: the victim learned a second id for an *honest* member, dialed it, verified it truthfully, and adopted it on the attacker's single vouch plus its own — 47 of 90 sample rooms took the ghost into their replica set and 13 believed it their leader, none of which the honest node ever speaks as. That is not the own-host residual: the minted id is on a host the attacker does not own, and it works under the strongest posture. All trailing dots now fold, a host with an empty label is refused, and a certificate name carrying a run of root labels binds nothing — refused as malformed rather than folded, which keeps C13's narrow binding *and* the fixpoint. The same pass found a bracketed host with trailing junk being given a *wrong* canonical form (the port silently dropped) rather than refused, which is worse than none because it is accepted. +**Pass 6 found that a fixpoint checked over a list is not a fixpoint, and that a claim's *sender* was ungated.** Pass 5's fix came with a test asserting the canonical form is idempotent over a list of spellings, which is a check of the list rather than of the reduction — and the family it omitted was the bracketed one. `PeerEndpoint::parse` enforced that a bracketed host is followed by a port and nothing else, but never that the brackets *contain* an IPv6 literal, so `host_of` returned everything between them with its colons intact and the canonical form emitted that verbatim with the port appended: `[10.0.0.1:9000]:9443` became the node id `10.0.0.1:9000:9443`. That id is not refused, it is *accepted* — it parses, so startup validation and the peer-identity member loop both pass — and being written in configuration it is adopted from birth and placed on every room HRW gives it, while the URL the dialer builds from it (`ws://10.0.0.1:9000:9443/`) is rejected as an invalid authority at send time, which classifies as unreachable rather than as a permanent address error and so is redialed forever. The rooms it holds replicas of can reach no quorum. Written as `CRDTSYNC_NODE_ID` it is the doppelgänger again and worse: the id has no canonical form at all, so every peer drops it from gossip and refuses its `PeerAuth`, and the node runs believing it has joined a cluster that cannot see it. No attacker is needed for either — bracketing a `host:port` pair is an ordinary operator typo. Brackets now mean an IPv6 literal and nothing else. Separately, `merge_liveness` gated the *node* a claim was about by `can_be_verified` but stored the claim under a `sender` it never gated, which is the exact hazard `note_verified`'s own contract states and applies only to the node: peer admission takes the id a link claims, so one certified host opens a link per port, claims a distinct id on each, and banks a permanent verifier entry per link against every member. Those entries are keyed on ids that are on no roster, so no reap ever strikes them; they can never *count* (only adopted verifiers do) but they grow without bound. Both sides of a claim now clear one predicate. + **The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. The same gap has one more edge, stated rather than hidden: §Peer Identity deliberately lets a certificate spell a host several ways, and each spelling is a different host, so a machine holding a multi-host certificate holds one trust unit per name it carries. Until C27, a deployment that wants the bar to mean two *machines* issues one host per certificate. **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. diff --git a/KANBAN.md b/KANBAN.md index aaa6512b..af6a1b97 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,7 +30,7 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ -**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Pass 4 found three more, all in how an id is derived from configuration:** the bootstrap exception read a *de-duplicated set*, so a peer list of only this node's own address collapsed to one member and dropped the bar to a single vouch — the very state the empty-list refusal had been added to catch; `CRDTSYNC_NODE_ID` was the one door that did not canonicalize (a regression from the plain `trim()` it replaced), so a node written in a non-canonical spelling carried a **doppelgänger of itself** — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, dropping every follower-head report; and the port was not canonicalized, so `:9000`/`:09000`/`:009000` were three ring positions on one listener, which is the targeted write-stall pass 3 was supposed to have closed and which an operator reaches with no attacker by writing a leading zero. The bar now reads the list rather than the set, every door canonicalizes, the port is read as a number, and an address with no canonical form is refused where it is written. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Pass 4 found three more, all in how an id is derived from configuration:** the bootstrap exception read a *de-duplicated set*, so a peer list of only this node's own address collapsed to one member and dropped the bar to a single vouch — the very state the empty-list refusal had been added to catch; `CRDTSYNC_NODE_ID` was the one door that did not canonicalize (a regression from the plain `trim()` it replaced), so a node written in a non-canonical spelling carried a **doppelgänger of itself** — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, dropping every follower-head report; and the port was not canonicalized, so `:9000`/`:09000`/`:009000` were three ring positions on one listener, which is the targeted write-stall pass 3 was supposed to have closed and which an operator reaches with no attacker by writing a leading zero. The bar now reads the list rather than the set, every door canonicalizes, the port is read as a number, and an address with no canonical form is refused where it is written. **Passes 5 and 6 found two more, both places where a rule was stated but not made structural:** the canonical form was checked to be a fixpoint over a list of spellings rather than by construction, and brackets — the IPv6 literal's syntax — were never required to *contain* an IPv6 literal, so `[10.0.0.1:9000]:9443` (an operator typo, no attacker) parsed as a host whose own text carries colons and yielded the id `10.0.0.1:9000:9443`: accepted at the config door, adopted from birth as a configured member, placed on every room HRW gave it, and dialable by nobody, so those rooms could reach no quorum — and written as `CRDTSYNC_NODE_ID` it has no canonical form at all, so peers drop it from gossip and refuse its `PeerAuth` and the node joins nothing while believing it has. And a verifier claim gated the *node* it was about but never the **sender** it was stored under, so a certified host opening a link per port banked an unbounded verifier entry per link against every member — keyed on ids no reap ever strikes, because reaping strikes members. Both bars are now the same one predicate on both sides. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. @@ -431,6 +431,10 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C28 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. → *Cluster*. +**C30 — the canonical node id is finer-grained than the address the resolver dials, so one endpoint still holds many ring positions (crates/server) — READY, no dependencies. Found by cold review during C25 (#369), measured.** C25 gave an advertise address one canonical spelling so that one endpoint holds one ring position, and enforced it at every door. The reduction decides "is this an IP literal" with Rust's strict `IpAddr::from_str`, while the thing that actually resolves the address is `getaddrinfo`, which accepts a much larger language: `0xa.0.0.1`, `10.0.1`, `10.1`, `167772161` and `10.0.0.01` all connect to `10.0.0.1`, and each is a *distinct* canonical node id **and a distinct trust unit** (`member_trust_unit` reduces the host by the certificate binding's relation, which also reads them as text). `[::ffff:10.0.0.1]` is the same shape across families — a distinct id whose dual-stack connect reaches the IPv4 listener. So the pass-3 write-stall returns through the resolver instead of through the spelling: one vouch from the attacker, two honest nodes dial the alias, are answered by the honest member, and verify it **truthfully**, and the ghost is adopted onto rooms nobody ever speaks as. Under peer mTLS the dial never opens (`ServerName::try_from("0xa.0.0.1")` is an `InvalidDnsNameError`, and a 16-octet V6 name matches no 4-octet `iPAddress` SAN), so this bites in exactly the postures C25 concedes — but the mechanism is the *resolver*, not the binding, and C25's concession is phrased about the binding. Note C27 does not close it either: these are distinct hosts, so weighing the ring by trust unit leaves them distinct units. Wants the canonical form to reject or fold every host form the resolver would, rather than every one Rust's parser recognizes. → *Cluster*. + +**C31 — a member adopted after startup is placed on rooms this node cannot replicate to (crates/server) — READY, no dependencies. Found by cold review during C25 (#369).** `spawn_peers` builds the outbound `peer_conns` map **once**, from the roster as it stood at startup, and `dispatch_replication` silently drops a frame for a follower with no entry. Pre-existing — nothing in C25 changed either — but C25 is what makes the path ordinary: its stated success case is a genuine joiner clearing the adoption bar "within a few rounds", after which every node places rooms on it. Those rooms now have a replica this node never sends to and never hears from, so the joiner degrades exactly the quorum adoption exists to protect, and the drop is silent. Wants the peer connection set to follow the adopted set rather than the startup roster, with a signal when a placed follower has no link. → *Cluster*. + **C26 — the SDK apply seams report a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — READY, no dependencies. Split out of C19 (#367) rather than folded in.** `crdtsync_client_receive`/`WasmClient::receive` fold a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learns how many ops applied *now* and nothing else. C19 made the distinction answerable — `Op::is_admissible` is public and pure — but did not plumb it: a buffered op (which a later arrival commits) and an op no replica will ever hold are both reported as "not counted", so an SDK peered with a buggy or hostile writer sees edits vanish with no signal, and the two cases call for opposite responses (wait vs. surface a client bug). `client.rs` needs no change on this axis — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it, so nothing is missed. The server side is closed: `handle_ops` answers `ErrorCode::MalformedOp` and the ingest seams drop such a record, so this is the *offline / P2P / relay* seam, where no server sits between the writer and the fold. Wants a refused count (or a refused-op list) beside the applied one across FFI, wasm, JS, Python and Go — and is worth taking with **C23**, which has the same shape: a real refusal the SDKs render as an edit that silently did nothing. → *SDK / Ops*. **C14 — a redacted op delta leaves the same re-mint hole, and no frame can carry the ids it withholds (crates/server + crates/core) — READY, no dependencies. Found during C9 (#357), reproduced.** C9 closed the *snapshot* seam: a projected snapshot now names the recipient's own ids, so adoption never frees one the room's log holds. An **uncompacted** room serves an op delta instead, and the per-op read filter withholds the recipient's own ops on paths it may no longer read — leaving a hole in its own run with nothing to repair it from. Measured on C9's own fixture: a reader with write at `/` and read on `/a` alone writes into `/b` (seqs 0,1) then `/a` (seq 2); it restarts, its delta withholds `/b`, `next_seq` reports 0, and its next write mints seq 0 straight onto its own withheld id — the write never lands. The state encoding was the carrier for the snapshot case; an `Ops` frame has no equivalent, so closing this needs one: a frontier field on the delta, or a rule that a reader's *own* ops are never withheld from it (which trades the hole for materialising ids in a subtree it may not read — the reason C9 did not simply stop scrubbing). Note the second shape also has to answer what an op carries beyond its id: its target and content may name denied structure. → *Server / Fan-out*. diff --git a/crates/server/src/dial.rs b/crates/server/src/dial.rs index 44716251..36d1bdf0 100644 --- a/crates/server/src/dial.rs +++ b/crates/server/src/dial.rs @@ -115,9 +115,25 @@ impl PeerEndpoint { // canonical form, so the two would disagree about which endpoint the string // names — and a *wrong* canonical form is worse than none, because it is // accepted. + // Brackets mean one thing: an IPv6 literal. A bracketed anything-else is read + // here as a host whose own text contains colons, and the canonical form then + // emits that host verbatim with the port appended — `[10.0.0.1:9000]:9443` + // becomes the id `10.0.0.1:9000:9443`, which names no endpoint any dialer + // accepts. That id is worse than a refusal: it is *accepted*, so it is + // configured and adopted from birth, placed on every room HRW gives it, and + // never acks — the rooms it holds a replica of can reach no quorum. Written as + // this node's own id it is worse still, because it has no canonical form at + // all, so peers drop it from gossip and refuse its `PeerAuth`, and the node + // joins nothing while believing it has. if let Some(after) = authority.strip_prefix('[') { match after.split_once(']') { - Some((_, rest)) if rest.is_empty() || rest.starts_with(':') => {} + // Empty brackets name no host at all, which the check below classifies + // more precisely. + Some((host, rest)) if rest.is_empty() || rest.starts_with(':') => { + if !host.is_empty() && host.parse::().is_err() { + return Err(BadPeerAddress::NotAnAuthority); + } + } _ => return Err(BadPeerAddress::NotAnAuthority), } } @@ -568,6 +584,40 @@ mod tests { } } + #[test] + fn a_bracketed_host_that_is_no_ipv6_literal_is_refused() { + // Brackets are the IPv6 literal's syntax and nothing else. Bracketing a + // `host:port` pair — the operator typo `[10.0.0.1:9000]:9443` — otherwise reads + // as a host whose own text carries colons, and every reader downstream agrees + // on an id (`10.0.0.1:9000:9443`) that no dialer will build a URL for. It would + // be accepted at the config door, adopted from birth, and placed on rooms that + // could then reach no quorum, because the member holding their replicas is one + // nobody can reach. + for addr in [ + "[a.example:9000]:9443", + "[10.0.0.1:9000]:9443", + "[node-a.example:9000]:9443", + "wss://[a.example:9000]:9443", + "[a.example:]", + "[a:]", + "[[a]:9000", + "[a.example]", + "[10.0.0.1]:9000", + ] { + assert_eq!( + PeerEndpoint::parse(addr), + Err(BadPeerAddress::NotAnAuthority), + "{addr}" + ); + assert_eq!(canonical_member_addr(addr), None, "{addr}"); + } + // The literal itself still resolves, in either spelling of its own text. + for addr in ["[::1]:9000", "[2001:db8::6]:9000", "[2001:0db8::0:6]:9000"] { + assert!(PeerEndpoint::parse(addr).is_ok(), "{addr}"); + assert!(canonical_member_addr(addr).is_some(), "{addr}"); + } + } + #[test] fn a_dialed_url_names_the_same_host_the_binding_reads() { // The one invariant the refusal above exists for: whatever a reader takes out @@ -600,6 +650,17 @@ mod tests { "[2001:db8::6]:9000", "node_a.internal:9000", "node-a", + // Brackets around something that is not an IPv6 literal: the shape that + // used to hand back a host with colons still in it. + "[a.example:9000]:9443", + "[10.0.0.1:9000]:9443", + "[node-a.example:9000]:9443", + "wss://[a.example:9000]:9443", + "[a.example:]", + "[a:]", + "[[a]:9000", + "[a.example]", + "[10.0.0.1]:9000", ] { let once = canonical_member_addr(addr); if let Some(once) = &once { diff --git a/crates/server/src/gossip.rs b/crates/server/src/gossip.rs index 4d4331b4..7bc735f2 100644 --- a/crates/server/src/gossip.rs +++ b/crates/server/src/gossip.rs @@ -148,7 +148,19 @@ pub fn merge_into(membership: &mut Membership, sender: &NodeId, payload: Vec = initiator + .known_liveness() + .into_iter() + .filter(|(node, addr, ..)| { + peer.is_member(node) || (node == &initiator_id && addr.as_slice() == node.as_bytes()) + }) + .collect(); + peer.merge_liveness(&initiator_id, pushed); initiator.merge_liveness(&peer_id, peer.known_liveness()); initiator.note_gossip_reachable(&peer_id); initiator.note_verified(&peer_id); diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index ee1e983a..2117c87f 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -928,13 +928,23 @@ impl Membership { // above — a claim about a node that was dropped as malformed, or that is // tombstoned, is a claim about no member of this view — and only where the // member is coherently addressed, the same bar this node's own links clear. - for node in claimed { - if self.can_be_verified(&node) { - rebuilt |= self - .verifiers - .entry(node) - .or_default() - .insert(sender.clone()); + // + // The *sender* clears that same bar, because it is what the entry stores. A + // verifier off the roster can never count — `verifier_units` reads only adopted + // ones — but it would still be retained, and peer admission takes the id a link + // claims: one certified host opens a link per port, claims a distinct id on + // each, and banks an entry per link against every member, keyed on ids no reap + // will ever strike because they are on no roster. Bounding the evidence by the + // roster is what keeps it bounded at all. + if self.can_be_verified(sender) { + for node in claimed { + if self.can_be_verified(&node) { + rebuilt |= self + .verifiers + .entry(node) + .or_default() + .insert(sender.clone()); + } } } if rebuilt { diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index ca380b20..0bad6f97 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -856,6 +856,118 @@ fn a_reaped_member_loses_its_place_and_its_vouches() { assert!(!m.is_adopted(&joiner)); } +#[test] +fn vouches_banked_while_a_member_is_tombstoned_do_not_survive_its_return() { + // Evidence is about a member of *this* view, and a tombstoned node is not one. If + // claims about it were banked while it was gone, a member could be reaped and then + // return pre-adopted — placed on rooms on the strength of vouches collected while + // nobody could reach it, which is the unverified join the unit exists to close. The + // vouches must arrive after it does. + let mut m = membership_for(SELF_ADDR); + let departing = NodeId::from("10.9.9.9:9000"); + m.add_member(departing.clone()); + for _ in 0..crdtsync_server::membership::DEAD_AFTER_FAILURES { + m.note_gossip_unreachable(&departing); + } + for _ in 0..crdtsync_server::membership::REAP_AFTER_DEAD_TICKS { + m.reap_dead(); + } + assert!(!m.is_member(&departing)); + + // Every configured member vouches for it while it is tombstoned — far past the bar. + // The claims ride tuples that do not themselves lift the tombstone, which is the + // whole point: SWIM's own resurrection rule is not being invoked, only the evidence + // is being banked ahead of it. + for i in 0..6 { + let voucher = NodeId::from(format!("10.0.0.{i}:9000").as_str()); + m.merge_liveness( + &voucher, + [( + departing.clone(), + departing.as_bytes().to_vec(), + 0, + MemberState::Suspect, + true, + )], + ); + assert!( + !m.has_verified(&voucher, &departing), + "a claim about a node this view does not hold is about no member", + ); + } + assert!( + !m.is_member(&departing), + "and it did not slip back onto the roster", + ); + assert!(!m.is_adopted(&departing)); + + // It returns under SWIM's own rule, at a higher incarnation. It is a member again — + // and pending, because none of those claims were kept. + m.merge_liveness( + &NodeId::from("10.0.0.1:9000"), + [( + departing.clone(), + departing.as_bytes().to_vec(), + 1, + MemberState::Alive, + false, + )], + ); + assert!(m.is_member(&departing), "it rejoined"); + assert!( + !m.is_adopted(&departing), + "it returns pending: the vouches banked while it was gone are gone with it", + ); +} + +#[test] +fn a_claim_from_a_sender_off_the_roster_is_not_retained() { + // Peer admission takes the id a link claims, so one certified host can present as + // many ids as it has ports. A claim is stored under its *sender*, so a sender that + // is on no roster would bank an entry per link against every member — keyed on ids + // no reap will ever strike, because reaping only strikes members. The evidence is + // bounded by the roster or it is not bounded at all. + let mut m = membership_for(SELF_ADDR); + let joiner = NodeId::from("10.9.9.9:9000"); + m.add_member(joiner.clone()); + let before = m.members().len(); + + for port in 9100..9200 { + let stranger = NodeId::from(format!("evil.example:{port}").as_str()); + m.merge_liveness( + &stranger, + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + assert!( + !m.has_verified(&stranger, &joiner), + "a sender off the roster leaves nothing behind", + ); + } + assert_eq!(m.members().len(), before, "and joins nothing either"); + assert!(!m.is_adopted(&joiner)); + + // A sender that *is* a member is still recorded — the bar is roster membership, + // not distrust of gossip. + let member = NodeId::from("10.0.0.1:9000"); + m.merge_liveness( + &member, + [( + joiner.clone(), + joiner.as_bytes().to_vec(), + 0, + MemberState::Alive, + true, + )], + ); + assert!(m.has_verified(&member, &joiner)); +} + // --- a pending member is still a member --- #[test] @@ -1525,13 +1637,12 @@ fn a_second_spelling_of_an_honest_member_is_not_a_second_member() { "wss://a.example..:9000", "wss://a.example...:9000", "wss://A.Example.:09000", + "wss://[a.example:9000]:9000", + "[a.example:]", ] { m.add_member(NodeId::from(spelling)); assert!( - matches!( - crdtsync_server::dial::canonical_member_addr(spelling), - None | Some(_) - ) && NodeId::canonical(spelling.as_bytes()).is_none_or(|id| id == honest), + NodeId::canonical(spelling.as_bytes()).is_none_or(|id| id == honest), "{spelling} is either no address or the honest member", ); } From db8b8e5e1d5ba3e9c676df10bd387215fd7f9dc4 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Jul 2026 23:39:47 -0300 Subject: [PATCH 11/17] test(server): a newcomer is learned from a round this node initiated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 had A dial B and expected B to learn C from the push. An inbound frame introduces only its own sender, so that is a join the cluster has no path for — it converged here only because the in-process `exchange` helper was more permissive than `apply_gossip`. B dials A and learns C from the reply, which is the direction that actually carries a newcomer. --- crates/server/tests/gossip.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/server/tests/gossip.rs b/crates/server/tests/gossip.rs index 4870e731..6ef62abb 100644 --- a/crates/server/tests/gossip.rs +++ b/crates/server/tests/gossip.rs @@ -189,8 +189,12 @@ fn a_seed_only_node_converges_on_the_full_cluster() { // Round 1: C gossips its seed A. Push-pull syncs both — A learns C, C learns B. exchange(&mut c, &mut a); - // Round 2: A gossips B, so B learns C too. - exchange(&mut a, &mut b); + // Round 2: B gossips A, and learns C from the *reply*. Being dialed would not have + // taught it: an inbound frame introduces only its own sender, so a member is + // learned from a round this node initiated. Every node gossips on a timer, so this + // round happens on its own — it is the direction that carries a newcomer, not an + // extra step. + exchange(&mut b, &mut a); let full = vec![ NodeId::from_addr(A), From 8fc49fb66f74f54db57ab1ccc92caf9207e9643a Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Wed, 29 Jul 2026 00:18:52 -0300 Subject: [PATCH 12/17] test(server): pin that being dialed introduces only the dialer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A direction sweep over every comparison this unit adds found two that no test pinned. One was real: nothing failed when `gossip::exchange`'s inbound filter was replaced by `true`, so the rule that an inbound frame introduces only its own sender rested on the helper's implementation and not on a test. A newcomer admitted from a push is a join with no dial behind it — the unchecked join the adoption rules exist to close, one level down at the roster. The other was dead code rather than an untested rule: the claim guard also tested `!is_self(node)` and `!is_self(sender)`, both of which `can_be_verified` already decides where the claim is recorded, so dropping them changed nothing and no test could have failed. Both ends are now decided in one place. --- crates/server/src/membership.rs | 7 +++--- crates/server/tests/gossip.rs | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index 2117c87f..4c57a1e4 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -879,9 +879,10 @@ impl Membership { let Some(node) = NodeId::canonical(node.as_bytes()) else { continue; }; - // A claim is the *sender's*, so a frame that arrives on a link bound to this - // node's own id cannot make this node vouch for a member it never dialed. - if verified && &node != sender && !self.is_self(&node) && !self.is_self(sender) { + // A claim is the *sender's*, and a member does not vouch for itself. Both + // ends being something other than this node is decided once, by + // `can_be_verified`, where the claim is actually recorded. + if verified && &node != sender { claimed.push(node.clone()); } if self.is_self(&node) { diff --git a/crates/server/tests/gossip.rs b/crates/server/tests/gossip.rs index 6ef62abb..ade04315 100644 --- a/crates/server/tests/gossip.rs +++ b/crates/server/tests/gossip.rs @@ -216,6 +216,45 @@ fn a_seed_only_node_converges_on_the_full_cluster() { placement_agrees(&b, &c); } +#[test] +fn being_dialed_introduces_only_the_dialer() { + // An inbound frame carries whatever its sender chose to put in it, and admitting a + // *third* member from it is a join with no dial behind it — the unchecked join the + // adoption rules exist to close, reached one level down at the roster. Production + // (`Registry::apply_gossip`) admits only members the receiver already holds plus + // the sender's own tuple, so a newcomer is learned from a round this node + // initiated, never from being dialed. + let mut a = seeded(A, B); + let mut b = seeded(B, A); + let mut c = seeded(C, A); + + // A learns C the legitimate way: C dials A. + exchange(&mut c, &mut a); + assert!( + member_set(&a).contains(&NodeId::from_addr(C)), + "A learned C" + ); + + // Now A dials B and pushes a view that names C. B takes the dialer and nothing + // else. + exchange(&mut a, &mut b); + assert!( + !member_set(&b).contains(&NodeId::from_addr(C)), + "an inbound push introduces only its own sender", + ); + assert!( + member_set(&b).contains(&NodeId::from_addr(A)), + "B still holds A" + ); + + // B reaches C by gossiping *outward*, where the reply is a view it asked for. + exchange(&mut b, &mut a); + assert!( + member_set(&b).contains(&NodeId::from_addr(C)), + "and a round B initiated does carry the newcomer", + ); +} + #[test] fn convergence_holds_whatever_the_gossip_order() { // Drive several rounds of a fixed all-pairs schedule; the set must converge to From f1c92f73392479fc3aca67bd182f1d62dd839c08 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Wed, 29 Jul 2026 00:42:13 -0300 Subject: [PATCH 13/17] fix(server): an absent port is the scheme's default port, and both halves of a round ask one question MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 7. `canonical_member_addr` canonicalized the port as a number but never as a default, while the dial resolves an absent port from the scheme. So `wss://h` and `wss://h:443` were two node ids for one socket — and this one needs no malformed input: advertising a member portlessly is supported. One admitted member injects a single gossip tuple naming the other spelling of an *honest* member's address. Every node then dials it on the ordinary cadence, reaches that member's real listener, is answered by its real certificate, and records a truthful vouch; two hosts vouch and the phantom is adopted. It never speaks — the honest node's `PeerAuth` claims its configured spelling, so links bind to the other id, `is_self` is false for the phantom so that node cannot refute a suspicion of it, and its probe always succeeds because it is that node's own listener. Measured over 1000 rooms: 764 replicas held, effective primary of 245 that no node believes it leads, so those rooms accept no writes. Separately, the reply half computed `dial_establishes_identity(sender)` while the inbound half passed `true`. `verifiers` has to converge and this is what decides what enters it, so a plaintext member under `require_peer_identity` had its claims kept by every node it dialed and dropped by every node that dialed it: two nodes with identical evidence placed 197 of 256 rooms differently. Also: the gossip `verified` byte decoded leniently, so two byte strings encoded one message. Pass 7's third finding — a certified host self-introduces once per port per scheme, so C25's roster bound on claims does not bound that path — is C28's class and is folded into it rather than fixed here. --- DECISIONS.md | 4 ++++ KANBAN.md | 2 +- crates/core/src/protocol.rs | 13 ++++++++++++- crates/server/src/dial.rs | 21 +++++++++++++++++++++ crates/server/src/registry.rs | 15 +++++++++++---- 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index aa4cd996..d527b1e2 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -41,6 +41,10 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Pass 6 found that a fixpoint checked over a list is not a fixpoint, and that a claim's *sender* was ungated.** Pass 5's fix came with a test asserting the canonical form is idempotent over a list of spellings, which is a check of the list rather than of the reduction — and the family it omitted was the bracketed one. `PeerEndpoint::parse` enforced that a bracketed host is followed by a port and nothing else, but never that the brackets *contain* an IPv6 literal, so `host_of` returned everything between them with its colons intact and the canonical form emitted that verbatim with the port appended: `[10.0.0.1:9000]:9443` became the node id `10.0.0.1:9000:9443`. That id is not refused, it is *accepted* — it parses, so startup validation and the peer-identity member loop both pass — and being written in configuration it is adopted from birth and placed on every room HRW gives it, while the URL the dialer builds from it (`ws://10.0.0.1:9000:9443/`) is rejected as an invalid authority at send time, which classifies as unreachable rather than as a permanent address error and so is redialed forever. The rooms it holds replicas of can reach no quorum. Written as `CRDTSYNC_NODE_ID` it is the doppelgänger again and worse: the id has no canonical form at all, so every peer drops it from gossip and refuses its `PeerAuth`, and the node runs believing it has joined a cluster that cannot see it. No attacker is needed for either — bracketing a `host:port` pair is an ordinary operator typo. Brackets now mean an IPv6 literal and nothing else. Separately, `merge_liveness` gated the *node* a claim was about by `can_be_verified` but stored the claim under a `sender` it never gated, which is the exact hazard `note_verified`'s own contract states and applies only to the node: peer admission takes the id a link claims, so one certified host opens a link per port, claims a distinct id on each, and banks a permanent verifier entry per link against every member. Those entries are keyed on ids that are on no roster, so no reap ever strikes them; they can never *count* (only adopted verifiers do) but they grow without bound. Both sides of a claim now clear one predicate. +**Pass 7 found the port default, which is the doppelgänger reached from the gossip side.** `canonical_member_addr` canonicalized the port as a *number* but never as a *default*, while the dialer resolves an absent port from the scheme (`tokio-tungstenite` maps `wss`→443, `ws`→80). So `wss://h` and `wss://h:443` were two node ids for one socket — and unlike every earlier spelling defect, this one needs no malformed input at all: a member advertised portlessly is an explicitly supported deployment. One admitted member injects a single gossip tuple naming the other spelling of an **honest** member's address; the reply half of a round hands it on; every node then dials it on the ordinary cadence, reaches that honest member's real listener, is answered by its real certificate, and records a **truthful** vouch. Two hosts vouch and the phantom is adopted. It then never speaks — the honest node's `PeerAuth` claims its configured spelling, so every link binds to the other id; `is_self` is false for the phantom, so the honest node cannot even refute a suspicion of it; and its probe always succeeds, because it is that node's own listener. Measured over 1000 rooms: the phantom held 764 replicas and was *effective primary* of 245 rooms that no node believes it leads, so those rooms accept no writes, permanently. This is the harm C25 exists to prevent, reached with no forged evidence anywhere. An absent port is now the scheme's default port. + +**And the two halves of a round disagreed about attributability.** The reply half computed `dial_establishes_identity(sender)`; the inbound half passed `true` outright. `verifiers` has to converge, and this is what decides what enters it — so for a member advertising plaintext under `require_peer_identity` (a configuration that starts: the identity check requires a client certificate and a host-nameable id, not that members advertise TLS) its claims were kept by every node it dialed and dropped by every node that dialed it. Two nodes holding *identical* evidence built different rings: 197 of 256 rooms placed differently, permanently. Both halves now ask the one question, and it is a function of configuration and the member's own address, so every node computes it alike. + **The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. The same gap has one more edge, stated rather than hidden: §Peer Identity deliberately lets a certificate spell a host several ways, and each spelling is a different host, so a machine holding a multi-host certificate holds one trust unit per name it carries. Until C27, a deployment that wants the bar to mean two *machines* issues one host per certificate. **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. diff --git a/KANBAN.md b/KANBAN.md index af6a1b97..724fd796 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -429,7 +429,7 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C29 — a cluster of two adopted trust units can never re-admit a member it reaped, and a cluster whose members share a host can never admit one (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Adoption asks for `ADOPTION_VERIFIERS = 2` trust units other than the candidate's own, and two shapes cannot raise them. **Re-admission:** reaping drops a member from `configured`, so a returning node is a fresh join; in a two-node cluster the only other unit is the surviving node itself, so it verifies the returner a thousand times and the count never passes one — the survivor leads every room, the returner places over both, and the split is permanent with no operator signal and no repair short of restarting the survivor. Reachable by an ordinary long partition with no attacker, and reachable deliberately with C28's unrefutable `Dead`. **Admission:** a cluster whose members all share one host has exactly one unit, so no joiner ever clears the bar — a real shape (several instances per machine or pod), failing closed and silently. Both want the same thing: a bar that is expressible in the cluster it runs in, without becoming node-local (which is what made it non-convergent before). Candidates: a cluster-wide `configured`-derived denominator, an operator-set bar, or — cleanest — C27's placement over trust units, which makes "how many units exist" a thing the ring already knows. At minimum a node should *say* when its roster holds a member the bar cannot admit, instead of quietly not placing it. Also unbounded, and the same root: a **wildcard** SAN. The inbound binding refuses a wildcard deliberately (`cert_names_member` compares whole names), but the only thing that mints verification evidence is this node's *outbound* dial, which is plain TLS and honours one — so a member holding `*.x.example` answers truthfully at every name under it, each of which reads as a distinct trust unit, and each of which is then adoptable. Whatever C27 does about trust units has to close the outbound side too, or say that per-host certificates are a deployment requirement. → *Cluster*. -**C28 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. → *Cluster*. +**C28 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. **C25's pass 7 measured the third door into the same map, and it is the one C25 itself opened:** the inbound filter admits a sender's own tuple so that a peer can introduce itself, and `authenticate_peer` binds a link to any canonical id the presented certificate's host names — so one certified host self-introduces once per port per scheme, taking the roster from 4 to 1003 in a measured run, each entry then retained as a verifier claim. C25's own roster bound on claims (a sender must be a roster member) therefore does not bound this path, because the sender puts itself on the roster. No mint: they are all one trust unit and `verifier_units` counts units, so nothing is ever adopted by it. The cost is availability — `choose_peer` samples the roster uniformly, so at 1000 junk members a gossip round is useful 0.3% of the time and adoption evidence, `Dead` escalation and failover all stall. Wants the evidence bounded per trust unit rather than per id, and self-introduction capped at one pending id per certificate host. → *Cluster*. **C30 — the canonical node id is finer-grained than the address the resolver dials, so one endpoint still holds many ring positions (crates/server) — READY, no dependencies. Found by cold review during C25 (#369), measured.** C25 gave an advertise address one canonical spelling so that one endpoint holds one ring position, and enforced it at every door. The reduction decides "is this an IP literal" with Rust's strict `IpAddr::from_str`, while the thing that actually resolves the address is `getaddrinfo`, which accepts a much larger language: `0xa.0.0.1`, `10.0.1`, `10.1`, `167772161` and `10.0.0.01` all connect to `10.0.0.1`, and each is a *distinct* canonical node id **and a distinct trust unit** (`member_trust_unit` reduces the host by the certificate binding's relation, which also reads them as text). `[::ffff:10.0.0.1]` is the same shape across families — a distinct id whose dual-stack connect reaches the IPv4 listener. So the pass-3 write-stall returns through the resolver instead of through the spelling: one vouch from the attacker, two honest nodes dial the alias, are answered by the honest member, and verify it **truthfully**, and the ghost is adopted onto rooms nobody ever speaks as. Under peer mTLS the dial never opens (`ServerName::try_from("0xa.0.0.1")` is an `InvalidDnsNameError`, and a 16-octet V6 name matches no 4-octet `iPAddress` SAN), so this bites in exactly the postures C25 concedes — but the mechanism is the *resolver*, not the binding, and C25's concession is phrased about the binding. Note C27 does not close it either: these are distinct hosts, so weighing the ring by trust unit leaves them distinct units. Wants the canonical form to reject or fold every host form the resolver would, rather than every one Rust's parser recognizes. → *Cluster*. diff --git a/crates/core/src/protocol.rs b/crates/core/src/protocol.rs index 14d38595..b17a0dbf 100644 --- a/crates/core/src/protocol.rs +++ b/crates/core/src/protocol.rs @@ -1265,7 +1265,18 @@ pub fn decode_message(bytes: &[u8]) -> Result { let addr = cur.bytes()?; let incarnation = cur.u64()?; let state = member_state(cur.u8()?)?; - let verified = cur.u8()? != 0; + // One byte string per message: the encoder writes 0 or 1, so anything + // else is not a frame this cluster produced. + let verified = match cur.u8()? { + 0 => false, + 1 => true, + tag => { + return Err(ProtocolError::BadTag { + what: "gossip member verification", + tag, + }) + } + }; members.push((node, addr, incarnation, state, verified)); } Message::Gossip { members } diff --git a/crates/server/src/dial.rs b/crates/server/src/dial.rs index 36d1bdf0..8e49f9ce 100644 --- a/crates/server/src/dial.rs +++ b/crates/server/src/dial.rs @@ -217,6 +217,20 @@ pub fn canonical_member_addr(addr: &str) -> Option { Some("") => None, Some(digits) => Some(digits.parse::().ok()?), }; + // An absent port *is* the scheme's default port — that is how the dial resolves it + // — so the two spellings name one socket and must reduce to one id. Leaving them + // apart is the whole doppelgänger: a member advertised portlessly, or at `:443` + // under `wss`, has a second id that resolves to its own listener and is answered by + // its own certificate, so every honest node that dials the second spelling + // verifies it *truthfully* and adopts it. That id is then placed on rooms while the + // member itself only ever speaks as the first — it holds replicas that never ack, + // it is the effective primary of rooms no node believes it leads, and it cannot + // even refute a suspicion of itself, because `is_self` is false for it. + let default_port = match endpoint.transport { + PeerTransport::Tls => 443, + PeerTransport::Plain => 80, + }; + let port = port.filter(|port| *port != default_port); let host = normalized(host); let host = match host.parse::() { Ok(std::net::IpAddr::V6(v6)) => format!("[{v6}]"), @@ -712,6 +726,13 @@ mod tests { "[2001:db8::6]:9000", ), (" wss://Node-A.Example:9000 ", "wss://node-a.example:9000"), + // An absent port is the scheme's default port, because that is how the + // dial resolves it — so the two spellings are one socket and one id. + ("wss://node-a.example", "wss://node-a.example:443"), + ("node-a.example", "node-a.example:80"), + ("ws://node-a.example:80", "node-a.example"), + ("wss://10.0.0.1", "wss://10.0.0.1:443"), + ("[2001:db8::6]", "[2001:db8::6]:80"), ] { assert_eq!( canonical_member_addr(a), diff --git a/crates/server/src/registry.rs b/crates/server/src/registry.rs index dff93abc..c80a5ed3 100644 --- a/crates/server/src/registry.rs +++ b/crates/server/src/registry.rs @@ -985,10 +985,17 @@ impl Registry { membership.is_member(&node) || (&node == sender && addr == node.as_bytes()) }) .collect(); - // The link this arrived on is bound to `sender` (C13), and under a deployment - // that requires an identified peer it is bound by a certificate that names it, - // so the claims it carries are attributable to that member. - self.merge_gossip_attributed(sender, members, true); + // Whether a claim is attributable is asked the same way on both halves of a + // round, because `verifiers` has to converge and this is what decides what + // enters it. Reading the link here (bound to `sender`, C13) while the reply + // half reads the member's advertised transport made the two halves disagree + // about the same member: a plaintext member under `require_peer_identity` had + // its claims kept by every node it dialed and dropped by every node that dialed + // it, so nodes holding identical evidence built different rings and placed + // rooms differently — permanently. `dial_establishes_identity` is a function of + // configuration and the member's own address, so every node computes it alike. + let attributable = self.dial_establishes_identity(sender); + self.merge_gossip_attributed(sender, members, attributable); let reply = crate::gossip::gossip_frame(&self.known_liveness()); if let Some(conn) = self.conns.get_mut(&id) { conn.outbox.push(reply); From c425670df77a5694460ae56a6c6c54faf2d69460 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Wed, 29 Jul 2026 01:33:42 -0300 Subject: [PATCH 14/17] fix(server): read a member's transport off its id, and say what the evidence actually converges to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 8. `dial_establishes_identity` decides what enters `verifiers`, and `verifiers` is what the ring is derived from, so it has to be the same answer everywhere. It read `advertises_tls`, a roster lookup, which answers `false` for a member not yet learned — so the frame that *introduces* a sender was judged differently from every later one, and the claims it carried were kept by nodes that already knew it and dropped by nodes meeting it. One delivery, two states. A node id is an advertise address, so the transport is a property of the id and needs no state to read. The module docs claimed the evidence "converges however gossip interleaves". It does not. Liveness is relayed and anti-entropies; a verification claim is first-hand and is never passed on, deliberately, because with no signature on the wire "A and B verified X" is free for any node to write and relaying it would hand a compromised member a mint rather than a split. The consequence is that a compromised adopted member is a swing vote for any candidate exactly one other trust unit has reached: it sets the flag toward some peers and clears it toward others, and the groups place rooms differently — measured at 721/1000 rooms, 149 of them with two nodes each believing itself primary, surviving 30 further anti-entropy sweeps. The docs now state what holds, and the residual is pinned as a passing test and filed as C39. Board: C15 (#368/#373) filed its own C27-C33 first, so this branch's units are renumbered C34-C38 and the stale duplicate C26 line is dropped. --- DECISIONS.md | 4 +- KANBAN.md | 15 ++--- crates/server/src/membership.rs | 46 ++++++++++---- crates/server/tests/adoption.rs | 108 ++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 19 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index d527b1e2..a615d844 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -45,7 +45,9 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **And the two halves of a round disagreed about attributability.** The reply half computed `dial_establishes_identity(sender)`; the inbound half passed `true` outright. `verifiers` has to converge, and this is what decides what enters it — so for a member advertising plaintext under `require_peer_identity` (a configuration that starts: the identity check requires a client certificate and a host-nameable id, not that members advertise TLS) its claims were kept by every node it dialed and dropped by every node that dialed it. Two nodes holding *identical* evidence built different rings: 197 of 256 rooms placed differently, permanently. Both halves now ask the one question, and it is a function of configuration and the member's own address, so every node computes it alike. -**The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. The same gap has one more edge, stated rather than hidden: §Peer Identity deliberately lets a certificate spell a host several ways, and each spelling is a different host, so a machine holding a multi-host certificate holds one trust unit per name it carries. Until C27, a deployment that wants the bar to mean two *machines* issues one host per certificate. +**The residual, pinned as a passing test.** Under peer mTLS a member that owns a host owns every id under it: it answers at each ground id, and the honest nodes that dial it verify one *truthfully*. Adoption bounds the mint to the member's own host — already a shared trust unit per C13 — and to ids the cluster can actually reach, and it cannot bound it further, because every verification involved is genuine. So a compromised member still chooses which rooms it replicates, within its own host. Closing that needs the ring to weigh trust units rather than node ids — placement over hosts, where a host's ring position follows from a name it cannot grind — which is a placement change and not an evidence one. Filed as **C27**. With no certificates configured the bar is not a bar at all: a secret-holder binds a link to any member id it likes (C13's own residual), so it raises both verifiers itself, and what adoption still costs it is reachability at the address the minted id names. The same gap has one more edge, stated rather than hidden: §Peer Identity deliberately lets a certificate spell a host several ways, and each spelling is a different host, so a machine holding a multi-host certificate holds one trust unit per name it carries. Until C34, a deployment that wants the bar to mean two *machines* issues one host per certificate. + +**Pass 8: the evidence itself does not converge between nodes, and that is now stated rather than assumed.** Liveness is relayed, so it anti-entropies; `verifiers` is not, because `known_liveness` emits `verified_by_self` and a node passes on nobody else's word. That was the deliberate choice — unsigned relaying would let any node write "A and B verified X" and hand a compromised member a full mint rather than a split — but it means the one component placement is derived from has no exchange protocol between nodes. A compromised *adopted* member is therefore a swing vote for any candidate exactly one other trust unit has reached: it sets the flag toward some peers and clears it toward others, both frames well-formed and correctly attributed, and the groups place rooms differently. Measured over 1000 rooms with 5 honest nodes: 721 placed differently, 149 with two self-believed primaries, and the split survives 30 further anti-entropy sweeps with the attacker silent. For a candidate every honest node can reach it heals in one gossip period instead, so reachability is what makes it permanent. The module documentation claimed this set "converges however gossip interleaves"; that claim was false and is corrected to say what actually holds — union within a node, and between nodes only as far as each claim's own maker chose to send it. Filed as C39, whose fix is a claim signed by the verifier's peer key, which makes relaying safe and the union convergent. Pinned here as a passing test. **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. diff --git a/KANBAN.md b/KANBAN.md index 724fd796..76487a6b 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,7 +30,7 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ -**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Pass 4 found three more, all in how an id is derived from configuration:** the bootstrap exception read a *de-duplicated set*, so a peer list of only this node's own address collapsed to one member and dropped the bar to a single vouch — the very state the empty-list refusal had been added to catch; `CRDTSYNC_NODE_ID` was the one door that did not canonicalize (a regression from the plain `trim()` it replaced), so a node written in a non-canonical spelling carried a **doppelgänger of itself** — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, dropping every follower-head report; and the port was not canonicalized, so `:9000`/`:09000`/`:009000` were three ring positions on one listener, which is the targeted write-stall pass 3 was supposed to have closed and which an operator reaches with no attacker by writing a leading zero. The bar now reads the list rather than the set, every door canonicalizes, the port is read as a number, and an address with no canonical form is refused where it is written. **Passes 5 and 6 found two more, both places where a rule was stated but not made structural:** the canonical form was checked to be a fixpoint over a list of spellings rather than by construction, and brackets — the IPv6 literal's syntax — were never required to *contain* an IPv6 literal, so `[10.0.0.1:9000]:9443` (an operator typo, no attacker) parsed as a host whose own text carries colons and yielded the id `10.0.0.1:9000:9443`: accepted at the config door, adopted from birth as a configured member, placed on every room HRW gave it, and dialable by nobody, so those rooms could reach no quorum — and written as `CRDTSYNC_NODE_ID` it has no canonical form at all, so peers drop it from gossip and refuse its `PeerAuth` and the node joins nothing while believing it has. And a verifier claim gated the *node* it was about but never the **sender** it was stored under, so a certified host opening a link per port banked an unbounded verifier entry per link against every member — keyed on ids no reap ever strikes, because reaping strikes members. Both bars are now the same one predicate on both sides. **Residual, pinned as a passing test and filed as C27:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Pass 4 found three more, all in how an id is derived from configuration:** the bootstrap exception read a *de-duplicated set*, so a peer list of only this node's own address collapsed to one member and dropped the bar to a single vouch — the very state the empty-list refusal had been added to catch; `CRDTSYNC_NODE_ID` was the one door that did not canonicalize (a regression from the plain `trim()` it replaced), so a node written in a non-canonical spelling carried a **doppelgänger of itself** — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, dropping every follower-head report; and the port was not canonicalized, so `:9000`/`:09000`/`:009000` were three ring positions on one listener, which is the targeted write-stall pass 3 was supposed to have closed and which an operator reaches with no attacker by writing a leading zero. The bar now reads the list rather than the set, every door canonicalizes, the port is read as a number, and an address with no canonical form is refused where it is written. **Passes 5 and 6 found two more, both places where a rule was stated but not made structural:** the canonical form was checked to be a fixpoint over a list of spellings rather than by construction, and brackets — the IPv6 literal's syntax — were never required to *contain* an IPv6 literal, so `[10.0.0.1:9000]:9443` (an operator typo, no attacker) parsed as a host whose own text carries colons and yielded the id `10.0.0.1:9000:9443`: accepted at the config door, adopted from birth as a configured member, placed on every room HRW gave it, and dialable by nobody, so those rooms could reach no quorum — and written as `CRDTSYNC_NODE_ID` it has no canonical form at all, so peers drop it from gossip and refuse its `PeerAuth` and the node joins nothing while believing it has. And a verifier claim gated the *node* it was about but never the **sender** it was stored under, so a certified host opening a link per port banked an unbounded verifier entry per link against every member — keyed on ids no reap ever strikes, because reaping strikes members. Both bars are now the same one predicate on both sides. **Residual, pinned as a passing test and filed as C34:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. @@ -425,17 +425,18 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C16 — the reveal back-fill's batch dedup drops the copy that was zone-protected (crates/server) — READY, no dependencies. Found by cold review during C11 (#359), reasoned not reproduced.** The live fan-out's reveal prefix force-stamps every shell and back-filled op with the placing move's zone, precisely so the per-channel `zone_filter` cannot drop the content while keeping the shell. C11 made the back-fill skip any op the batch already carries — correct, since the duplicate took that op out of its transaction's count — but the surviving copy is then the *batch's*, which carries the zone its author computed at emit time. `zone_of_op` resolves against the live tree, so in one `atomic_transact` an edit inside a node carries the node's origin zone while the move carries the destination's: the per-channel filter can now drop the content and keep the shell, leaving the reader with a materialised node it never fills. Reaching it needs a node crossing zones, which a named-branch write does un-tokened (`handle_ops` gates `batch_zone_crossings` on `branch == MAIN_BRANCH`). Wants either the batch copy stamped with the move's zone the way the prefix copy was, or the zone filter taught to keep a shell and its content together by node rather than by zone. The `registry.rs` comment claiming the co-travel guarantee overstates it until then. → *Server / Fan-out*. -**C27 — a member that owns a host owns every ring position under it, because placement keys on node ids while a certificate names a host (crates/server) — READY, no dependencies. Found by cold review during C25 (#369) and pinned there as a passing test (`a_certified_member_still_mints_ids_on_its_own_host`).** C25 made rooms place on *adopted* members and made adoption the cluster's decision, which closes minting on anyone else's host, minting an id nobody can reach, and any single member manufacturing the evidence. What it cannot close is the mint on a member's **own** host: HRW scores a node id, C13 binds a link to the *host* a certificate names, and a host answers at as many ports as it likes — so a compromised member grinds `own-host:` until HRW places it on the room it wants, answers there, and every honest node that dials it verifies it **truthfully**. No evidence rule can help, because no one is lying. The consequence is the one C25 removed everywhere else: a member chooses which rooms it replicates, and from inside the replica set it reaches the leadership and durability gates. **The fix is a placement change, not an evidence one: weigh the ring by trust unit.** Score hosts rather than node ids, so a room's replica set is `n` distinct hosts and which rooms a host holds follows from a name it cannot grind; the member serving a host's position is then a local choice among that host's adopted members, and it does not matter which, since they are one trust unit already. That is also the better replication property — `n` replicas on `n` failure domains instead of `n` ids that may share a machine — and it makes the `replicas_for(room).contains(sender)` gate ask about exactly what the certificate vouches for. Costs to settle: two members on one host stop both replicating a room (a capacity change, not a correctness one), the replication factor becomes a host count, and every gate/redirect/quorum that names a member has to resolve host → member. Note this is *not* reachable without a compromised member's certificate: without peer mTLS the whole binding vouches for nothing anyway (C13's residual), and with it the mint is bounded to a host the operator already trusts to run a node. → *Cluster*. +**C39 — adoption evidence is first-hand and never relayed, so a compromised member that speaks selectively splits the ring permanently (crates/server) — READY, no dependencies. Found by cold review during C25 (#369), measured, and pinned there as a passing test (`a_compromised_member_that_speaks_selectively_splits_the_ring`).** C25 made the ring a function of `configured` + `verifiers` and disseminated the *evidence* rather than the verdict, precisely so that placement would be identical on every node. Liveness converges because every node re-gossips what it heard. `verifiers` does not: `known_liveness` emits one bit per member and that bit is `verified_by_self`, so a node passes on nobody else's word. That was deliberate — with no signature on the wire, "A and B verified X" is free for any node to write, and relaying unsigned claims would hand a compromised member a full mint rather than a split. The cost is that the one component placement depends on has no exchange protocol between nodes: a claim reaches exactly the nodes its maker chooses, and no honest node can carry it further or contradict it. So a compromised **adopted** member is a swing vote for any candidate that exactly one other trust unit has reached — it sets the `verified` flag toward some peers and clears it toward others, both frames well-formed and correctly attributed, and the two groups place rooms differently. Measured with 5 honest nodes and one attacker over 1000 rooms: **721/1000 rooms placed differently, 149 rooms where two honest nodes each believe themselves primary, 111 rooms one holds that the other denies it** — and the split survives 30 further full anti-entropy sweeps with the attacker silent, because there is no path by which the withheld claim could arrive. Those rooms are permanent `gate_replica_frame` → `Reject` → link-drop loops. For a joiner every honest node *can* reach the disagreement instead heals in one sweep (measured: 164/1000 rooms for one gossip period), so reachability is what makes it permanent, not what makes it exist. C35's unrefutable eviction chains into it: reaping strips a member from `configured`, so it returns as a pending joiner needing two units, handing the attacker a swing vote at every re-admission. **The fix is evidence a receiver can attribute without trusting the relay:** a claim signed by the verifier's peer key — which every node already holds under mTLS and every receiver can already check against that member's certificate — carried as `(verifier, target, signature)` instead of one self-scoped bool. Relaying then converges by union keyed on the *original* verifier, and a forging relayer gains nothing. The alternative is to stop deriving the ring from locally-held evidence and make adoption an agreed decision on the existing epoch/leadership machinery. → *Cluster*. -**C29 — a cluster of two adopted trust units can never re-admit a member it reaped, and a cluster whose members share a host can never admit one (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Adoption asks for `ADOPTION_VERIFIERS = 2` trust units other than the candidate's own, and two shapes cannot raise them. **Re-admission:** reaping drops a member from `configured`, so a returning node is a fresh join; in a two-node cluster the only other unit is the surviving node itself, so it verifies the returner a thousand times and the count never passes one — the survivor leads every room, the returner places over both, and the split is permanent with no operator signal and no repair short of restarting the survivor. Reachable by an ordinary long partition with no attacker, and reachable deliberately with C28's unrefutable `Dead`. **Admission:** a cluster whose members all share one host has exactly one unit, so no joiner ever clears the bar — a real shape (several instances per machine or pod), failing closed and silently. Both want the same thing: a bar that is expressible in the cluster it runs in, without becoming node-local (which is what made it non-convergent before). Candidates: a cluster-wide `configured`-derived denominator, an operator-set bar, or — cleanest — C27's placement over trust units, which makes "how many units exist" a thing the ring already knows. At minimum a node should *say* when its roster holds a member the bar cannot admit, instead of quietly not placing it. Also unbounded, and the same root: a **wildcard** SAN. The inbound binding refuses a wildcard deliberately (`cert_names_member` compares whole names), but the only thing that mints verification evidence is this node's *outbound* dial, which is plain TLS and honours one — so a member holding `*.x.example` answers truthfully at every name under it, each of which reads as a distinct trust unit, and each of which is then adoptable. Whatever C27 does about trust units has to close the outbound side too, or say that per-host certificates are a deployment requirement. → *Cluster*. +**C34 — a member that owns a host owns every ring position under it, because placement keys on node ids while a certificate names a host (crates/server) — READY, no dependencies. Found by cold review during C25 (#369) and pinned there as a passing test (`a_certified_member_still_mints_ids_on_its_own_host`).** C25 made rooms place on *adopted* members and made adoption the cluster's decision, which closes minting on anyone else's host, minting an id nobody can reach, and any single member manufacturing the evidence. What it cannot close is the mint on a member's **own** host: HRW scores a node id, C13 binds a link to the *host* a certificate names, and a host answers at as many ports as it likes — so a compromised member grinds `own-host:` until HRW places it on the room it wants, answers there, and every honest node that dials it verifies it **truthfully**. No evidence rule can help, because no one is lying. The consequence is the one C25 removed everywhere else: a member chooses which rooms it replicates, and from inside the replica set it reaches the leadership and durability gates. **The fix is a placement change, not an evidence one: weigh the ring by trust unit.** Score hosts rather than node ids, so a room's replica set is `n` distinct hosts and which rooms a host holds follows from a name it cannot grind; the member serving a host's position is then a local choice among that host's adopted members, and it does not matter which, since they are one trust unit already. That is also the better replication property — `n` replicas on `n` failure domains instead of `n` ids that may share a machine — and it makes the `replicas_for(room).contains(sender)` gate ask about exactly what the certificate vouches for. Costs to settle: two members on one host stop both replicating a room (a capacity change, not a correctness one), the replication factor becomes a host count, and every gate/redirect/quorum that names a member has to resolve host → member. Note this is *not* reachable without a compromised member's certificate: without peer mTLS the whole binding vouches for nothing anyway (C13's residual), and with it the mint is bounded to a host the operator already trusts to run a node. → *Cluster*. -**C28 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. **C25's pass 7 measured the third door into the same map, and it is the one C25 itself opened:** the inbound filter admits a sender's own tuple so that a peer can introduce itself, and `authenticate_peer` binds a link to any canonical id the presented certificate's host names — so one certified host self-introduces once per port per scheme, taking the roster from 4 to 1003 in a measured run, each entry then retained as a verifier claim. C25's own roster bound on claims (a sender must be a roster member) therefore does not bound this path, because the sender puts itself on the roster. No mint: they are all one trust unit and `verifier_units` counts units, so nothing is ever adopted by it. The cost is availability — `choose_peer` samples the roster uniformly, so at 1000 junk members a gossip round is useful 0.3% of the time and adoption evidence, `Dead` escalation and failover all stall. Wants the evidence bounded per trust unit rather than per id, and self-introduction capped at one pending id per certificate host. → *Cluster*. +**C36 — a cluster of two adopted trust units can never re-admit a member it reaped, and a cluster whose members share a host can never admit one (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Adoption asks for `ADOPTION_VERIFIERS = 2` trust units other than the candidate's own, and two shapes cannot raise them. **Re-admission:** reaping drops a member from `configured`, so a returning node is a fresh join; in a two-node cluster the only other unit is the surviving node itself, so it verifies the returner a thousand times and the count never passes one — the survivor leads every room, the returner places over both, and the split is permanent with no operator signal and no repair short of restarting the survivor. Reachable by an ordinary long partition with no attacker, and reachable deliberately with C35's unrefutable `Dead`. **Admission:** a cluster whose members all share one host has exactly one unit, so no joiner ever clears the bar — a real shape (several instances per machine or pod), failing closed and silently. Both want the same thing: a bar that is expressible in the cluster it runs in, without becoming node-local (which is what made it non-convergent before). Candidates: a cluster-wide `configured`-derived denominator, an operator-set bar, or — cleanest — C34's placement over trust units, which makes "how many units exist" a thing the ring already knows. At minimum a node should *say* when its roster holds a member the bar cannot admit, instead of quietly not placing it. Also unbounded, and the same root: a **wildcard** SAN. The inbound binding refuses a wildcard deliberately (`cert_names_member` compares whole names), but the only thing that mints verification evidence is this node's *outbound* dial, which is plain TLS and honours one — so a member holding `*.x.example` answers truthfully at every name under it, each of which reads as a distinct trust unit, and each of which is then adoptable. Whatever C34 does about trust units has to close the outbound side too, or say that per-host certificates are a deployment requirement. → *Cluster*. -**C30 — the canonical node id is finer-grained than the address the resolver dials, so one endpoint still holds many ring positions (crates/server) — READY, no dependencies. Found by cold review during C25 (#369), measured.** C25 gave an advertise address one canonical spelling so that one endpoint holds one ring position, and enforced it at every door. The reduction decides "is this an IP literal" with Rust's strict `IpAddr::from_str`, while the thing that actually resolves the address is `getaddrinfo`, which accepts a much larger language: `0xa.0.0.1`, `10.0.1`, `10.1`, `167772161` and `10.0.0.01` all connect to `10.0.0.1`, and each is a *distinct* canonical node id **and a distinct trust unit** (`member_trust_unit` reduces the host by the certificate binding's relation, which also reads them as text). `[::ffff:10.0.0.1]` is the same shape across families — a distinct id whose dual-stack connect reaches the IPv4 listener. So the pass-3 write-stall returns through the resolver instead of through the spelling: one vouch from the attacker, two honest nodes dial the alias, are answered by the honest member, and verify it **truthfully**, and the ghost is adopted onto rooms nobody ever speaks as. Under peer mTLS the dial never opens (`ServerName::try_from("0xa.0.0.1")` is an `InvalidDnsNameError`, and a 16-octet V6 name matches no 4-octet `iPAddress` SAN), so this bites in exactly the postures C25 concedes — but the mechanism is the *resolver*, not the binding, and C25's concession is phrased about the binding. Note C27 does not close it either: these are distinct hosts, so weighing the ring by trust unit leaves them distinct units. Wants the canonical form to reject or fold every host form the resolver would, rather than every one Rust's parser recognizes. → *Cluster*. +**C35 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. **C25's pass 7 measured the third door into the same map, and it is the one C25 itself opened:** the inbound filter admits a sender's own tuple so that a peer can introduce itself, and `authenticate_peer` binds a link to any canonical id the presented certificate's host names — so one certified host self-introduces once per port per scheme, taking the roster from 4 to 1003 in a measured run, each entry then retained as a verifier claim. C25's own roster bound on claims (a sender must be a roster member) therefore does not bound this path, because the sender puts itself on the roster. No mint: they are all one trust unit and `verifier_units` counts units, so nothing is ever adopted by it. The cost is availability — `choose_peer` samples the roster uniformly, so at 1000 junk members a gossip round is useful 0.3% of the time and adoption evidence, `Dead` escalation and failover all stall. Wants the evidence bounded per trust unit rather than per id, and self-introduction capped at one pending id per certificate host. → *Cluster*. -**C31 — a member adopted after startup is placed on rooms this node cannot replicate to (crates/server) — READY, no dependencies. Found by cold review during C25 (#369).** `spawn_peers` builds the outbound `peer_conns` map **once**, from the roster as it stood at startup, and `dispatch_replication` silently drops a frame for a follower with no entry. Pre-existing — nothing in C25 changed either — but C25 is what makes the path ordinary: its stated success case is a genuine joiner clearing the adoption bar "within a few rounds", after which every node places rooms on it. Those rooms now have a replica this node never sends to and never hears from, so the joiner degrades exactly the quorum adoption exists to protect, and the drop is silent. Wants the peer connection set to follow the adopted set rather than the startup roster, with a signal when a placed follower has no link. → *Cluster*. +**C37 — the canonical node id is finer-grained than the address the resolver dials, so one endpoint still holds many ring positions (crates/server) — READY, no dependencies. Found by cold review during C25 (#369), measured.** C25 gave an advertise address one canonical spelling so that one endpoint holds one ring position, and enforced it at every door. The reduction decides "is this an IP literal" with Rust's strict `IpAddr::from_str`, while the thing that actually resolves the address is `getaddrinfo`, which accepts a much larger language: `0xa.0.0.1`, `10.0.1`, `10.1`, `167772161` and `10.0.0.01` all connect to `10.0.0.1`, and each is a *distinct* canonical node id **and a distinct trust unit** (`member_trust_unit` reduces the host by the certificate binding's relation, which also reads them as text). `[::ffff:10.0.0.1]` is the same shape across families — a distinct id whose dual-stack connect reaches the IPv4 listener. So the pass-3 write-stall returns through the resolver instead of through the spelling: one vouch from the attacker, two honest nodes dial the alias, are answered by the honest member, and verify it **truthfully**, and the ghost is adopted onto rooms nobody ever speaks as. Under peer mTLS the dial never opens (`ServerName::try_from("0xa.0.0.1")` is an `InvalidDnsNameError`, and a 16-octet V6 name matches no 4-octet `iPAddress` SAN), so this bites in exactly the postures C25 concedes — but the mechanism is the *resolver*, not the binding, and C25's concession is phrased about the binding. Note C34 does not close it either: these are distinct hosts, so weighing the ring by trust unit leaves them distinct units. Wants the canonical form to reject or fold every host form the resolver would, rather than every one Rust's parser recognizes. → *Cluster*. + +**C38 — a member adopted after startup is placed on rooms this node cannot replicate to (crates/server) — READY, no dependencies. Found by cold review during C25 (#369).** `spawn_peers` builds the outbound `peer_conns` map **once**, from the roster as it stood at startup, and `dispatch_replication` silently drops a frame for a follower with no entry. Pre-existing — nothing in C25 changed either — but C25 is what makes the path ordinary: its stated success case is a genuine joiner clearing the adoption bar "within a few rounds", after which every node places rooms on it. Those rooms now have a replica this node never sends to and never hears from, so the joiner degrades exactly the quorum adoption exists to protect, and the drop is silent. Wants the peer connection set to follow the adopted set rather than the startup roster, with a signal when a placed follower has no link. → *Cluster*. -**C26 — the SDK apply seams report a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — READY, no dependencies. Split out of C19 (#367) rather than folded in.** `crdtsync_client_receive`/`WasmClient::receive` fold a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learns how many ops applied *now* and nothing else. C19 made the distinction answerable — `Op::is_admissible` is public and pure — but did not plumb it: a buffered op (which a later arrival commits) and an op no replica will ever hold are both reported as "not counted", so an SDK peered with a buggy or hostile writer sees edits vanish with no signal, and the two cases call for opposite responses (wait vs. surface a client bug). `client.rs` needs no change on this axis — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it, so nothing is missed. The server side is closed: `handle_ops` answers `ErrorCode::MalformedOp` and the ingest seams drop such a record, so this is the *offline / P2P / relay* seam, where no server sits between the writer and the fold. Wants a refused count (or a refused-op list) beside the applied one across FFI, wasm, JS, Python and Go — and is worth taking with **C23**, which has the same shape: a real refusal the SDKs render as an edit that silently did nothing. → *SDK / Ops*. **C14 — a redacted op delta leaves the same re-mint hole, and no frame can carry the ids it withholds (crates/server + crates/core) — READY, no dependencies. Found during C9 (#357), reproduced.** C9 closed the *snapshot* seam: a projected snapshot now names the recipient's own ids, so adoption never frees one the room's log holds. An **uncompacted** room serves an op delta instead, and the per-op read filter withholds the recipient's own ops on paths it may no longer read — leaving a hole in its own run with nothing to repair it from. Measured on C9's own fixture: a reader with write at `/` and read on `/a` alone writes into `/b` (seqs 0,1) then `/a` (seq 2); it restarts, its delta withholds `/b`, `next_seq` reports 0, and its next write mints seq 0 straight onto its own withheld id — the write never lands. The state encoding was the carrier for the snapshot case; an `Ops` frame has no equivalent, so closing this needs one: a frontier field on the delta, or a rule that a reader's *own* ops are never withheld from it (which trades the hole for materialising ids in a subtree it may not read — the reason C9 did not simply stop scrubbing). Note the second shape also has to answer what an op carries beyond its id: its target and content may name denied structure. → *Server / Fan-out*. diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index 4c57a1e4..46c560a8 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -51,11 +51,24 @@ //! shared member set. A node records only what it knows first-hand — that it has //! completed an identity-checked peer link to a member ([`note_verified`]) — and //! that claim rides gossip attributed to the node that made it. A member is adopted -//! once [`ADOPTION_VERIFIERS`] *already-adopted* members have verified it, so the -//! evidence is a grow-only set that merges the same way liveness does and converges -//! on the same anti-entropy. A member never verifies itself, so no node can place -//! itself; the members a node was *configured* with are adopted from birth, since -//! the operator's config is the root of trust the cluster starts from. +//! once [`ADOPTION_VERIFIERS`] *already-adopted* members have verified it. A member +//! never verifies itself, so no node can place itself; the members a node was +//! *configured* with are adopted from birth, since the operator's config is the root +//! of trust the cluster starts from. +//! +//! **What converges, and what a Byzantine member can still do.** A claim is +//! first-hand and is never relayed: this node tells its peers what *it* verified, and +//! passes on nobody else's word, because with no signature on the wire "A and B +//! verified X" is free for any node to write. So among honest members the evidence is +//! grow-only and converges — every node dials every member it knows and eventually +//! tells every other node — but a *compromised* member's claim reaches exactly the +//! nodes it chooses to send it to. It can set the flag toward some peers and clear it +//! toward others, and no honest node can carry the claim past them or contradict it. +//! Where such a member is the swing vote — a candidate that exactly one other trust +//! unit has reached — honest nodes then disagree about the ring, and if the candidate +//! stays unreachable to the rest, that disagreement does not heal. Closing it needs +//! evidence a receiver can attribute without trusting the relay, i.e. a claim signed +//! by the verifier's peer key; see KANBAN C39. //! //! [`note_verified`]: Membership::note_verified @@ -264,9 +277,14 @@ pub struct Membership { /// (attributed to the member its link is bound to), so every entry is a /// first-hand claim by the node named — a member can vouch for itself only, and /// never *for* itself ([`note_verified`](Self::note_verified) refuses that). - /// Grow-only per member, so it merges by union and converges however gossip - /// interleaves. Claims by members that are not (yet) adopted are retained but do - /// not count, so a pending member cannot vouch another pending member in. + /// Grow-only per member, so it merges by union: within one node the order claims + /// arrive in cannot change the result. Between nodes it converges only as far as + /// the claims travel, and a claim travels only from the member that made it — + /// nothing here relays another node's word, so a member that sends different + /// claims to different peers leaves them holding different evidence permanently + /// (see the module docs, and KANBAN C39). Claims by members that are not (yet) + /// adopted are retained but do not count, so a pending member cannot vouch + /// another pending member in. verifiers: HashMap>, } @@ -489,10 +507,16 @@ impl Membership { /// direction, where a member's advertised scheme describes its own listener and /// says nothing about the link carrying its identity here, an outbound dial runs /// over exactly the transport that address declares. + /// Read off the **id**, not off the roster. A node id is an advertise address, so + /// the transport is a property of the id itself and needs no state to decide — + /// which is what makes this answer the same on every node. A roster lookup would + /// answer `false` for a member this node has not learned yet, so the frame that + /// *introduces* a member would be judged differently from every later one, and the + /// claims it carries would be kept on the nodes that already knew the sender and + /// dropped on the nodes meeting it — one delivery, two states. pub fn advertises_tls(&self, node: &NodeId) -> bool { - self.addrs - .get(node) - .and_then(|addr| std::str::from_utf8(addr).ok()) + std::str::from_utf8(node.as_bytes()) + .ok() .and_then(|addr| crate::dial::PeerEndpoint::parse(addr).ok()) .is_some_and(|endpoint| endpoint.is_tls()) } diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index 0bad6f97..f61e0e3a 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -468,6 +468,43 @@ fn a_member_cannot_verify_itself_into_the_ring() { assert!(!view.is_adopted(&minted)); } +#[test] +fn whether_a_dial_establishes_identity_is_read_off_the_id_alone() { + // Whether a member's claims are attributable decides what enters `verifiers`, and + // `verifiers` is what the ring is derived from — so it has to be the same answer on + // every node, including one meeting that member for the first time. A node id *is* + // an advertise address, so the transport is a property of the id. Were this a + // roster lookup it would answer `false` for a member not yet learned, and the frame + // that *introduces* a sender would be judged differently from every later one: the + // claims it carries kept by the nodes that already knew it and dropped by the nodes + // meeting it, from one delivery. + let known = membership_for(SELF_ADDR); + let stranger = + Membership::from_static_config(None, Some(SELF_ADDR), "10.7.7.7:9000", N).unwrap(); + for addr in [ + "wss://node-a.example:9000", + "node-a.example:9000", + "wss://10.0.0.1:9000", + "10.0.0.1:9000", + ] { + let node = NodeId::from(addr); + assert!( + known.is_member(&node) || !stranger.is_member(&node), + "{addr}: the two views hold different rosters", + ); + assert_eq!( + known.advertises_tls(&node), + stranger.advertises_tls(&node), + "{addr}: a view that has never seen the member answers the same", + ); + assert_eq!( + stranger.advertises_tls(&node), + addr.starts_with("wss://"), + "{addr}: and the answer is the id's own transport", + ); + } +} + #[test] fn a_claim_is_recorded_against_the_member_whose_link_carried_it() { // The claim is worth exactly the link it arrived on, so it is attributed to that @@ -1648,3 +1685,74 @@ fn a_second_spelling_of_an_honest_member_is_not_a_second_member() { } assert_eq!(m.members().len(), before, "no ghost joined the roster"); } + +#[test] +fn a_compromised_member_that_speaks_selectively_splits_the_ring() { + // The accepted residual, pinned so it is a decision and not a surprise (KANBAN + // C39). A claim is first-hand and never relayed — with no signature on the wire, + // "A and B verified X" is free for any node to write — so a claim reaches exactly + // the nodes its maker chooses to send it to, and no honest node can carry it + // further or contradict it. A compromised member is therefore a swing vote for any + // candidate that exactly one other trust unit has reached: it sets the flag toward + // some peers and clears it toward others, and the two groups place rooms + // differently. Where the candidate stays unreachable to the rest, no later gossip + // repairs it, because there is no path by which the claim could arrive. + let candidate = NodeId::from("wss://joiner.example:9000"); + // Both are configured members, so both are adopted and their word counts. + let honest = "10.0.0.1:9000"; + let attacker = "10.0.0.2:9000"; + + let mut told = membership_for(SELF_ADDR); + let mut untold = membership_for(SELF_ADDR); + for m in [&mut told, &mut untold] { + m.add_member(candidate.clone()); + // The one honest unit that reached the candidate says so to both. + vouch(m, honest, &candidate); + } + assert!(!told.is_adopted(&candidate), "one unit is not the bar"); + assert!(!untold.is_adopted(&candidate)); + + // The compromised member sets the flag toward one node and clears it toward the + // other. Both frames are well-formed and each is attributed to the member whose + // link carried it, which is exactly what the rules require. + vouch(&mut told, attacker, &candidate); + untold.merge_liveness( + &NodeId::from(attacker), + [( + candidate.clone(), + candidate.as_bytes().to_vec(), + 0, + MemberState::Alive, + false, + )], + ); + + assert!( + told.is_adopted(&candidate), + "two units, on the node it told" + ); + assert!( + !untold.is_adopted(&candidate), + "and one unit on the node it did not", + ); + + // The rings differ, and they differ about rooms: the two nodes disagree on who + // holds and who leads. + let differing = (0..256u32) + .map(|i| format!("room-{i}")) + .filter(|room| told.replicas_for(room.as_bytes()) != untold.replicas_for(room.as_bytes())) + .count(); + assert!(differing > 0, "a ring split is what the disagreement costs",); + + // And it does not heal: every honest exchange between the two carries only what + // each verified itself, so neither can learn what the attacker told the other. + for _ in 0..8 { + crdtsync_server::gossip::exchange(&mut told, &mut untold); + crdtsync_server::gossip::exchange(&mut untold, &mut told); + } + assert!(told.is_adopted(&candidate)); + assert!( + !untold.is_adopted(&candidate), + "no anti-entropy carries a claim its maker withheld", + ); +} From 52c6188f2f9ee05179b038017f1c145292c9d4ac Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Wed, 29 Jul 2026 01:35:38 -0300 Subject: [PATCH 15/17] docs: state C39's residual plainly on the board, and why C25 shipped with it open --- DECISIONS.md | 2 ++ KANBAN.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DECISIONS.md b/DECISIONS.md index a615d844..c0ed7dd8 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -49,6 +49,8 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Pass 8: the evidence itself does not converge between nodes, and that is now stated rather than assumed.** Liveness is relayed, so it anti-entropies; `verifiers` is not, because `known_liveness` emits `verified_by_self` and a node passes on nobody else's word. That was the deliberate choice — unsigned relaying would let any node write "A and B verified X" and hand a compromised member a full mint rather than a split — but it means the one component placement is derived from has no exchange protocol between nodes. A compromised *adopted* member is therefore a swing vote for any candidate exactly one other trust unit has reached: it sets the flag toward some peers and clears it toward others, both frames well-formed and correctly attributed, and the groups place rooms differently. Measured over 1000 rooms with 5 honest nodes: 721 placed differently, 149 with two self-believed primaries, and the split survives 30 further anti-entropy sweeps with the attacker silent. For a candidate every honest node can reach it heals in one gossip period instead, so reachability is what makes it permanent. The module documentation claimed this set "converges however gossip interleaves"; that claim was false and is corrected to say what actually holds — union within a node, and between nodes only as far as each claim's own maker chose to send it. Filed as C39, whose fix is a claim signed by the verifier's peer key, which makes relaying safe and the union convergent. Pinned here as a passing test. +**Why this shipped with that open.** It is a real cost, not a technicality: C25 adds a failure *dimension* that did not exist before, because the member set converges and the verifier set does not, and "two honest nodes disagree about the ring permanently" is the precise case the unit was written to prevent. It ships anyway because it is strictly dominated. Under the same threat model — a compromised *adopted* member — a cluster without C25 hands that attacker a superset of this capability: it grinds a node id, places itself into any room's replica set unilaterally, supersedes the leader with a forged epoch, and makes majority-ack release a client `Accepted` for a write no majority holds. With C25 it is reduced to swinging candidates that sit exactly at the bar. No threat model comes out worse, so withholding this would have preserved the easier attack rather than a safe state. The real fix needs signing infrastructure that is not wired, which by the dependency rule makes it a separate unit rather than scope creep here. + **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. ## 2026-07-28 · C26 refused-op signal (#370) · the SDK fold answers two counts, because at the seam with no server the applied count alone renders a permanent refusal as silence diff --git a/KANBAN.md b/KANBAN.md index 76487a6b..0e33092b 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -425,7 +425,7 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C16 — the reveal back-fill's batch dedup drops the copy that was zone-protected (crates/server) — READY, no dependencies. Found by cold review during C11 (#359), reasoned not reproduced.** The live fan-out's reveal prefix force-stamps every shell and back-filled op with the placing move's zone, precisely so the per-channel `zone_filter` cannot drop the content while keeping the shell. C11 made the back-fill skip any op the batch already carries — correct, since the duplicate took that op out of its transaction's count — but the surviving copy is then the *batch's*, which carries the zone its author computed at emit time. `zone_of_op` resolves against the live tree, so in one `atomic_transact` an edit inside a node carries the node's origin zone while the move carries the destination's: the per-channel filter can now drop the content and keep the shell, leaving the reader with a materialised node it never fills. Reaching it needs a node crossing zones, which a named-branch write does un-tokened (`handle_ops` gates `batch_zone_crossings` on `branch == MAIN_BRANCH`). Wants either the batch copy stamped with the move's zone the way the prefix copy was, or the zone filter taught to keep a shell and its content together by node rather than by zone. The `registry.rs` comment claiming the co-travel guarantee overstates it until then. → *Server / Fan-out*. -**C39 — adoption evidence is first-hand and never relayed, so a compromised member that speaks selectively splits the ring permanently (crates/server) — READY, no dependencies. Found by cold review during C25 (#369), measured, and pinned there as a passing test (`a_compromised_member_that_speaks_selectively_splits_the_ring`).** C25 made the ring a function of `configured` + `verifiers` and disseminated the *evidence* rather than the verdict, precisely so that placement would be identical on every node. Liveness converges because every node re-gossips what it heard. `verifiers` does not: `known_liveness` emits one bit per member and that bit is `verified_by_self`, so a node passes on nobody else's word. That was deliberate — with no signature on the wire, "A and B verified X" is free for any node to write, and relaying unsigned claims would hand a compromised member a full mint rather than a split. The cost is that the one component placement depends on has no exchange protocol between nodes: a claim reaches exactly the nodes its maker chooses, and no honest node can carry it further or contradict it. So a compromised **adopted** member is a swing vote for any candidate that exactly one other trust unit has reached — it sets the `verified` flag toward some peers and clears it toward others, both frames well-formed and correctly attributed, and the two groups place rooms differently. Measured with 5 honest nodes and one attacker over 1000 rooms: **721/1000 rooms placed differently, 149 rooms where two honest nodes each believe themselves primary, 111 rooms one holds that the other denies it** — and the split survives 30 further full anti-entropy sweeps with the attacker silent, because there is no path by which the withheld claim could arrive. Those rooms are permanent `gate_replica_frame` → `Reject` → link-drop loops. For a joiner every honest node *can* reach the disagreement instead heals in one sweep (measured: 164/1000 rooms for one gossip period), so reachability is what makes it permanent, not what makes it exist. C35's unrefutable eviction chains into it: reaping strips a member from `configured`, so it returns as a pending joiner needing two units, handing the attacker a swing vote at every re-admission. **The fix is evidence a receiver can attribute without trusting the relay:** a claim signed by the verifier's peer key — which every node already holds under mTLS and every receiver can already check against that member's certificate — carried as `(verifier, target, signature)` instead of one self-scoped bool. Relaying then converges by union keyed on the *original* verifier, and a forging relayer gains nothing. The alternative is to stop deriving the ring from locally-held evidence and make adoption an agreed decision on the existing epoch/leadership machinery. → *Cluster*. +**C39 — adoption evidence is first-hand and never relayed, so a compromised member that speaks selectively splits the ring permanently (crates/server) — READY, no dependencies. Found by cold review during C25 (#369), measured, and pinned there as a passing test (`a_compromised_member_that_speaks_selectively_splits_the_ring`).** **Read this first: C25's central guarantee — that placement is identical on every node — does not hold against a compromised *adopted* member.** That is the guarantee the unit was written to establish, and it holds against every attacker except this one. The disagreement is **permanent, not eventually-consistent**: it survives unbounded further anti-entropy, because the mechanism is a claim that was never sent rather than one that was sent late. Measured with 5 honest nodes and one compromised member over 1000 rooms: **721/1000 rooms placed differently, 149 rooms in which two honest nodes each believe themselves primary, 111 rooms one node holds a replica of that the other denies it holds.** Reachability is what makes it *permanent*, not what makes it exist — against a candidate every honest node can reach, the identical trick still splits the ring, and merely heals one gossip period later (measured: 164/1000 rooms). C25 was merged with this open on the ground that it is strictly dominated: the same attacker on a cluster without C25 obtains a superset of this capability — it grinds an id, places itself into any room's replica set unilaterally, supersedes the leader with a forged epoch, and makes majority-ack release a client `Accepted` for a write no majority holds — so no threat model comes out worse, and holding the fix would have preserved the easier attack rather than a safe state. C25 does nonetheless add a failure *dimension* that did not exist before, because the member set converges and the verifier set does not. C25 made the ring a function of `configured` + `verifiers` and disseminated the *evidence* rather than the verdict, precisely so that placement would be identical on every node. Liveness converges because every node re-gossips what it heard. `verifiers` does not: `known_liveness` emits one bit per member and that bit is `verified_by_self`, so a node passes on nobody else's word. That was deliberate — with no signature on the wire, "A and B verified X" is free for any node to write, and relaying unsigned claims would hand a compromised member a full mint rather than a split. The cost is that the one component placement depends on has no exchange protocol between nodes: a claim reaches exactly the nodes its maker chooses, and no honest node can carry it further or contradict it. So a compromised **adopted** member is a swing vote for any candidate that exactly one other trust unit has reached — it sets the `verified` flag toward some peers and clears it toward others, both frames well-formed and correctly attributed, and the two groups place rooms differently. Measured with 5 honest nodes and one attacker over 1000 rooms: **721/1000 rooms placed differently, 149 rooms where two honest nodes each believe themselves primary, 111 rooms one holds that the other denies it** — and the split survives 30 further full anti-entropy sweeps with the attacker silent, because there is no path by which the withheld claim could arrive. Those rooms are permanent `gate_replica_frame` → `Reject` → link-drop loops. For a joiner every honest node *can* reach the disagreement instead heals in one sweep (measured: 164/1000 rooms for one gossip period), so reachability is what makes it permanent, not what makes it exist. C35's unrefutable eviction chains into it: reaping strips a member from `configured`, so it returns as a pending joiner needing two units, handing the attacker a swing vote at every re-admission. **The fix is evidence a receiver can attribute without trusting the relay:** a claim signed by the verifier's peer key — which every node already holds under mTLS and every receiver can already check against that member's certificate — carried as `(verifier, target, signature)` instead of one self-scoped bool. Relaying then converges by union keyed on the *original* verifier, and a forging relayer gains nothing. The alternative is to stop deriving the ring from locally-held evidence and make adoption an agreed decision on the existing epoch/leadership machinery. → *Cluster*. **C34 — a member that owns a host owns every ring position under it, because placement keys on node ids while a certificate names a host (crates/server) — READY, no dependencies. Found by cold review during C25 (#369) and pinned there as a passing test (`a_certified_member_still_mints_ids_on_its_own_host`).** C25 made rooms place on *adopted* members and made adoption the cluster's decision, which closes minting on anyone else's host, minting an id nobody can reach, and any single member manufacturing the evidence. What it cannot close is the mint on a member's **own** host: HRW scores a node id, C13 binds a link to the *host* a certificate names, and a host answers at as many ports as it likes — so a compromised member grinds `own-host:` until HRW places it on the room it wants, answers there, and every honest node that dials it verifies it **truthfully**. No evidence rule can help, because no one is lying. The consequence is the one C25 removed everywhere else: a member chooses which rooms it replicates, and from inside the replica set it reaches the leadership and durability gates. **The fix is a placement change, not an evidence one: weigh the ring by trust unit.** Score hosts rather than node ids, so a room's replica set is `n` distinct hosts and which rooms a host holds follows from a name it cannot grind; the member serving a host's position is then a local choice among that host's adopted members, and it does not matter which, since they are one trust unit already. That is also the better replication property — `n` replicas on `n` failure domains instead of `n` ids that may share a machine — and it makes the `replicas_for(room).contains(sender)` gate ask about exactly what the certificate vouches for. Costs to settle: two members on one host stop both replicating a room (a capacity change, not a correctness one), the replication factor becomes a host count, and every gate/redirect/quorum that names a member has to resolve host → member. Note this is *not* reachable without a compromised member's certificate: without peer mTLS the whole binding vouches for nothing anyway (C13's residual), and with it the mint is bounded to a host the operator already trusts to run a node. → *Cluster*. From 099aabc99a9e4ee925670e628541a1c25d7b407b Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Wed, 29 Jul 2026 02:31:36 -0300 Subject: [PATCH 16/17] fix(server): a claim about a member this view has not met is held, not dropped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 9. Two seams asked "is this node on the roster?" before retaining a verification claim, and both dropped rather than deferred. The frame that vouches for a member and the frame that introduces it are two frames, and which lands first is ordinary network ordering — so the identical three frames delivered as [joiner, s1, s2] adopted the joiner and as [s1, s2, joiner] left it pending, placing 959 of 2000 sample rooms differently. No compromised member and no selective speech: `adopted` is derived from `verifiers`, so making retention a function of arrival order made the ring one too, a level further down. It healed as soon as a voucher gossiped again, so it was a window and not a split. The part worth fixing is that the documentation asserted the opposite — that within one node the order claims arrive in cannot change the result. A claim is now held for a member the view has not met, and counts for nothing until that member exists, because the fixpoint only ever scans the roster. The tombstone rule survives: a reaped member refuses a held claim exactly as it refuses a carried one, so it cannot return pre-adopted. Pass 9's other findings stay on the board: the 64 MiB default frame cap and the roster x senders cross-product are C35's class, with measured costs added there; `require_peer_identity` is a cluster-wide protocol parameter that must be uniform, recorded in DECISIONS rather than filed. No HIGH found by pass 9. --- DECISIONS.md | 4 ++ KANBAN.md | 2 +- crates/server/src/membership.rs | 53 ++++++++++++++++++++- crates/server/src/registry.rs | 28 +++++++++-- crates/server/tests/adoption.rs | 83 +++++++++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 7 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index c0ed7dd8..9c877e21 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -51,6 +51,10 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Why this shipped with that open.** It is a real cost, not a technicality: C25 adds a failure *dimension* that did not exist before, because the member set converges and the verifier set does not, and "two honest nodes disagree about the ring permanently" is the precise case the unit was written to prevent. It ships anyway because it is strictly dominated. Under the same threat model — a compromised *adopted* member — a cluster without C25 hands that attacker a superset of this capability: it grinds a node id, places itself into any room's replica set unilaterally, supersedes the leader with a forged epoch, and makes majority-ack release a client `Accepted` for a write no majority holds. With C25 it is reduced to swinging candidates that sit exactly at the bar. No threat model comes out worse, so withholding this would have preserved the easier attack rather than a safe state. The real fix needs signing infrastructure that is not wired, which by the dependency rule makes it a separate unit rather than scope creep here. +**Pass 9: a claim about a member this view had not met yet was dropped, so the ring was a function of arrival order.** Two seams asked "is this node on the roster?" before retaining a claim, and both dropped rather than deferred. The frame that vouches for a member and the frame that introduces it are two frames, and which lands first is ordinary network ordering. So the identical three frames delivered as `[joiner, s1, s2]` adopted the joiner and as `[s1, s2, joiner]` left it pending — 959 of 2000 sample rooms placed differently, with no compromised member and no selective speech. It healed as soon as a voucher gossiped again, so it was a window rather than a split; but the module documentation asserted that within one node "the order claims arrive in cannot change the result", and that was false, which is the part worth fixing. A claim is now *held* for a member the view has not met and counts for nothing until that member exists, since the fixpoint only ever scans the roster. The tombstone rule survives it: a reaped member still refuses held claims, so it cannot return pre-adopted. Distinct from C39 — there the nodes receive different content, here byte-identical content in a different order. + +`require_peer_identity` is per-node configuration that feeds `verifiers`, so a rolling flip of it places rooms differently for the duration of the rollout (measured: 1021/2000 rooms, plaintext-advertised members only). It is self-limiting — the flag is an env var, a flip is a restart, and a restart rebuilds `verifiers` empty — and the fix suggested by review (dropping the `!require_peer_identity` short-circuit) would stop a plaintext cluster adopting anyone at all. It stands as a cluster-wide protocol parameter that must be uniform, like the cluster secret, rather than a per-node preference. + **Deployment.** No new configuration. A cluster grown by gossip now places a joiner a few rounds later than it did — after members have reached it rather than after they have heard of it — and a cluster whose members are all configured is byte-identical to before. `Message::Gossip`'s tuple gains a trailing flag byte per member, so nodes must be upgraded together. ## 2026-07-28 · C26 refused-op signal (#370) · the SDK fold answers two counts, because at the seam with no server the applied count alone renders a permanent refusal as silence diff --git a/KANBAN.md b/KANBAN.md index 0e33092b..720ba523 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -431,7 +431,7 @@ scalar / counter / register / element / map (#22–#27), list Fugue (#24), text **C36 — a cluster of two adopted trust units can never re-admit a member it reaped, and a cluster whose members share a host can never admit one (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Adoption asks for `ADOPTION_VERIFIERS = 2` trust units other than the candidate's own, and two shapes cannot raise them. **Re-admission:** reaping drops a member from `configured`, so a returning node is a fresh join; in a two-node cluster the only other unit is the surviving node itself, so it verifies the returner a thousand times and the count never passes one — the survivor leads every room, the returner places over both, and the split is permanent with no operator signal and no repair short of restarting the survivor. Reachable by an ordinary long partition with no attacker, and reachable deliberately with C35's unrefutable `Dead`. **Admission:** a cluster whose members all share one host has exactly one unit, so no joiner ever clears the bar — a real shape (several instances per machine or pod), failing closed and silently. Both want the same thing: a bar that is expressible in the cluster it runs in, without becoming node-local (which is what made it non-convergent before). Candidates: a cluster-wide `configured`-derived denominator, an operator-set bar, or — cleanest — C34's placement over trust units, which makes "how many units exist" a thing the ring already knows. At minimum a node should *say* when its roster holds a member the bar cannot admit, instead of quietly not placing it. Also unbounded, and the same root: a **wildcard** SAN. The inbound binding refuses a wildcard deliberately (`cert_names_member` compares whole names), but the only thing that mints verification evidence is this node's *outbound* dial, which is plain TLS and honours one — so a member holding `*.x.example` answers truthfully at every name under it, each of which reads as a distinct trust unit, and each of which is then adoptable. Whatever C34 does about trust units has to close the outbound side too, or say that per-host certificates are a deployment requirement. → *Cluster*. -**C35 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. **C25's pass 7 measured the third door into the same map, and it is the one C25 itself opened:** the inbound filter admits a sender's own tuple so that a peer can introduce itself, and `authenticate_peer` binds a link to any canonical id the presented certificate's host names — so one certified host self-introduces once per port per scheme, taking the roster from 4 to 1003 in a measured run, each entry then retained as a verifier claim. C25's own roster bound on claims (a sender must be a roster member) therefore does not bound this path, because the sender puts itself on the roster. No mint: they are all one trust unit and `verifier_units` counts units, so nothing is ever adopted by it. The cost is availability — `choose_peer` samples the roster uniformly, so at 1000 junk members a gossip round is useful 0.3% of the time and adoption evidence, `Dead` escalation and failover all stall. Wants the evidence bounded per trust unit rather than per id, and self-introduction capped at one pending id per certificate host. → *Cluster*. +**C35 — one member can evict any other from the ring for good, and can flood every node's roster (crates/server) — READY, no dependencies. Raised by cold review during C25 (#369), both reproduced.** Two gossip-plane inputs are unbounded and one is unrefutable. **Eviction:** `refute_if_stale` bumps a victim's incarnation with `received.max(me.incarnation).saturating_add(1)`, so a peer advertising `(victim, u64::MAX, Dead)` saturates the counter — the victim re-asserts `Alive` at `u64::MAX`, the equal-incarnation rule keeps `Dead` winning, the refutation never lands anywhere, and the victim is suspected, declared dead and reaped cluster-wide. The merge primitive predates C25; what C25 changes is the consequence — reaping now strips a member from `configured`, so the victim loses root-of-trust status and must be verified by two trust units to return. One compromised member can walk the cluster this way. Wants either a refusal of a self-tuple at or near the ceiling, or a bound on how far one merge may move a member's incarnation. **Flooding:** the *reply* half of a round learns every tuple a dialed peer sends, with no cap on tuples per frame or on roster size; each entry then rides every later gossip frame and takes a probe slot, so a compromised member replying with 10⁵ fabricated ids dilutes the roster on every node that dials it. Also pre-existing, and also worse now: a genuine joiner is adopted only once two members have *individually dialed* it, so diluting the roster starves adoption and keeps honest joiners out of the ring indefinitely without the attacker ever touching them. Wants a per-frame tuple cap and a roster ceiling, with an eviction rule that is not itself a lever. **C25's pass 9 measured the cost, and it is a latency attack on the whole node, not just a memory one:** `accept_hdr_async` is called with no `WebSocketConfig`, so tungstenite's default 64 MiB `max_message_size` applies, and one 31.1 MB reply frame (500 000 tuples, under half the cap) is absorbed in 868 ms — after which *every subsequent 1-tuple merge* costs 316 ms and each `known_liveness()` snapshot 269 ms, inside the single-threaded registry actor that serves all client traffic, and the node re-emits that 31.1 MB frame to every peer on every round. Measured scaling: 10 k tuples → 5.3 ms per later merge, 50 k → 27.9 ms, 200 k → 120 ms. The evidence map is a roster × senders cross-product, so it compounds: merge time rose 29 ms → 57 ms over 20 self-introduced aliases at roster 20 k. A `WebSocketConfig` bounding a peer frame to `expected_members × tuple_size` is the cheap half of the fix. **C25's pass 7 measured the third door into the same map, and it is the one C25 itself opened:** the inbound filter admits a sender's own tuple so that a peer can introduce itself, and `authenticate_peer` binds a link to any canonical id the presented certificate's host names — so one certified host self-introduces once per port per scheme, taking the roster from 4 to 1003 in a measured run, each entry then retained as a verifier claim. C25's own roster bound on claims (a sender must be a roster member) therefore does not bound this path, because the sender puts itself on the roster. No mint: they are all one trust unit and `verifier_units` counts units, so nothing is ever adopted by it. The cost is availability — `choose_peer` samples the roster uniformly, so at 1000 junk members a gossip round is useful 0.3% of the time and adoption evidence, `Dead` escalation and failover all stall. Wants the evidence bounded per trust unit rather than per id, and self-introduction capped at one pending id per certificate host. → *Cluster*. **C37 — the canonical node id is finer-grained than the address the resolver dials, so one endpoint still holds many ring positions (crates/server) — READY, no dependencies. Found by cold review during C25 (#369), measured.** C25 gave an advertise address one canonical spelling so that one endpoint holds one ring position, and enforced it at every door. The reduction decides "is this an IP literal" with Rust's strict `IpAddr::from_str`, while the thing that actually resolves the address is `getaddrinfo`, which accepts a much larger language: `0xa.0.0.1`, `10.0.1`, `10.1`, `167772161` and `10.0.0.01` all connect to `10.0.0.1`, and each is a *distinct* canonical node id **and a distinct trust unit** (`member_trust_unit` reduces the host by the certificate binding's relation, which also reads them as text). `[::ffff:10.0.0.1]` is the same shape across families — a distinct id whose dual-stack connect reaches the IPv4 listener. So the pass-3 write-stall returns through the resolver instead of through the spelling: one vouch from the attacker, two honest nodes dial the alias, are answered by the honest member, and verify it **truthfully**, and the ghost is adopted onto rooms nobody ever speaks as. Under peer mTLS the dial never opens (`ServerName::try_from("0xa.0.0.1")` is an `InvalidDnsNameError`, and a 16-octet V6 name matches no 4-octet `iPAddress` SAN), so this bites in exactly the postures C25 concedes — but the mechanism is the *resolver*, not the binding, and C25's concession is phrased about the binding. Note C34 does not close it either: these are distinct hosts, so weighing the ring by trust unit leaves them distinct units. Wants the canonical form to reject or fold every host form the resolver would, rather than every one Rust's parser recognizes. → *Cluster*. diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index 46c560a8..e424ebad 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -278,7 +278,10 @@ pub struct Membership { /// first-hand claim by the node named — a member can vouch for itself only, and /// never *for* itself ([`note_verified`](Self::note_verified) refuses that). /// Grow-only per member, so it merges by union: within one node the order claims - /// arrive in cannot change the result. Between nodes it converges only as far as + /// arrive in cannot change the result — which is why a claim about a member this + /// view has not met is *held* rather than dropped (see + /// [`claim_may_be_retained`](Self::claim_may_be_retained)), since dropping it made + /// the union, and so the ring, a function of arrival order. Between nodes it converges only as far as /// the claims travel, and a claim travels only from the member that made it — /// nothing here relays another node's word, so a member that sends different /// claims to different peers leaves them holding different evidence permanently @@ -661,6 +664,29 @@ impl Membership { } } + /// Record `sender`'s claims about members this view has not learned, for the seam + /// that admits a tuple's evidence without admitting the member it names. Same bars + /// as the claims carried on an admitted tuple: the sender is on the roster, and the + /// member it names is neither this node nor tombstoned. + pub fn note_claims(&mut self, sender: &NodeId, nodes: impl IntoIterator) { + if !self.can_be_verified(sender) { + return; + } + let mut rebuilt = false; + for node in nodes { + if self.claim_may_be_retained(&node) { + rebuilt |= self + .verifiers + .entry(node) + .or_default() + .insert(sender.clone()); + } + } + if rebuilt { + self.rebuild_placement(); + } + } + /// Whether a verification of `node` is admissible evidence at all — the bar every /// verifier claim clears, this node's own and a peer's alike. See /// [`note_verified`](Self::note_verified) for what each clause refuses. @@ -668,6 +694,29 @@ impl Membership { !self.is_self(node) && self.addrs.contains_key(node) } + /// Whether a *peer's* claim about `node` may be retained. Weaker than + /// [`can_be_verified`](Self::can_be_verified) in one place, deliberately: the claim + /// does not require `node` to be on the roster **yet**. + /// + /// The frame that carries a claim about a member and the frame that introduces + /// that member are two frames, and which arrives first is ordinary network + /// ordering. Requiring the member first made retention a function of arrival order, + /// and `adopted` is derived from `verifiers`, so the *ring* became a function of + /// arrival order one level down — two nodes given the identical frames in different + /// orders placed different rooms until the vouchers happened to gossip again. A + /// claim about a member this view has not met is held rather than dropped; it + /// counts for nothing until that member exists, because [`rebuild_placement`] only + /// ever scans the roster. + /// + /// A tombstoned node is still refused. It is not a member of this view, and banking + /// claims against it would let it return pre-adopted on evidence collected while + /// nobody could reach it. + /// + /// [`rebuild_placement`]: Self::rebuild_placement + fn claim_may_be_retained(&self, node: &NodeId) -> bool { + !self.is_self(node) && !self.reaped.contains_key(node) + } + /// How many **trust units** have vouched for `node`: the distinct hosts of the /// adopted members that verified it, its own host excluded. /// @@ -963,7 +1012,7 @@ impl Membership { // roster is what keeps it bounded at all. if self.can_be_verified(sender) { for node in claimed { - if self.can_be_verified(&node) { + if self.claim_may_be_retained(&node) { rebuilt |= self .verifiers .entry(node) diff --git a/crates/server/src/registry.rs b/crates/server/src/registry.rs index c80a5ed3..a3b89d2d 100644 --- a/crates/server/src/registry.rs +++ b/crates/server/src/registry.rs @@ -976,15 +976,30 @@ impl Registry { let Some(membership) = &self.membership else { return false; }; - let members = members - .into_iter() - .filter(|(node, addr, ..)| { + // An inbound frame introduces only its own sender: admitting a *third* member + // from it would be a join with no dial behind it. But refusing to admit the + // member is not a reason to discard the *claim* the same tuple carries — the + // frame naming a member and the frame introducing it are two frames, and which + // arrives first is ordinary network ordering. Dropping the claim made the ring + // a function of that order: the same three frames delivered as + // `[joiner, s1, s2]` adopted the joiner and as `[s1, s2, joiner]` left it + // pending, placing 959 of 2000 sample rooms differently until a voucher + // happened to gossip again. The claim is carried over separately, where it is + // held against a member this view does not have yet and counts for nothing + // until it does. + let (admitted, deferred): (Vec<_>, Vec<_>) = + members.into_iter().partition(|(node, addr, ..)| { // `is_member` holds for self too — a node is a member of its own view // from construction and is never reaped out of it. let node = NodeId::from(node.clone()); membership.is_member(&node) || (&node == sender && addr == node.as_bytes()) - }) + }); + let deferred: Vec = deferred + .into_iter() + .filter(|(_, _, _, _, verified)| *verified) + .filter_map(|(node, ..)| NodeId::canonical(&node)) .collect(); + let members = admitted; // Whether a claim is attributable is asked the same way on both halves of a // round, because `verifiers` has to converge and this is what decides what // enters it. Reading the link here (bound to `sender`, C13) while the reply @@ -996,6 +1011,11 @@ impl Registry { // configuration and the member's own address, so every node computes it alike. let attributable = self.dial_establishes_identity(sender); self.merge_gossip_attributed(sender, members, attributable); + if attributable && !deferred.is_empty() { + if let Some(membership) = &mut self.membership { + membership.note_claims(sender, deferred); + } + } let reply = crate::gossip::gossip_frame(&self.known_liveness()); if let Some(conn) = self.conns.get_mut(&id) { conn.outbox.push(reply); diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index f61e0e3a..f44b9f7d 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -1756,3 +1756,86 @@ fn a_compromised_member_that_speaks_selectively_splits_the_ring() { "no anti-entropy carries a claim its maker withheld", ); } + +#[test] +fn the_same_frames_in_any_order_build_the_same_ring() { + // `adopted` is derived from `verifiers`, so if retention depends on the order + // frames arrive in, the ring does too — one level down, and invisibly. The frame + // that vouches for a member and the frame that introduces it are two frames, and + // which lands first is ordinary network ordering, not evidence. + let joiner = NodeId::from("10.9.9.9:9000"); + let vouchers = ["10.0.0.1:9000", "10.0.0.2:9000"]; + + let introduce = |m: &mut Membership| m.add_member(joiner.clone()); + let vouch_from = |m: &mut Membership, who: &str| vouch(m, who, &joiner); + + // Member first, then the vouches. + let mut forward = membership_for(SELF_ADDR); + introduce(&mut forward); + for v in vouchers { + vouch_from(&mut forward, v); + } + + // The vouches first, then the member they are about. + let mut backward = membership_for(SELF_ADDR); + for v in vouchers { + vouch_from(&mut backward, v); + } + introduce(&mut backward); + + assert!(forward.is_adopted(&joiner), "vouched then placed"); + assert!( + backward.is_adopted(&joiner), + "a claim about a member this view has not met yet is held, not dropped", + ); + assert_eq!( + forward.adopted_members(), + backward.adopted_members(), + "the ring is a function of the evidence, not of its arrival order", + ); + let differing = (0..512u32) + .map(|i| format!("room-{i}")) + .filter(|r| forward.replicas_for(r.as_bytes()) != backward.replicas_for(r.as_bytes())) + .count(); + assert_eq!(differing, 0, "and so is every room's replica set"); +} + +#[test] +fn a_claim_held_for_an_unmet_member_still_respects_the_tombstone() { + // Holding a claim about a member this view has not met must not become a way to + // bank evidence against one it has *reaped*: that member is not absent, it is + // refused, and it must be re-verified after it returns. + let mut m = membership_for(SELF_ADDR); + let departed = NodeId::from("10.9.9.9:9000"); + m.add_member(departed.clone()); + for _ in 0..crdtsync_server::membership::DEAD_AFTER_FAILURES { + m.note_gossip_unreachable(&departed); + } + for _ in 0..crdtsync_server::membership::REAP_AFTER_DEAD_TICKS { + m.reap_dead(); + } + assert!(!m.is_member(&departed)); + + m.note_claims( + &NodeId::from("10.0.0.1:9000"), + [departed.clone(), departed.clone()], + ); + m.note_claims(&NodeId::from("10.0.0.2:9000"), [departed.clone()]); + assert!( + !m.has_verified(&NodeId::from("10.0.0.1:9000"), &departed), + "a tombstone refuses a held claim exactly as it refuses a carried one", + ); + + m.merge_liveness( + &NodeId::from("10.0.0.1:9000"), + [( + departed.clone(), + departed.as_bytes().to_vec(), + 1, + MemberState::Alive, + false, + )], + ); + assert!(m.is_member(&departed), "it rejoined"); + assert!(!m.is_adopted(&departed), "and it rejoined pending"); +} From 777f84d3bd04fd004f519145ab3030818dff78a7 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Wed, 29 Jul 2026 03:03:53 -0300 Subject: [PATCH 17/17] revert(server): drop a claim that outruns its subject, and pin that it is re-sent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold review pass 10, on my own pass-9 fix. Pass 9 measured a real window: a claim naming a member the receiver had not met was dropped with its tuple, so retention — and the ring, since `adopted` is derived from `verifiers` — depended on frame arrival order (959/2000 rooms). Holding the claim closed that and opened something worse. Held ids are on no roster. `reap_dead` strikes `verifiers` only for ids it reaps off the roster, and `rebuild_placement` only ever scans the roster, so a held claim is state nothing reclaims and nothing can see. One inbound 24.9 MB frame retained 376.5 MB — 15x the wire — and stalled the single-threaded registry actor for 591 ms; at tungstenite's default 64 MiB cap that is 3.3 million claims, about 1 GB, from one frame. It also made the tombstone refusal temporal, because `reaped` is pruned after TOMBSTONE_RETENTION_TICKS: past about five minutes a reaped id was neither on the roster nor tombstoned, so claims banked again and a returning member was adopted pre-verified (980/2000 rooms). The comment above the sibling site still stated the invariant the code had stopped holding. The window was the cheaper defect, and it is not even a loss: a verifier re-advertises what it verified every round, so a claim that outruns its subject is re-sent within one gossip interval and the view converges then. That healing property is now pinned; the order-independence it replaced was never true and is no longer claimed. The regression test I wrote for the held-claim version was vacuous. It drove `merge_liveness`, whose `None` branch admits the unmet member from the same tuple, so the claim was retained either way — it passes on the commit it was written to catch. The replacement drives the seam that actually drops. --- DECISIONS.md | 4 +- KANBAN.md | 2 +- crates/server/src/membership.rs | 58 +++---------------- crates/server/src/registry.rs | 36 +++++------- crates/server/tests/adoption.rs | 98 ++++++++------------------------- 5 files changed, 48 insertions(+), 150 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index 9c877e21..b5d8eccb 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -51,7 +51,9 @@ The entries below (2026-07-02) are a backfill: design changes made during the v0 **Why this shipped with that open.** It is a real cost, not a technicality: C25 adds a failure *dimension* that did not exist before, because the member set converges and the verifier set does not, and "two honest nodes disagree about the ring permanently" is the precise case the unit was written to prevent. It ships anyway because it is strictly dominated. Under the same threat model — a compromised *adopted* member — a cluster without C25 hands that attacker a superset of this capability: it grinds a node id, places itself into any room's replica set unilaterally, supersedes the leader with a forged epoch, and makes majority-ack release a client `Accepted` for a write no majority holds. With C25 it is reduced to swinging candidates that sit exactly at the bar. No threat model comes out worse, so withholding this would have preserved the easier attack rather than a safe state. The real fix needs signing infrastructure that is not wired, which by the dependency rule makes it a separate unit rather than scope creep here. -**Pass 9: a claim about a member this view had not met yet was dropped, so the ring was a function of arrival order.** Two seams asked "is this node on the roster?" before retaining a claim, and both dropped rather than deferred. The frame that vouches for a member and the frame that introduces it are two frames, and which lands first is ordinary network ordering. So the identical three frames delivered as `[joiner, s1, s2]` adopted the joiner and as `[s1, s2, joiner]` left it pending — 959 of 2000 sample rooms placed differently, with no compromised member and no selective speech. It healed as soon as a voucher gossiped again, so it was a window rather than a split; but the module documentation asserted that within one node "the order claims arrive in cannot change the result", and that was false, which is the part worth fixing. A claim is now *held* for a member the view has not met and counts for nothing until that member exists, since the fixpoint only ever scans the roster. The tombstone rule survives it: a reaped member still refuses held claims, so it cannot return pre-adopted. Distinct from C39 — there the nodes receive different content, here byte-identical content in a different order. +**Pass 9 saw a window and pass 10 priced the fix, which cost more than the window.** A claim naming a member the receiver had not yet met was dropped with the tuple carrying it, so retention — and therefore the ring, since `adopted` is derived from `verifiers` — depended on frame arrival order: the identical three frames as `[joiner, s1, s2]` adopted the joiner and as `[s1, s2, joiner]` left it pending, 959 of 2000 rooms placed differently. Holding the claim instead closed that, and opened something worse. The ids in a held claim are on no roster, so `reap_dead` — which strikes `verifiers` only for ids it reaps *off the roster* — can never reach them, and `rebuild_placement` never scans them, so they are pure retained state that nothing reclaims. One inbound 24.9 MB frame of attacker-chosen ids retained 376.5 MB, a 15x amplification of the wire, and 591 ms inside the single-threaded registry actor; at tungstenite's default 64 MiB cap that is 3.3 million claims, about 1 GB, per frame. It also made the tombstone refusal temporal — `reaped` is pruned after `TOMBSTONE_RETENTION_TICKS`, so past roughly five minutes a reaped id was neither on the roster nor tombstoned and banked claims again, letting a returning member be adopted pre-verified (measured: 980/2000 rooms). The window was the cheaper defect, and it is not even a loss: a verifier re-advertises what it verified on every round, so a claim that outruns its subject is re-sent within one gossip interval and the view converges then. Reverted to dropping, with that healing property pinned instead of the order-independence that was never true. The regression test written for the held-claim version was itself vacuous — it drove `merge_liveness`, whose `None` branch admits the unmet member from the same tuple, so it passed on the code it was written to catch; the replacement drives the seam that actually drops. + +`require_peer_identity` is per-node configuration that feeds `verifiers`, so a rolling flip of it places rooms differently for the duration of the rollout (measured: 1021/2000 rooms, plaintext-advertised members only). It is self-limiting — the flag is an env var, a flip is a restart, and a restart rebuilds `verifiers` empty — and the fix suggested by review (dropping the `!require_peer_identity` short-circuit) would stop a plaintext cluster adopting anyone at all. It stands as a cluster-wide protocol parameter that must be uniform, like the cluster secret, rather than a per-node preference. `require_peer_identity` is per-node configuration that feeds `verifiers`, so a rolling flip of it places rooms differently for the duration of the rollout (measured: 1021/2000 rooms, plaintext-advertised members only). It is self-limiting — the flag is an env var, a flip is a restart, and a restart rebuilds `verifiers` empty — and the fix suggested by review (dropping the `!require_peer_identity` short-circuit) would stop a plaintext cluster adopting anyone at all. It stands as a cluster-wide protocol parameter that must be uniform, like the cluster secret, rather than a per-node preference. diff --git a/KANBAN.md b/KANBAN.md index 720ba523..7ba4381e 100644 --- a/KANBAN.md +++ b/KANBAN.md @@ -30,7 +30,7 @@ Element + Map are one coupled unit (Map slots hold Elements; Element forwards li _Derived from code + git; a convenience view, not the source of truth._ -**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Pass 4 found three more, all in how an id is derived from configuration:** the bootstrap exception read a *de-duplicated set*, so a peer list of only this node's own address collapsed to one member and dropped the bar to a single vouch — the very state the empty-list refusal had been added to catch; `CRDTSYNC_NODE_ID` was the one door that did not canonicalize (a regression from the plain `trim()` it replaced), so a node written in a non-canonical spelling carried a **doppelgänger of itself** — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, dropping every follower-head report; and the port was not canonicalized, so `:9000`/`:09000`/`:009000` were three ring positions on one listener, which is the targeted write-stall pass 3 was supposed to have closed and which an operator reaches with no attacker by writing a leading zero. The bar now reads the list rather than the set, every door canonicalizes, the port is read as a number, and an address with no canonical form is refused where it is written. **Passes 5 and 6 found two more, both places where a rule was stated but not made structural:** the canonical form was checked to be a fixpoint over a list of spellings rather than by construction, and brackets — the IPv6 literal's syntax — were never required to *contain* an IPv6 literal, so `[10.0.0.1:9000]:9443` (an operator typo, no attacker) parsed as a host whose own text carries colons and yielded the id `10.0.0.1:9000:9443`: accepted at the config door, adopted from birth as a configured member, placed on every room HRW gave it, and dialable by nobody, so those rooms could reach no quorum — and written as `CRDTSYNC_NODE_ID` it has no canonical form at all, so peers drop it from gossip and refuse its `PeerAuth` and the node joins nothing while believing it has. And a verifier claim gated the *node* it was about but never the **sender** it was stored under, so a certified host opening a link per port banked an unbounded verifier entry per link against every member — keyed on ids no reap ever strikes, because reaping strikes members. Both bars are now the same one predicate on both sides. **Residual, pinned as a passing test and filed as C34:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. +**C25 — a member placed itself into any room's replica set by minting a node id, so C13's leadership and durability gates did not hold against a compromised member (crates/server) — DONE (#369).** C13 (#365) bound a peer link to a member and gated replication, leadership and durability on it. Every one of those gates asks the same question — is the sender in `replicas_for(room)` — and placement is HRW over the member set, a pure and publicly computable function, so **which rooms a node replicates follows from its node id**. Gossip's join path lets an unknown node introduce itself, and `add_members`/`merge_liveness` put it straight into the `Cluster` the ring is built from. So an admitted member ground an id HRW placed on the room it wanted, introduced itself under it, and was inside that room's replica set: it superseded the leader with a forged epoch, pushed ops in, and acked as one of five so majority-ack released a client `Accepted` for a write two nodes held. **The fix is that learning a member and placing rooms on it are two admissions.** The roster is what a node dials, probes and gossips about; the ring is built from the **adopted** members alone, and a gossip-learned member is *pending* — reachable, converging, advertised back, in no room's replica set and no room's quorum — until the cluster adopts it. **Adoption could not be a local check**: placement must be identical on every node or two nodes disagree about who leads a room permanently, and "verified" is per-node by construction. So the *evidence* is disseminated instead of the verdict — a node records only that it completed an identity-checked peer link to a member, and that claim rides the same anti-entropy as liveness, as one `verified` flag per `Message::Gossip` tuple, recorded **against the member the receiving link is bound to** and against nobody the payload names. A member is adopted once two already-adopted **trust units** have verified it. **Cold review overturned four things in the first cut, and each is the interesting part.** (1) *Adoption is derived, never accumulated.* Sticky adoption made the ring a function of history — a member adopted before its vouchers were reaped stays placed on the node that saw that order and is never placed on one that did not, permanently — so the adopted set is recomputed from `configured` + `verifiers` to a fixpoint on every change, and is a pure function of state. (2) *A verifier is a trust unit, and a trust unit is a host.* Counting node ids was the mint one level up: a certificate names a host (C13) and a host mints unlimited ids, so a member holding two of them raised the whole bar alone. The bar counts distinct verifier **hosts** and excludes the candidate's own — a member vouching for a sibling on its host vouches for itself. (3) *An inbound link verifies nothing*, however well its certificate names the member: the member chooses when to dial in, so the vouch is one it caused, and it could dial in under each ground id in turn. Verification is this node's own dial; an indirect (ping-req) confirmation is liveness only. (4) *A member is dialed at its own node id.* The recorded advertise address was first-write-wins over a field any peer may set, and the ring turned on it, so whoever advertised a joiner first decided what every later dial verified — two nodes that saw it in a different order placed 27 of 64 sample rooms differently, forever. A node id *is* an advertise address, so the second name is dropped and the dial address is a function of the id alone; that also closes the reply half's freedom at the root rather than by a filter. **The bar is a constant**, not a fraction of the cluster (clamping it made the bar node-local and let a reap *lower* it and retroactively adopt), with one exception keyed on configuration: a node configured with no peers has no cluster to be outvoted by and its ring is what it has itself reached — the single-node deployment. Configured members are adopted from birth. A majority was rejected (an honest node dials every member it knows, so the marginal verifier past the second only delays a joiner) and so was relaying another node's verifications (with no signatures on the wire, "A and B verified X" is free to write). On the reply half a claim counts only where the dial established the sender, so under `CRDTSYNC_CLUSTER_REQUIRE_PEER_IDENTITY=1` a plaintext member's vouches are dropped — sound outbound, and exactly what C13's pass 3 rejected inbound. **Cold review ran twice and pass 2 overturned two more, both about what a trust unit is:** the unit was the host as *raw text* while the certificate binding compares it semantically, so `evil.example` beside `evil.example.` (or an IPv6 literal beside its expanded form) was one certificate and two vouchers — one machine holding two units adopts any id **anywhere**, which is not the accepted residual; and the bootstrap exception read `configured.len() <= 1` off a set that shrinks, so a node given one seed fell to the single-node rule the moment that seed was reaped and began placing on one vouch while its peers held the constant. **Pass 3 found two more, one root: an advertise address had many spellings and nothing reduced them.** The dial URL was built from the authority verbatim while the host was read as everything before the first `:`, so `wss://a.example:1@b.example:9000` *read* as `a.example` and *dialed* `b.example` — a member certified for `a.example` alone ground such an id onto any room, bound its own certificate to it, and let honest nodes verify it by dialing the honest node it embedded (a full bypass under mTLS, not the bounded own-host residual). And ids were never canonicalized, so `x:9000` / `ws://x:9000` / `WS://x:9000` / `x:9000/anything` were four members one honest endpoint answers for: all truthfully verified, all adopted, none of them ever speaks, so a room whose replica set filled with them waited on acks that never came — targeted, since HRW is public. An advertise address is now an authority and nothing else, in one canonical spelling, enforced at config, at the gossip tuple and at a link's claim. A present-but-empty `CRDTSYNC_CLUSTER_PEERS` (a clustered node running the single-node bar of one) is refused at startup. **Pass 4 found three more, all in how an id is derived from configuration:** the bootstrap exception read a *de-duplicated set*, so a peer list of only this node's own address collapsed to one member and dropped the bar to a single vouch — the very state the empty-list refusal had been added to catch; `CRDTSYNC_NODE_ID` was the one door that did not canonicalize (a regression from the plain `trim()` it replaced), so a node written in a non-canonical spelling carried a **doppelgänger of itself** — adopted on two honest vouchers, placed on 468 of 2000 sample rooms it never answers for, unable to refute a suspicion of itself, dropping every follower-head report; and the port was not canonicalized, so `:9000`/`:09000`/`:009000` were three ring positions on one listener, which is the targeted write-stall pass 3 was supposed to have closed and which an operator reaches with no attacker by writing a leading zero. The bar now reads the list rather than the set, every door canonicalizes, the port is read as a number, and an address with no canonical form is refused where it is written. **Pass 10 caught a fix that was worse than the defect it closed, which is the reason the loop runs to a clean pass rather than to a fixed one.** Pass 9 measured a real window — a claim naming a member the receiver had not yet met was dropped with its tuple, so retention, and therefore the ring, depended on frame arrival order (959/2000 rooms). Holding the claim instead closed it and opened an unbounded map: those ids are on no roster, so `reap_dead` can never strike them and `rebuild_placement` never scans them, making them state nothing reclaims. One inbound 24.9 MB frame retained **376.5 MB — 15x the wire** — and stalled the single-threaded registry actor 591 ms; at the default 64 MiB frame cap that is 3.3 million claims per frame. It also made the tombstone refusal expire with `TOMBSTONE_RETENTION_TICKS`, letting a returned member be adopted pre-verified (980/2000 rooms). The window was the cheaper defect and is not even a loss, since a verifier re-advertises what it verified every round: a claim that outruns its subject is re-sent within one gossip interval. Reverted, with the healing property pinned instead. The regression test written for the held-claim version was itself vacuous — it drove `merge_liveness`, which admits the unmet member from the same tuple, so it passed on the very commit it was written to catch. **Passes 5 and 6 found two more, both places where a rule was stated but not made structural:** the canonical form was checked to be a fixpoint over a list of spellings rather than by construction, and brackets — the IPv6 literal's syntax — were never required to *contain* an IPv6 literal, so `[10.0.0.1:9000]:9443` (an operator typo, no attacker) parsed as a host whose own text carries colons and yielded the id `10.0.0.1:9000:9443`: accepted at the config door, adopted from birth as a configured member, placed on every room HRW gave it, and dialable by nobody, so those rooms could reach no quorum — and written as `CRDTSYNC_NODE_ID` it has no canonical form at all, so peers drop it from gossip and refuse its `PeerAuth` and the node joins nothing while believing it has. And a verifier claim gated the *node* it was about but never the **sender** it was stored under, so a certified host opening a link per port banked an unbounded verifier entry per link against every member — keyed on ids no reap ever strikes, because reaping strikes members. Both bars are now the same one predicate on both sides. **Residual, pinned as a passing test and filed as C34:** a member that owns a host owns every id under it, answers at each ground id, and honest nodes verify one *truthfully* — so adoption bounds the mint to the member's own host and to reachable ids, and cannot bound it further; closing it needs the ring to weigh trust units rather than ids. With no certificates the bar is not a bar at all (a secret-holder binds a link to any id, C13's own residual) and what remains is reachability. Suites: `crates/server/tests/adoption.rs` (28); `gossip.rs` convergence tests now drive rounds until the ring converges as well as the roster; C13's residual test is removed from `peer_identity.rs`. Wire: `Message::Gossip`'s tuple gains a trailing flag byte (`MemberAdvert`). See DECISIONS (2026-07-28). → *Cluster*. **C26 — the SDK apply seams reported a permanent refusal as silence (crates/ffi + crates/wasm + sdks) — DONE (#370).** `crdtsync_doc_apply` and `WasmDocument::apply` folded a decoded batch as `ops.iter().filter(|op| doc.apply(op)).count()`, so an app learned how many ops applied and nothing else — while `Document::apply` answers `false` for three unrelated reasons of which only one is permanent. A buffered op (which a later arrival commits) and an op no replica will ever hold both read as "not counted", and the two want opposite responses: wait, versus surface a client bug. C19 (#367) made the distinction answerable (`Op::is_admissible`, public and pure) without plumbing it; this plumbs it. **Every binding now reports both counts, classified by that same predicate**, so this seam and the server's ingress cannot disagree about *which* refusals are permanent: the C ABI takes a nullable `uint32_t *out_refused` (header regenerated), wasm's `apply` returns `{ applied, refused }`, JS an `ApplyOutcome` object, Python an `ApplyOutcome` NamedTuple, Go a second return value. **The judgement is shared with the server's ingress; the batch policy deliberately is not** — this fold applies the admissible remainder where `handle_ops` refuses the whole frame, because only the server has an ack frontier taken as a max over the batch. **A refused-op *list* was considered and rejected** (a `takeRefused` drain mirroring `takeRejected`): per-handle state with its own lifetime rules, a semantically muddy stub on every client-backed replica whose refusals arrive as the server's `OpsRejected` instead, and a refusal is a fact about *this* batch. `crates/core/src/client.rs` is untouched — advancing `last_seen_seq` past a refused op is correct, since no replica will hold it. **One caveat is stated in all five bindings and pinned by a test rather than left to prose:** `applied` counts ops as they arrive, so an op that a later op in the *same* batch releases from the buffer is not among them; making that truthful needs a core seam this unit does not need. Suites: a refused-vs-buffered discrimination test per binding (FFI, wasm, JS, Python, Go), each also pinning the mixed batch (2 applied, 1 refused), a duplicate replay as not-refused, and a malformed batch as neither — with the C ABI's count proven *written* as zero there, so a caller reusing one variable across batches cannot read the last batch's refusals as this one's. The three SDK tests read the stamp's client id back before forging it, so a codec reorder fails by name. **Three local cold-review passes**, each caught something the last did not: the uninitialised-out-param reuse and an unreachable refusal cause in the doc comments; then that the count was still written from inside the panic guard, contradicting its own contract, and that the `applied` undercount had prose but no test; then that the seam was described as *relay*, which is false — `handle_ops`' `is_admissible` gate is schema-independent, so a relay-tier connection answers `MalformedOp` like any other server, and the seam with nothing upstream is offline, direct exchange, or a byte pipe the app carries itself. **Mutation-swept (`--no-fail-fast`) against an isolated clone of the committed tree**, named-test kills only: FFI 6 mutants (refused always zero 1, unfiltered count 1, `len - applied` 1, predicate negated 1, counts swapped 2, out-param never written 2); wasm 3 (1, 1, 6); and the always-zero lib kills 2 in the Python suite and 2 in the Go suite, so the SDK layers are not blind to it. The unmutated tree fails 0. Miri-clean. See DECISIONS (2026-07-28). → *SDK / Ops*. diff --git a/crates/server/src/membership.rs b/crates/server/src/membership.rs index e424ebad..e96b119a 100644 --- a/crates/server/src/membership.rs +++ b/crates/server/src/membership.rs @@ -278,10 +278,12 @@ pub struct Membership { /// first-hand claim by the node named — a member can vouch for itself only, and /// never *for* itself ([`note_verified`](Self::note_verified) refuses that). /// Grow-only per member, so it merges by union: within one node the order claims - /// arrive in cannot change the result — which is why a claim about a member this - /// view has not met is *held* rather than dropped (see - /// [`claim_may_be_retained`](Self::claim_may_be_retained)), since dropping it made - /// the union, and so the ring, a function of arrival order. Between nodes it converges only as far as + /// arrive in cannot change the *set*. It can change when the set is complete — a + /// claim naming a member this view has not met is dropped with the tuple carrying + /// it, because retaining claims about ids that are on no roster is unbounded and + /// no reap could ever strike them. The claim is late rather than lost: a verifier + /// re-advertises what it verified every round, so the next round this node + /// initiates carries the member and the claim together. Between nodes it converges only as far as /// the claims travel, and a claim travels only from the member that made it — /// nothing here relays another node's word, so a member that sends different /// claims to different peers leaves them holding different evidence permanently @@ -664,29 +666,6 @@ impl Membership { } } - /// Record `sender`'s claims about members this view has not learned, for the seam - /// that admits a tuple's evidence without admitting the member it names. Same bars - /// as the claims carried on an admitted tuple: the sender is on the roster, and the - /// member it names is neither this node nor tombstoned. - pub fn note_claims(&mut self, sender: &NodeId, nodes: impl IntoIterator) { - if !self.can_be_verified(sender) { - return; - } - let mut rebuilt = false; - for node in nodes { - if self.claim_may_be_retained(&node) { - rebuilt |= self - .verifiers - .entry(node) - .or_default() - .insert(sender.clone()); - } - } - if rebuilt { - self.rebuild_placement(); - } - } - /// Whether a verification of `node` is admissible evidence at all — the bar every /// verifier claim clears, this node's own and a peer's alike. See /// [`note_verified`](Self::note_verified) for what each clause refuses. @@ -694,29 +673,6 @@ impl Membership { !self.is_self(node) && self.addrs.contains_key(node) } - /// Whether a *peer's* claim about `node` may be retained. Weaker than - /// [`can_be_verified`](Self::can_be_verified) in one place, deliberately: the claim - /// does not require `node` to be on the roster **yet**. - /// - /// The frame that carries a claim about a member and the frame that introduces - /// that member are two frames, and which arrives first is ordinary network - /// ordering. Requiring the member first made retention a function of arrival order, - /// and `adopted` is derived from `verifiers`, so the *ring* became a function of - /// arrival order one level down — two nodes given the identical frames in different - /// orders placed different rooms until the vouchers happened to gossip again. A - /// claim about a member this view has not met is held rather than dropped; it - /// counts for nothing until that member exists, because [`rebuild_placement`] only - /// ever scans the roster. - /// - /// A tombstoned node is still refused. It is not a member of this view, and banking - /// claims against it would let it return pre-adopted on evidence collected while - /// nobody could reach it. - /// - /// [`rebuild_placement`]: Self::rebuild_placement - fn claim_may_be_retained(&self, node: &NodeId) -> bool { - !self.is_self(node) && !self.reaped.contains_key(node) - } - /// How many **trust units** have vouched for `node`: the distinct hosts of the /// adopted members that verified it, its own host excluded. /// @@ -1012,7 +968,7 @@ impl Membership { // roster is what keeps it bounded at all. if self.can_be_verified(sender) { for node in claimed { - if self.claim_may_be_retained(&node) { + if self.can_be_verified(&node) { rebuilt |= self .verifiers .entry(node) diff --git a/crates/server/src/registry.rs b/crates/server/src/registry.rs index a3b89d2d..3c409b3d 100644 --- a/crates/server/src/registry.rs +++ b/crates/server/src/registry.rs @@ -977,29 +977,24 @@ impl Registry { return false; }; // An inbound frame introduces only its own sender: admitting a *third* member - // from it would be a join with no dial behind it. But refusing to admit the - // member is not a reason to discard the *claim* the same tuple carries — the - // frame naming a member and the frame introducing it are two frames, and which - // arrives first is ordinary network ordering. Dropping the claim made the ring - // a function of that order: the same three frames delivered as - // `[joiner, s1, s2]` adopted the joiner and as `[s1, s2, joiner]` left it - // pending, placing 959 of 2000 sample rooms differently until a voucher - // happened to gossip again. The claim is carried over separately, where it is - // held against a member this view does not have yet and counts for nothing - // until it does. - let (admitted, deferred): (Vec<_>, Vec<_>) = - members.into_iter().partition(|(node, addr, ..)| { + // from it would be a join with no dial behind it. A tuple that fails that test + // is dropped whole, claim included. Holding the claim instead — against a + // member this view has not met — sounds strictly better, and is not: those ids + // are on no roster, so no reap can ever strike them, and one frame of + // attacker-chosen ids banks entries that nothing reclaims. Measured, one 24.9 MB + // frame retained 376.5 MB, a 15x amplification of the wire. Dropping costs a + // *bounded* window instead: the maker re-advertises `verified` for every member + // it has verified on every round, so a claim that arrives before its subject is + // re-sent within one gossip interval and the view converges then. + let members = members + .into_iter() + .filter(|(node, addr, ..)| { // `is_member` holds for self too — a node is a member of its own view // from construction and is never reaped out of it. let node = NodeId::from(node.clone()); membership.is_member(&node) || (&node == sender && addr == node.as_bytes()) - }); - let deferred: Vec = deferred - .into_iter() - .filter(|(_, _, _, _, verified)| *verified) - .filter_map(|(node, ..)| NodeId::canonical(&node)) + }) .collect(); - let members = admitted; // Whether a claim is attributable is asked the same way on both halves of a // round, because `verifiers` has to converge and this is what decides what // enters it. Reading the link here (bound to `sender`, C13) while the reply @@ -1011,11 +1006,6 @@ impl Registry { // configuration and the member's own address, so every node computes it alike. let attributable = self.dial_establishes_identity(sender); self.merge_gossip_attributed(sender, members, attributable); - if attributable && !deferred.is_empty() { - if let Some(membership) = &mut self.membership { - membership.note_claims(sender, deferred); - } - } let reply = crate::gossip::gossip_frame(&self.known_liveness()); if let Some(conn) = self.conns.get_mut(&id) { conn.outbox.push(reply); diff --git a/crates/server/tests/adoption.rs b/crates/server/tests/adoption.rs index f44b9f7d..b80a9c90 100644 --- a/crates/server/tests/adoption.rs +++ b/crates/server/tests/adoption.rs @@ -1758,84 +1758,34 @@ fn a_compromised_member_that_speaks_selectively_splits_the_ring() { } #[test] -fn the_same_frames_in_any_order_build_the_same_ring() { - // `adopted` is derived from `verifiers`, so if retention depends on the order - // frames arrive in, the ring does too — one level down, and invisibly. The frame - // that vouches for a member and the frame that introduces it are two frames, and - // which lands first is ordinary network ordering, not evidence. +fn a_claim_that_outruns_its_subject_is_re_sent_within_one_round() { + // An inbound frame introduces only its own sender, so a claim naming a member this + // view has not met is dropped with the tuple that carried it. Holding it instead + // would be unbounded — those ids are on no roster, so no reap could ever strike + // them. What makes dropping safe is that the claim is not lost, only late: a + // verifier re-advertises what it verified on every round, so the very next round + // this node initiates carries the member and the claim about it together. let joiner = NodeId::from("10.9.9.9:9000"); - let vouchers = ["10.0.0.1:9000", "10.0.0.2:9000"]; - - let introduce = |m: &mut Membership| m.add_member(joiner.clone()); - let vouch_from = |m: &mut Membership, who: &str| vouch(m, who, &joiner); - - // Member first, then the vouches. - let mut forward = membership_for(SELF_ADDR); - introduce(&mut forward); - for v in vouchers { - vouch_from(&mut forward, v); - } - - // The vouches first, then the member they are about. - let mut backward = membership_for(SELF_ADDR); - for v in vouchers { - vouch_from(&mut backward, v); - } - introduce(&mut backward); - - assert!(forward.is_adopted(&joiner), "vouched then placed"); + let mut voucher = membership_for("10.0.0.1:9000"); + let mut late = membership_for(SELF_ADDR); + voucher.add_member(joiner.clone()); + voucher.note_verified(&joiner); + assert!(voucher.has_verified(voucher.self_id(), &joiner)); + + // Being dialed teaches `late` nothing about the joiner, so the claim rides a frame + // whose subject it cannot admit, and goes with it. + crdtsync_server::gossip::exchange(&mut voucher, &mut late); assert!( - backward.is_adopted(&joiner), - "a claim about a member this view has not met yet is held, not dropped", - ); - assert_eq!( - forward.adopted_members(), - backward.adopted_members(), - "the ring is a function of the evidence, not of its arrival order", + !late.is_member(&joiner), + "an inbound frame introduces only its own sender", ); - let differing = (0..512u32) - .map(|i| format!("room-{i}")) - .filter(|r| forward.replicas_for(r.as_bytes()) != backward.replicas_for(r.as_bytes())) - .count(); - assert_eq!(differing, 0, "and so is every room's replica set"); -} - -#[test] -fn a_claim_held_for_an_unmet_member_still_respects_the_tombstone() { - // Holding a claim about a member this view has not met must not become a way to - // bank evidence against one it has *reaped*: that member is not absent, it is - // refused, and it must be re-verified after it returns. - let mut m = membership_for(SELF_ADDR); - let departed = NodeId::from("10.9.9.9:9000"); - m.add_member(departed.clone()); - for _ in 0..crdtsync_server::membership::DEAD_AFTER_FAILURES { - m.note_gossip_unreachable(&departed); - } - for _ in 0..crdtsync_server::membership::REAP_AFTER_DEAD_TICKS { - m.reap_dead(); - } - assert!(!m.is_member(&departed)); + assert!(!late.has_verified(voucher.self_id(), &joiner)); - m.note_claims( - &NodeId::from("10.0.0.1:9000"), - [departed.clone(), departed.clone()], - ); - m.note_claims(&NodeId::from("10.0.0.2:9000"), [departed.clone()]); + // One round `late` initiates, and the reply carries both. + crdtsync_server::gossip::exchange(&mut late, &mut voucher); + assert!(late.is_member(&joiner), "the member arrived"); assert!( - !m.has_verified(&NodeId::from("10.0.0.1:9000"), &departed), - "a tombstone refuses a held claim exactly as it refuses a carried one", - ); - - m.merge_liveness( - &NodeId::from("10.0.0.1:9000"), - [( - departed.clone(), - departed.as_bytes().to_vec(), - 1, - MemberState::Alive, - false, - )], + late.has_verified(voucher.self_id(), &joiner), + "and the claim about it arrived in the same frame, one round later", ); - assert!(m.is_member(&departed), "it rejoined"); - assert!(!m.is_adopted(&departed), "and it rejoined pending"); }