From eae63a349987d5d7f5549bc520ce5c8e9c8828da Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Tue, 11 Aug 2026 07:13:56 +1000 Subject: [PATCH 1/4] Identify add-to batches by batch message hash, not address set Batch identity is the batch message hash, which covers time (SPEC SS11/SS12): re-issuing the same add-to addresses at a new time is a distinct batch (a new sibling branch), not a duplicate. Previously addToBatchRecorded keyed on the recipient-address set, so a re-issued batch was rejected code 10. The receiving host computes the batch hash at header exchange from the add-to header plus the stored parent's payload, stores it on msg_add_to_batch.sha256, and duplicate-checks against it. msg_add_to uniqueness is relaxed from (msg_id, addr) to (batch_id, addr) so a later batch may re-add an address an earlier batch added. Co-Authored-By: Claude Fable 5 --- cmd/fmsgd/host.go | 24 ++++++++++----- cmd/fmsgd/store.go | 74 +++++++++++++++++++++++++--------------------- dd.sql | 17 +++++++++-- 3 files changed, 71 insertions(+), 44 deletions(-) diff --git a/cmd/fmsgd/host.go b/cmd/fmsgd/host.go index 63825fc..b7f62eb 100644 --- a/cmd/fmsgd/host.go +++ b/cmd/fmsgd/host.go @@ -607,8 +607,24 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) { return h, fmt.Errorf("add-to: time travel detected (parent time %f, current %f)", parentMsg.Timestamp, h.Timestamp) } + // Batch identity is the batch message hash — this add-to header combined + // with the stored parent's payload (SPEC §11) — so map the parent's data + // in and compute it before the duplicate check. + h.Filepath = parentMsg.Filepath + for i := range h.Attachments { + if i < len(parentMsg.Attachments) { + h.Attachments[i].Filepath = parentMsg.Attachments[i].Filepath + } + } + batchHash, err := h.GetMessageHash() + if err != nil { + return h, err + } + // A batch this host already recorded is a duplicate (SPEC §10.4 step 1). - recorded, err := addToBatchRecorded(parentID, h.AddTo) + // The same addresses re-issued at a new time hash differently and are a + // distinct batch — a new sibling branch — not a duplicate (SPEC §12). + recorded, err := addToBatchRecorded(parentID, batchHash) if err != nil { return h, err } @@ -624,12 +640,6 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) { return h, nil } - h.Filepath = parentMsg.Filepath - for i := range h.Attachments { - if i < len(parentMsg.Attachments) { - h.Attachments[i].Filepath = parentMsg.Attachments[i].Filepath - } - } h.InitialResponseCode = AcceptCodeAddTo return h, nil } diff --git a/cmd/fmsgd/store.go b/cmd/fmsgd/store.go index 583cdc5..d636b37 100644 --- a/cmd/fmsgd/store.go +++ b/cmd/fmsgd/store.go @@ -286,14 +286,13 @@ func existingMsgIDForAddTo(tx *sql.Tx, msg *FMsgHeader, msgHash []byte) (int64, return id, err } -// addToBatchRecorded reports whether an incoming add-to batch carries nothing -// this host has not already recorded against stored message msgID: every -// address is unique per message across batches (msg_add_to unique (msg_id, -// addr)), so when every address in the incoming batch is already attached the -// delivery is a re-send of a recorded batch and is a duplicate (code 10, -// SPEC §10.4 step 1). -func addToBatchRecorded(msgID int64, addTo []FMsgAddress) (bool, error) { - if len(addTo) == 0 { +// addToBatchRecorded reports whether this host has already recorded an add-to +// batch with this batch message hash against stored message msgID. Batch +// identity IS the batch message hash, which covers time (SPEC §11): the same +// addresses re-issued at a new time hash differently and are a distinct +// batch, not a duplicate (SPEC §12). +func addToBatchRecorded(msgID int64, batchHash []byte) (bool, error) { + if len(batchHash) == 0 { return false, nil } db, err := sql.Open("postgres", "") @@ -302,29 +301,22 @@ func addToBatchRecorded(msgID int64, addTo []FMsgAddress) (bool, error) { } defer db.Close() - for i := range addTo { - var exists bool - err = db.QueryRow(`SELECT EXISTS ( - SELECT 1 FROM msg_add_to WHERE msg_id = $1 AND lower(addr) = $2 - )`, msgID, strings.ToLower(addTo[i].ToString())).Scan(&exists) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - } - return true, nil -} - -// insertAddToBatch records one add-to delivery as a batch (its sender and the -// time this host recorded it) and returns the new batch id. Recipients carried -// by the delivery are linked to this batch so readers can reconstruct who added -// which recipients and when (SPEC §12). -func insertAddToBatch(tx *sql.Tx, msgID int64, addToFrom string, now float64) (int64, error) { + var exists bool + err = db.QueryRow(`SELECT EXISTS ( + SELECT 1 FROM msg_add_to_batch WHERE msg_id = $1 AND sha256 = $2 + )`, msgID, batchHash).Scan(&exists) + return exists, err +} + +// insertAddToBatch records one add-to delivery as a batch (its sender, the +// time this host recorded it, and its identifying batch message hash, SPEC +// §11) and returns the new batch id. Recipients carried by the delivery are +// linked to this batch so readers can reconstruct who added which recipients +// and when (SPEC §12). +func insertAddToBatch(tx *sql.Tx, msgID int64, addToFrom string, now float64, batchHash []byte) (int64, error) { var batchID int64 - err := tx.QueryRow(`insert into msg_add_to_batch (msg_id, add_to_from, time_added) -values ($1, $2, $3) returning id`, msgID, addToFrom, now).Scan(&batchID) + err := tx.QueryRow(`insert into msg_add_to_batch (msg_id, add_to_from, time_added, sha256) +values ($1, $2, $3, $4) returning id`, msgID, addToFrom, now, batchHash).Scan(&batchID) return batchID, err } @@ -365,7 +357,14 @@ on conflict (msg_id, addr) do nothing`, msgID, addr.ToString(), delivered, code) if msg.AddToFrom != nil { addToFrom = msg.AddToFrom.ToString() } - batchID, err := insertAddToBatch(tx, msgID, addToFrom, now) + // Cached since the header exchange computed it against the stored + // parent's payload (handleAddToPath); it is the batch's identity (SPEC + // §11) and what duplicate detection compares against. + batchHash, err := msg.GetMessageHash() + if err != nil { + return fmt.Errorf("compute add-to batch hash: %w", err) + } + batchID, err := insertAddToBatch(tx, msgID, addToFrom, now, batchHash) if err != nil { return err } @@ -380,7 +379,7 @@ on conflict (msg_id, addr) do nothing`, msgID, addr.ToString(), delivered, code) } if _, err := tx.Exec(`insert into msg_add_to (msg_id, batch_id, addr, time_delivered, response_code) values ($1, $2, $3, $4, $5) -on conflict (msg_id, addr) do nothing`, msgID, batchID, addr.ToString(), delivered, code); err != nil { +on conflict (batch_id, addr) do nothing`, msgID, batchID, addr.ToString(), delivered, code); err != nil { return err } } @@ -494,7 +493,13 @@ values ($1, $2, $3, $4)`) if msg.AddToFrom != nil { addToFrom = msg.AddToFrom.ToString() } - batchID, err := insertAddToBatch(tx, msgID, addToFrom, now) + // Cached from download verification: the wire hash of an add-to + // message is its batch hash, the batch's identity (SPEC §11). + batchHash, err := msg.GetMessageHash() + if err != nil { + return fmt.Errorf("compute add-to batch hash: %w", err) + } + batchID, err := insertAddToBatch(tx, msgID, addToFrom, now, batchHash) if err != nil { return err } @@ -624,7 +629,8 @@ returning id`, if msg.AddToFrom != nil { addToFrom = msg.AddToFrom.ToString() } - batchID, err := insertAddToBatch(tx, msgID, addToFrom, timeutil.TimestampNow().Float64()) + // No stored parent payload here, so the batch hash cannot be computed. + batchID, err := insertAddToBatch(tx, msgID, addToFrom, timeutil.TimestampNow().Float64(), nil) if err != nil { return err } diff --git a/dd.sql b/dd.sql index bae9deb..762adba 100644 --- a/dd.sql +++ b/dd.sql @@ -55,13 +55,19 @@ create index if not exists msg_to_lower_idx on msg_to ((lower(addr))); -- Each add-to delivery for a shared message is one batch: a single sender -- (add_to_from) added a set of recipients at a point in time. Storing batches -- separately lets readers reconstruct who added which recipients and when, --- which a single flat recipient list cannot preserve (SPEC §12). +-- which a single flat recipient list cannot preserve (SPEC §12). A batch's +-- identity is its message hash (sha256), which covers the batch's time: the +-- same addresses re-issued at a new time are a distinct batch, not a +-- duplicate (SPEC §11/§12). sha256 is null for rows recorded before this +-- column existed and for locally originated batches not yet hashed. create table if not exists msg_add_to_batch ( id bigserial primary key, msg_id bigint not null references msg (id), add_to_from varchar(255) not null, -- sender that added this batch's recipients - time_added double precision not null -- when this host recorded the batch + time_added double precision not null, -- when this host recorded the batch + sha256 bytea -- batch message hash: the batch's identity (SPEC §11) ); +alter table msg_add_to_batch add column if not exists sha256 bytea; create index if not exists msg_add_to_batch_msg_id_idx on msg_add_to_batch (msg_id); create table if not exists msg_add_to ( @@ -74,8 +80,13 @@ create table if not exists msg_add_to ( time_read double precision, -- time recipient read the message; null if unread response_code smallint, -- when sending, response code of last delivery attempt if failed; when receiving, the per-recipient code this host responded, or a negative local sentinel (-1 attempt got no response, retryable; -2 recorded from an exchange, another host's delivery) attempt_count int not null default 0, -- number of failed delivery attempts; used for exponential back-off - unique (msg_id, addr) + unique (batch_id, addr) ); +-- An address is unique within a batch, not across batches: distinct batches +-- may re-add the same address (each batch is its own sibling branch, SPEC +-- §12). Migrate existing databases off the old per-message constraint. +alter table msg_add_to drop constraint if exists msg_add_to_msg_id_addr_key; +create unique index if not exists msg_add_to_batch_id_addr_key on msg_add_to (batch_id, addr); create index if not exists msg_add_to_lower_idx on msg_add_to ((lower(addr))); create index if not exists msg_add_to_batch_id_idx on msg_add_to (batch_id); From 4adfa11f6f0472ec71cd6a16b33682a66de99a53 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Tue, 11 Aug 2026 07:37:41 +1000 Subject: [PATCH 2/4] Resolve reply parents referencing add-to batch hashes A reply to an add-to batch message carries the batch hash as pid. Per SPEC SS11 the batch message is reconstructible from what the host already holds - the stored shared message plus the batch's sender, recipients and wire time - even though the batch's data was never downloaded again. Reply validation now falls back to reconstructing the batch wire form when pid matches msg_add_to_batch.sha256, and the relational parent link and challenge-participation thread walk resolve batch hashes to the shared message row. For that reconstruction to be faithful the receive path now records the batch's wire time in time_added, matching the outbound path where time_added already serves as the batch header's timestamp. Co-Authored-By: Claude Fable 5 --- cmd/fmsgd/host.go | 21 ++++++++------ cmd/fmsgd/store.go | 72 +++++++++++++++++++++++++++++++++++++++++++--- dd.sql | 2 +- 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/cmd/fmsgd/host.go b/cmd/fmsgd/host.go index b7f62eb..aaf288c 100644 --- a/cmd/fmsgd/host.go +++ b/cmd/fmsgd/host.go @@ -583,6 +583,9 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) { addToHasOurDomain := hasDomainRecipient(h.AddTo, Domain) hasLocalRecipient := addToHasOurDomain || hasDomainRecipient(h.To, Domain) + // Deliberately resolves canonical message hashes only: batches do not + // chain, so an add-to whose pid is another batch's hash must not resolve + // (SPEC §12). parentID, err := lookupMsgIdByHash(h.Pid) if err != nil { return h, err @@ -653,14 +656,16 @@ func validatePidReplyPath(c net.Conn, h *FMsgHeader) error { if err != nil { return err } - if parentID == 0 { - if err := sendCode(c, RejectCodeParentNotFound); err != nil { - return err - } - return fmt.Errorf("pid reply: parent not found for pid %s", hex.EncodeToString(h.Pid)) - } - parentMsg, err := getMsgByID(parentID) + var parentMsg *FMsgHeader + if parentID != 0 { + parentMsg, err = getMsgByID(parentID) + } else { + // A reply may reference an add-to batch message via pid (SPEC §12); + // its wire form is reconstructed from the stored shared message and + // batch fields (SPEC §11). + parentMsg, err = getMsgByBatchHash(h.Pid) + } if err != nil { return err } @@ -668,7 +673,7 @@ func validatePidReplyPath(c net.Conn, h *FMsgHeader) error { if err := sendCode(c, RejectCodeParentNotFound); err != nil { return err } - return fmt.Errorf("pid reply: parent message not found by ID %d", parentID) + return fmt.Errorf("pid reply: parent not found for pid %s", hex.EncodeToString(h.Pid)) } if !isMessageRetrievable(parentMsg) { if err := sendCode(c, RejectCodeParentNotFound); err != nil { diff --git a/cmd/fmsgd/store.go b/cmd/fmsgd/store.go index d636b37..1c0311b 100644 --- a/cmd/fmsgd/store.go +++ b/cmd/fmsgd/store.go @@ -86,6 +86,7 @@ func threadHasFromDomain(hash []byte, domain string) (bool, error) { SELECT id, from_addr, pid, 1 AS depth, ARRAY[id] AS seen FROM msg WHERE sha256 = $1 + OR id IN (SELECT msg_id FROM msg_add_to_batch WHERE sha256 = $1) UNION ALL SELECT m.id, m.from_addr, m.pid, t.depth + 1, t.seen || m.id FROM msg m @@ -155,8 +156,15 @@ type txParentLinkStore struct { } func (s txParentLinkStore) lookupParentID(parentHash []byte) (int64, error) { + // A reply's pid may reference a message's canonical hash or one of its + // add-to batch hashes (SPEC §12); either way the relational parent is the + // shared message row. var id int64 - err := s.tx.QueryRow("SELECT id FROM msg WHERE sha256 = $1", parentHash).Scan(&id) + err := s.tx.QueryRow(` + SELECT id FROM msg WHERE sha256 = $1 + UNION ALL + SELECT msg_id FROM msg_add_to_batch WHERE sha256 = $1 + LIMIT 1`, parentHash).Scan(&id) if err == sql.ErrNoRows { return 0, nil } @@ -269,6 +277,57 @@ func getMsgByID(msgID int64) (*FMsgHeader, error) { return h, nil } +// getMsgByBatchHash reconstructs the wire form of an add-to batch message +// identified by its batch hash, or nil when no such batch is recorded. A reply +// may reference a batch via pid, and per SPEC §11 the batch message is +// reconstructible from what the host already holds — the stored shared message +// plus the batch's sender, recipients and wire time — even though the batch's +// data was never downloaded again. +func getMsgByBatchHash(batchHash []byte) (*FMsgHeader, error) { + if len(batchHash) == 0 { + return nil, nil + } + db, err := sql.Open("postgres", "") + if err != nil { + return nil, err + } + defer db.Close() + + tx, err := db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + var msgID, batchID int64 + err = tx.QueryRow(`SELECT msg_id, id FROM msg_add_to_batch WHERE sha256 = $1`, batchHash).Scan(&msgID, &batchID) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + + m, err := loadMsgFields(tx, msgID) + if err != nil { + return nil, err + } + batches, err := loadAddToBatches(tx, msgID) + if err != nil { + return nil, err + } + for i := range batches { + if batches[i].ID == batchID { + sharedHash, err := m.sharedHash() + if err != nil { + return nil, err + } + return m.addToHeader(batches[i], sharedHash), nil + } + } + return nil, fmt.Errorf("add-to batch %d missing for msg %d", batchID, msgID) +} + // existingMsgIDForAddTo returns the id of an already-stored message row whose // canonical sha256 matches msgHash, for an add-to delivery. It returns 0 when // the message is not an add-to message or no such row exists, so the caller @@ -364,7 +423,10 @@ on conflict (msg_id, addr) do nothing`, msgID, addr.ToString(), delivered, code) if err != nil { return fmt.Errorf("compute add-to batch hash: %w", err) } - batchID, err := insertAddToBatch(tx, msgID, addToFrom, now, batchHash) + // The batch's WIRE time is stored, not the local record time: the batch + // message must be reconstructible exactly as transmitted so its hash can + // be recomputed and replies referencing it verified (SPEC §11). + batchID, err := insertAddToBatch(tx, msgID, addToFrom, msg.Timestamp, batchHash) if err != nil { return err } @@ -494,12 +556,14 @@ values ($1, $2, $3, $4)`) addToFrom = msg.AddToFrom.ToString() } // Cached from download verification: the wire hash of an add-to - // message is its batch hash, the batch's identity (SPEC §11). + // message is its batch hash, the batch's identity (SPEC §11). The + // batch's wire time is stored so the batch message stays + // reconstructible exactly as transmitted. batchHash, err := msg.GetMessageHash() if err != nil { return fmt.Errorf("compute add-to batch hash: %w", err) } - batchID, err := insertAddToBatch(tx, msgID, addToFrom, now, batchHash) + batchID, err := insertAddToBatch(tx, msgID, addToFrom, msg.Timestamp, batchHash) if err != nil { return err } diff --git a/dd.sql b/dd.sql index 762adba..9d86e2f 100644 --- a/dd.sql +++ b/dd.sql @@ -64,7 +64,7 @@ create table if not exists msg_add_to_batch ( id bigserial primary key, msg_id bigint not null references msg (id), add_to_from varchar(255) not null, -- sender that added this batch's recipients - time_added double precision not null, -- when this host recorded the batch + time_added double precision not null, -- the batch message's wire time field (for locally originated batches, when the batch was created) sha256 bytea -- batch message hash: the batch's identity (SPEC §11) ); alter table msg_add_to_batch add column if not exists sha256 bytea; From 07d649846822191edc643cccac3ae199ac0c38a2 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Tue, 11 Aug 2026 07:42:52 +1000 Subject: [PATCH 3/4] Persist the batch hash at first delivery of a locally originated batch The receive path stores a batch's hash when recording it, but batches this host originates never had theirs stored - so a remote reply referencing our own batch could not resolve here (SPEC SS11: a host verifies messages it sent, not only ones it received). Mirror ensureSharedHash: compute once at first delivery (the challenge response reuses the cached value) and persist when null. Co-Authored-By: Claude Fable 5 --- cmd/fmsgd/sender.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/cmd/fmsgd/sender.go b/cmd/fmsgd/sender.go index 83cb111..5a79dee 100644 --- a/cmd/fmsgd/sender.go +++ b/cmd/fmsgd/sender.go @@ -491,6 +491,19 @@ func deliverMessage(target pendingTarget) { } h := m.addToHeader(b, sharedHash) d.applyTo(h) + // Persist the batch hash — the batch's identity (SPEC §11) — once, + // so replies referencing this batch resolve at this host too, which + // must verify messages it sent, not only ones it received. Cached on + // h, so the challenge response reuses this computation. + batchHash, err := h.GetMessageHash() + if err != nil { + log.Printf("ERROR: sender: computing batch hash for batch %d of msg %d: %s", b.ID, target.MsgID, err) + continue + } + if err := ensureBatchHash(db, b.ID, batchHash); err != nil { + log.Printf("ERROR: sender: %s", err) + continue + } deliverUnit(db, target, h, "msg_add_to", b.ID) } } @@ -515,6 +528,17 @@ func markLocalDelivered(target pendingTarget) { } } +// ensureBatchHash persists an add-to batch's message hash when not yet stored. +// Like ensureSharedHash for the canonical hash, this is what lets replies that +// reference the batch via pid resolve on the host that originated the batch +// (SPEC §11: a host verifies messages it sent, not only ones it received). +func ensureBatchHash(db *sql.DB, batchID int64, batchHash []byte) error { + if _, err := db.Exec(`UPDATE msg_add_to_batch SET sha256 = $1 WHERE id = $2 AND sha256 IS NULL`, batchHash, batchID); err != nil { + return fmt.Errorf("storing sha256 for add-to batch %d: %w", batchID, err) + } + return nil +} + // ensureSharedHash persists the message's canonical hash when not yet stored and // resolves any pending child (reply/add-to) links that reference it. func ensureSharedHash(db *sql.DB, msgID int64, sharedHash []byte) error { From 647c36e4a7a215f43642080268b46bc257382579 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Tue, 11 Aug 2026 09:04:07 +1000 Subject: [PATCH 4/4] Allow reply psha256 to match a parent's add-to batch hash The populate-psha256 trigger required psha256 to equal the relational parent's canonical sha256, rejecting stored replies that reference one of the parent's add-to batch messages by batch hash (SPEC SS12). Co-Authored-By: Claude Fable 5 --- dd.sql | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dd.sql b/dd.sql index 9d86e2f..2d4e14b 100644 --- a/dd.sql +++ b/dd.sql @@ -134,7 +134,14 @@ begin if NEW.psha256 is null or octet_length(NEW.psha256) = 0 then NEW.psha256 = parent_sha256; elsif NEW.psha256 <> parent_sha256 then - raise exception 'psha256 does not match parent message % sha256', NEW.pid; + -- a reply may reference one of the parent's add-to batch messages by + -- its batch hash (SPEC §12); the relational parent is the shared row + if not exists ( + select 1 from msg_add_to_batch b + where b.msg_id = NEW.pid and b.sha256 = NEW.psha256 + ) then + raise exception 'psha256 does not match parent message % sha256 or any of its add-to batch hashes', NEW.pid; + end if; end if; return NEW;