From 4ce979d7bbdb13c92ad47a1ce2acd8a773263c8d Mon Sep 17 00:00:00 2001 From: Enrique Lacal Date: Tue, 28 Jul 2026 20:51:00 +0100 Subject: [PATCH 1/2] fix: IPFS peering never persists, and bitswap hangs even when peered Two separate bugs were still causing shared storage downloads to hang/time out intermittently even after the earlier peering fix (#359): 1. swarm/peering/add returns success but never actually writes to the on-disk Peering.Peers config - verified directly by checking the config immediately after calling it. It only registers the peer with the running daemon's in-memory peering service for that session. Since first-time setup restarts the IPFS containers partway through (to apply finalized config), any peering set up before that restart was silently lost, leaving nodes to fall back on unreliable mDNS. Switched to POST /api/v0/config to persist Peering.Peers for real; verified it survives a full container restart with both nodes reconnecting automatically. 2. Even once genuinely peered, content fetches still hang: this is a long-standing upstream Kubo/bitswap bug (ipfs/kubo#8346, open since 2021) where in a small private network, bitswap discovers a provider via the DHT but never sends it a WANT message if the swarm connection to that peer already existed - reproduced directly (bitswap stat showed 0 partners despite an active, bitswap-protocol-negotiated swarm connection). Routing.Type is now set to "none" instead of "dht": with no DHT to depend on, bitswap has no choice but to broadcast wants directly to its connected peers, which is all a small private swarm needs anyway. Verified 5/5 sequential cross-node fetches plus the HTTP gateway path all succeed reliably with this change. Signed-off-by: Enrique Lacal --- internal/stacks/ipfs_config.go | 10 +++++++- internal/stacks/stack_manager.go | 41 +++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/internal/stacks/ipfs_config.go b/internal/stacks/ipfs_config.go index e05f854..1777d65 100644 --- a/internal/stacks/ipfs_config.go +++ b/internal/stacks/ipfs_config.go @@ -34,6 +34,14 @@ func GenerateSwarmKey() (string, error) { // disables Kubo's AutoConf/public-network defaults, which are incompatible // with a private swarm (swarm.key / LIBP2P_FORCE_PNET) and otherwise prevent // the daemon from starting or from ever peering with other members. +// +// Routing.Type is set to "none" rather than "dht": a small private swarm hits +// a long-standing upstream bug (https://github.com/ipfs/kubo/issues/8346) +// where bitswap finds a provider via the DHT but never sends it a WANT +// message if the swarm connection to that peer already existed, so content +// fetches hang even though the peers are connected. With routing disabled, +// bitswap has no choice but to broadcast wants directly to its connected +// peers, which is all a 2+ node private swarm actually needs. func GenerateIPFSPrivateNetInitScript() string { return `#!/bin/sh ipfs config --json AutoConf.Enabled false @@ -41,7 +49,7 @@ ipfs config --json Bootstrap '[]' ipfs config --json DNS.Resolvers '{}' ipfs config --json Routing.DelegatedRouters '[]' ipfs config --json Ipns.DelegatedPublishers '[]' -ipfs config Routing.Type dht +ipfs config Routing.Type none ipfs config --json AutoTLS.Enabled false ipfs config --json Swarm.Transports.Network.Websocket false ` diff --git a/internal/stacks/stack_manager.go b/internal/stacks/stack_manager.go index d95fd8e..d2cc68a 100644 --- a/internal/stacks/stack_manager.go +++ b/internal/stacks/stack_manager.go @@ -22,6 +22,7 @@ import ( "fmt" "net" "net/http" + "net/url" "os" "os/exec" "path" @@ -614,24 +615,46 @@ func (s *StackManager) peerIPFSNodes() error { peerIDs[member.ID] = idResp.ID } + type peeringPeer struct { + ID string `json:"ID"` + Addrs []string `json:"Addrs"` + } + for _, member := range s.Stack.Members { + peers := []peeringPeer{} for _, other := range s.Stack.Members { if member.ID == other.ID { continue } - addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID]) + peers = append(peers, peeringPeer{ + ID: peerIDs[other.ID], + Addrs: []string{fmt.Sprintf("/dns4/ipfs_%s/tcp/4001", other.ID)}, + }) + } + peersJSON, err := json.Marshal(peers) + if err != nil { + return err + } - // swarm/peering/add only registers the peer with Kubo's background - // reconnect service - it returns success even when the peer is - // unreachable, so it does not confirm a connection was made. - peeringURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr) - if err := core.RequestWithRetry(s.ctx, http.MethodPost, peeringURL, nil, nil); err != nil { - return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err) - } + // swarm/peering/add only registers the peer with the *running* + // daemon's in-memory peering service and returns success even when + // the peer is unreachable - it does not persist to Peering.Peers in + // the on-disk config, so the pairing is lost on the next restart. + // Writing through /api/v0/config persists it for real, and Kubo + // picks up the change live (no restart needed). + configURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/config?arg=Peering.Peers&arg=%s&json=true", member.ExposedIPFSApiPort, url.QueryEscape(string(peersJSON))) + if err := core.RequestWithRetry(s.ctx, http.MethodPost, configURL, nil, nil); err != nil { + return fmt.Errorf("failed to persist IPFS peering config for member %s: %w", member.ID, err) + } + for _, other := range s.Stack.Members { + if member.ID == other.ID { + continue + } + addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID]) // swarm/connect dials synchronously and errors if the connection // fails, so retrying it confirms the nodes are actually connected - // rather than just registered to reconnect in the background. + // now rather than waiting on the peering service's own timing. connectURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/connect?arg=%s", member.ExposedIPFSApiPort, addr) if err := core.RequestWithRetry(s.ctx, http.MethodPost, connectURL, nil, nil); err != nil { return fmt.Errorf("failed to connect IPFS node %s to %s: %w", member.ID, other.ID, err) From b4d70eb12d58839589f0a2fb85fabd0403bd1960 Mon Sep 17 00:00:00 2001 From: Enrique Lacal Date: Tue, 28 Jul 2026 21:01:51 +0100 Subject: [PATCH 2/2] chore: log IPFS peering progress and confirmation Makes it possible to confirm from a run's own log output whether peering happened, instead of having to dig into container-level config/bitswap state after the fact - which is how the persistence bug in this PR was found in the first place. Signed-off-by: Enrique Lacal --- internal/stacks/stack_manager.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/stacks/stack_manager.go b/internal/stacks/stack_manager.go index d2cc68a..5c55c46 100644 --- a/internal/stacks/stack_manager.go +++ b/internal/stacks/stack_manager.go @@ -601,6 +601,8 @@ func (s *StackManager) peerIPFSNodes() error { return nil } + s.Log.Info("peering IPFS nodes") + type ipfsIDResponse struct { ID string `json:"ID"` } @@ -659,8 +661,11 @@ func (s *StackManager) peerIPFSNodes() error { if err := core.RequestWithRetry(s.ctx, http.MethodPost, connectURL, nil, nil); err != nil { return fmt.Errorf("failed to connect IPFS node %s to %s: %w", member.ID, other.ID, err) } + s.Log.Info(fmt.Sprintf("IPFS node %s peered with %s (%s)", member.ID, other.ID, peerIDs[other.ID])) } } + + s.Log.Info("IPFS nodes peered successfully") return nil }