From 037d09c32e453bc1ca68830f82090e6655716536 Mon Sep 17 00:00:00 2001 From: Enrique Lacal Date: Thu, 23 Jul 2026 15:49:50 +0100 Subject: [PATCH 1/3] fix: private IPFS swarm nodes never peer, breaking shared storage The pinned ipfs/go-ipfs:v0.10.0 image has a private-network (pnet) connection bug where swarm connections drop immediately after handshake, so org IPFS nodes never peer with each other. This causes "failed to bootstrap (no peers found)" and shared storage downloads to time out fetching content pinned only on another member's node. Bump to ipfs/kubo:v0.42.0, and since modern Kubo refuses to start in private-network mode with its default AutoConf/public bootstrap config, add a container-init.d script that disables it for private-mode stacks. mDNS auto-discovery, which worked locally, proved unreliable on native Linux Docker bridge networks - members' nodes never found each other there. Add an explicit peering step after first-time-setup that queries each member's real PeerID via its own API and calls swarm/peering/add against every other member, so nodes connect (and persistently reconnect) regardless of whether mDNS works in a given environment. Signed-off-by: Enrique Lacal --- internal/constants/constants.go | 2 +- internal/docker/docker_config.go | 5 +++ internal/stacks/ipfs_config.go | 17 ++++++++ internal/stacks/stack_manager.go | 71 ++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 1 deletion(-) diff --git a/internal/constants/constants.go b/internal/constants/constants.go index ac2d7bbf..21f47066 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -23,7 +23,7 @@ import ( var StacksDir = checkHome() var FireFlyCoreImageName = "ghcr.io/hyperledger-firefly/firefly" -var IPFSImageName = "ipfs/go-ipfs:v0.10.0" +var IPFSImageName = "ipfs/kubo:v0.42.0" var PostgresImageName = "postgres" var PrometheusImageName = "prom/prometheus" var SandboxImageName = "ghcr.io/hyperledger-firefly/sandbox:latest" diff --git a/internal/docker/docker_config.go b/internal/docker/docker_config.go index 97088d5a..383c4d17 100644 --- a/internal/docker/docker_config.go +++ b/internal/docker/docker_config.go @@ -167,6 +167,11 @@ func CreateDockerCompose(s *types.Stack) *DockerComposeConfig { "LIBP2P_FORCE_PNET": "1", }, ) + // Kubo's AutoConf/public-network defaults are incompatible with a + // private swarm and prevent peering with other members unless + // disabled via a container-init.d script - see ipfs_config.go. + sharedStorage.Volumes = append(sharedStorage.Volumes, fmt.Sprintf("ipfs_init_%s:/container-init.d", member.ID)) + compose.Volumes[fmt.Sprintf("ipfs_init_%s", member.ID)] = struct{}{} } else { sharedStorage.Environment = s.EnvironmentVars } diff --git a/internal/stacks/ipfs_config.go b/internal/stacks/ipfs_config.go index a63f006c..e05f8541 100644 --- a/internal/stacks/ipfs_config.go +++ b/internal/stacks/ipfs_config.go @@ -29,3 +29,20 @@ func GenerateSwarmKey() (string, error) { hexKey := hex.EncodeToString(key) return "/key/swarm/psk/1.0.0/\n/base16/\n" + hexKey, nil } + +// GenerateIPFSPrivateNetInitScript returns a container-init.d script that +// disables Kubo's AutoConf/public-network defaults, which are incompatible +// with a private swarm (swarm.key / LIBP2P_FORCE_PNET) and otherwise prevent +// the daemon from starting or from ever peering with other members. +func GenerateIPFSPrivateNetInitScript() string { + return `#!/bin/sh +ipfs config --json AutoConf.Enabled false +ipfs config --json Bootstrap '[]' +ipfs config --json DNS.Resolvers '{}' +ipfs config --json Routing.DelegatedRouters '[]' +ipfs config --json Ipns.DelegatedPublishers '[]' +ipfs config Routing.Type dht +ipfs config --json AutoTLS.Enabled false +ipfs config --json Swarm.Transports.Network.Websocket false +` +} diff --git a/internal/stacks/stack_manager.go b/internal/stacks/stack_manager.go index c8bfb7f6..5907b26e 100644 --- a/internal/stacks/stack_manager.go +++ b/internal/stacks/stack_manager.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "net" + "net/http" "os" "os/exec" "path" @@ -505,6 +506,13 @@ func (s *StackManager) writeConfig(options *types.InitOptions) error { } } + if s.Stack.IPFSMode.Equals(types.IPFSModePrivate) { + initScript := GenerateIPFSPrivateNetInitScript() + if err := os.WriteFile(path.Join(s.Stack.InitDir, "config", "ipfs_privatenet_init.sh"), []byte(initScript), 0755); err != nil { + return err + } + } + return nil } @@ -566,6 +574,61 @@ func (s *StackManager) copyDataExchangeConfigToVolumes() error { return nil } +func (s *StackManager) copyIPFSInitScriptToVolumes() error { + if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) { + return nil + } + configDir := filepath.Join(s.Stack.RuntimeDir, "config") + scriptPath := path.Join(configDir, "ipfs_privatenet_init.sh") + for _, member := range s.Stack.Members { + volumeName := fmt.Sprintf("%s_ipfs_init_%s", s.Stack.Name, member.ID) + if err := docker.CopyFileToVolume(s.ctx, volumeName, scriptPath, "/privatenet-init.sh"); err != nil { + return err + } + } + return nil +} + +// peerIPFSNodes explicitly peers every private-mode IPFS node with every +// other member's node. mDNS auto-discovery has proven unreliable across +// different Docker networking environments (it connected nodes on Docker +// Desktop but not on a native Linux Docker bridge network, such as GitHub +// Actions runners use), so without this, members' IPFS nodes may never +// connect to each other and shared storage downloads will hang/time out. +func (s *StackManager) peerIPFSNodes() error { + if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) || len(s.Stack.Members) < 2 { + return nil + } + + type ipfsIDResponse struct { + ID string `json:"ID"` + } + + peerIDs := make(map[string]string, len(s.Stack.Members)) + for _, member := range s.Stack.Members { + var idResp ipfsIDResponse + url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/id", member.ExposedIPFSApiPort) + if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, &idResp); err != nil { + return fmt.Errorf("failed to get IPFS peer ID for member %s: %w", member.ID, err) + } + peerIDs[member.ID] = idResp.ID + } + + for _, member := range s.Stack.Members { + for _, other := range s.Stack.Members { + if member.ID == other.ID { + continue + } + addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID]) + url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr) + if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, nil); err != nil { + return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err) + } + } + } + return nil +} + func (s *StackManager) createMember(id string, index int, options *types.InitOptions, external bool) (*types.Organization, error) { serviceBase := options.ServicesBasePort + (index * 100) ptmBase := options.PtmBasePort + (index * 10) @@ -908,6 +971,10 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages return messages, err } + if err := s.copyIPFSInitScriptToVolumes(); err != nil { + return messages, err + } + pullOptions := &types.PullOptions{ Retries: 2, } @@ -919,6 +986,10 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages return messages, err } + if err := s.peerIPFSNodes(); err != nil { + return messages, err + } + for i, tp := range s.tokenProviders { if !s.Stack.DisableTokenFactories { result, err := tp.DeploySmartContracts(i) From 61fef2f278d9201eecd6916d28f2b958fbfb43ac Mon Sep 17 00:00:00 2001 From: Enrique Lacal Date: Fri, 24 Jul 2026 16:05:50 +0100 Subject: [PATCH 2/3] Peer the nodes after restart Signed-off-by: Enrique Lacal --- internal/stacks/stack_manager.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/stacks/stack_manager.go b/internal/stacks/stack_manager.go index 5907b26e..1f23492d 100644 --- a/internal/stacks/stack_manager.go +++ b/internal/stacks/stack_manager.go @@ -986,10 +986,6 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages return messages, err } - if err := s.peerIPFSNodes(); err != nil { - return messages, err - } - for i, tp := range s.tokenProviders { if !s.Stack.DisableTokenFactories { result, err := tp.DeploySmartContracts(i) @@ -1106,6 +1102,13 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages return messages, err } + // Peer the IPFS nodes on the finalized containers, after the config + // restart above, so peering is applied to the containers that will keep + // running rather than relying on it surviving the restart. + if err := s.peerIPFSNodes(); err != nil { + return messages, err + } + if s.Stack.MultipartyEnabled { if s.Stack.ContractAddress == "" { s.Log.Info("registering FireFly identities") From 41405ff4d81f44981548e5ca8aa3588f82eaa5f2 Mon Sep 17 00:00:00 2001 From: Daniel Raper Date: Mon, 27 Jul 2026 10:46:18 +0100 Subject: [PATCH 3/3] Attempt to resolve CI issue Signed-off-by: Daniel Raper --- internal/stacks/stack_manager.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/stacks/stack_manager.go b/internal/stacks/stack_manager.go index 1f23492d..d95fd8eb 100644 --- a/internal/stacks/stack_manager.go +++ b/internal/stacks/stack_manager.go @@ -620,10 +620,22 @@ func (s *StackManager) peerIPFSNodes() error { continue } addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID]) - url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr) - if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, nil); err != nil { + + // swarm/peering/add only registers the peer with Kubo's background + // reconnect service - it returns success even when the peer is + // unreachable, so it does not confirm a connection was made. + peeringURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr) + if err := core.RequestWithRetry(s.ctx, http.MethodPost, peeringURL, nil, nil); err != nil { return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err) } + + // swarm/connect dials synchronously and errors if the connection + // fails, so retrying it confirms the nodes are actually connected + // rather than just registered to reconnect in the background. + connectURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/connect?arg=%s", member.ExposedIPFSApiPort, addr) + if err := core.RequestWithRetry(s.ctx, http.MethodPost, connectURL, nil, nil); err != nil { + return fmt.Errorf("failed to connect IPFS node %s to %s: %w", member.ID, other.ID, err) + } } } return nil