fix: private IPFS swarm nodes never peer, breaking shared storage

internal/constants/constants.go

@@ -23,7 +23,7 @@ import (
 
 var StacksDir = checkHome()
 var FireFlyCoreImageName = "ghcr.io/hyperledger-firefly/firefly"
-var IPFSImageName = "ipfs/go-ipfs:v0.10.0"
+var IPFSImageName = "ipfs/kubo:v0.42.0"
 var PostgresImageName = "postgres"
 var PrometheusImageName = "prom/prometheus"
 var SandboxImageName = "ghcr.io/hyperledger-firefly/sandbox:latest"

internal/docker/docker_config.go

@@ -167,6 +167,11 @@ func CreateDockerCompose(s *types.Stack) *DockerComposeConfig {
 				"LIBP2P_FORCE_PNET": "1",
 			},
 			)
+			// Kubo's AutoConf/public-network defaults are incompatible with a
+			// private swarm and prevent peering with other members unless
+			// disabled via a container-init.d script - see ipfs_config.go.
+			sharedStorage.Volumes = append(sharedStorage.Volumes, fmt.Sprintf("ipfs_init_%s:/container-init.d", member.ID))
+			compose.Volumes[fmt.Sprintf("ipfs_init_%s", member.ID)] = struct{}{}
 		} else {
 			sharedStorage.Environment = s.EnvironmentVars
 		}

internal/stacks/ipfs_config.go

@@ -29,3 +29,20 @@ func GenerateSwarmKey() (string, error) {
 	hexKey := hex.EncodeToString(key)
 	return "/key/swarm/psk/1.0.0/\n/base16/\n" + hexKey, nil
 }
+
+// GenerateIPFSPrivateNetInitScript returns a container-init.d script that
+// disables Kubo's AutoConf/public-network defaults, which are incompatible
+// with a private swarm (swarm.key / LIBP2P_FORCE_PNET) and otherwise prevent
+// the daemon from starting or from ever peering with other members.
+func GenerateIPFSPrivateNetInitScript() string {
+	return `#!/bin/sh
+ipfs config --json AutoConf.Enabled false
+ipfs config --json Bootstrap '[]'
+ipfs config --json DNS.Resolvers '{}'
+ipfs config --json Routing.DelegatedRouters '[]'
+ipfs config --json Ipns.DelegatedPublishers '[]'
+ipfs config Routing.Type dht
+ipfs config --json AutoTLS.Enabled false
+ipfs config --json Swarm.Transports.Network.Websocket false
+`
+}

internal/stacks/stack_manager.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path"
@@ -505,6 +506,13 @@ func (s *StackManager) writeConfig(options *types.InitOptions) error {
 		}
 	}
 
+	if s.Stack.IPFSMode.Equals(types.IPFSModePrivate) {
+		initScript := GenerateIPFSPrivateNetInitScript()
+		if err := os.WriteFile(path.Join(s.Stack.InitDir, "config", "ipfs_privatenet_init.sh"), []byte(initScript), 0755); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -566,6 +574,73 @@ func (s *StackManager) copyDataExchangeConfigToVolumes() error {
 	return nil
 }
 
+func (s *StackManager) copyIPFSInitScriptToVolumes() error {
+	if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) {
+		return nil
+	}
+	configDir := filepath.Join(s.Stack.RuntimeDir, "config")
+	scriptPath := path.Join(configDir, "ipfs_privatenet_init.sh")
+	for _, member := range s.Stack.Members {
+		volumeName := fmt.Sprintf("%s_ipfs_init_%s", s.Stack.Name, member.ID)
+		if err := docker.CopyFileToVolume(s.ctx, volumeName, scriptPath, "/privatenet-init.sh"); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// peerIPFSNodes explicitly peers every private-mode IPFS node with every
+// other member's node. mDNS auto-discovery has proven unreliable across
+// different Docker networking environments (it connected nodes on Docker
+// Desktop but not on a native Linux Docker bridge network, such as GitHub
+// Actions runners use), so without this, members' IPFS nodes may never
+// connect to each other and shared storage downloads will hang/time out.
+func (s *StackManager) peerIPFSNodes() error {
+	if !s.Stack.IPFSMode.Equals(types.IPFSModePrivate) || len(s.Stack.Members) < 2 {
+		return nil
+	}
+
+	type ipfsIDResponse struct {
+		ID string `json:"ID"`
+	}
+
+	peerIDs := make(map[string]string, len(s.Stack.Members))
+	for _, member := range s.Stack.Members {
+		var idResp ipfsIDResponse
+		url := fmt.Sprintf("http://127.0.0.1:%d/api/v0/id", member.ExposedIPFSApiPort)
+		if err := core.RequestWithRetry(s.ctx, http.MethodPost, url, nil, &idResp); err != nil {
+			return fmt.Errorf("failed to get IPFS peer ID for member %s: %w", member.ID, err)
+		}
+		peerIDs[member.ID] = idResp.ID
+	}
+
+	for _, member := range s.Stack.Members {
+		for _, other := range s.Stack.Members {
+			if member.ID == other.ID {
+				continue
+			}
+			addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID])
+
+			// swarm/peering/add only registers the peer with Kubo's background
+			// reconnect service - it returns success even when the peer is
+			// unreachable, so it does not confirm a connection was made.
+			peeringURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr)
+			if err := core.RequestWithRetry(s.ctx, http.MethodPost, peeringURL, nil, nil); err != nil {
+				return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err)
+			}
+
+			// swarm/connect dials synchronously and errors if the connection
+			// fails, so retrying it confirms the nodes are actually connected
+			// rather than just registered to reconnect in the background.
+			connectURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/connect?arg=%s", member.ExposedIPFSApiPort, addr)
+			if err := core.RequestWithRetry(s.ctx, http.MethodPost, connectURL, nil, nil); err != nil {
+				return fmt.Errorf("failed to connect IPFS node %s to %s: %w", member.ID, other.ID, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (s *StackManager) createMember(id string, index int, options *types.InitOptions, external bool) (*types.Organization, error) {
 	serviceBase := options.ServicesBasePort + (index * 100)
 	ptmBase := options.PtmBasePort + (index * 10)
@@ -908,6 +983,10 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages
 		return messages, err
 	}
 
+	if err := s.copyIPFSInitScriptToVolumes(); err != nil {
+		return messages, err
+	}
+
 	pullOptions := &types.PullOptions{
 		Retries: 2,
 	}
@@ -1035,6 +1114,13 @@ func (s *StackManager) runFirstTimeSetup(options *types.StartOptions) (messages
 		return messages, err
 	}
 
+	// Peer the IPFS nodes on the finalized containers, after the config
+	// restart above, so peering is applied to the containers that will keep
+	// running rather than relying on it surviving the restart.
+	if err := s.peerIPFSNodes(); err != nil {
+		return messages, err
+	}
+
 	if s.Stack.MultipartyEnabled {
 		if s.Stack.ContractAddress == "" {
 			s.Log.Info("registering FireFly identities")