Implement cf-pcap to enable packet captures in apps

maxmoehl · maxmoehl · commit 490538c55ce5 · 2026-02-04T18:51:59.000+01:00
diff --git a/docs/090-cf-pcap.md b/docs/090-cf-pcap.md
@@ -0,0 +1,71 @@
+# cf-pcap
+
+## Enable the feature platform wide
+
+Set the rep property `diego.rep.enable_cf_pcap` to true (defaults to false) and
+re-deploy diego.
+
+This will set a file capability on the cf-pcap binary which is always inside the
+app container to allow the binary to perform packet captures.
+
+## Usage
+
+<!-- TODO: fix path -->
+The binary is placed in `/etc/cf-assets/...` and can be executed by any regular
+user. Invoking the binary will cause it to emit raw pcap on stdout which can
+mess up your terminal so make sure to redirect it into a file. The accepted
+parameters are:
+
+* `-interface`: the network interface on which to capture (usually `lo` or
+  `eth0`).
+* `-snaplen`: how many bytes to capture of each packet.
+* `-filter`: pcap-filter to specify which packets to capture.
+* `-v`: more verbose output.
+
+## Notes in network traffic in app containers
+
+When envoy is enabled, all traffic passes through both interfaces in the app
+container. First, envoy receives it on `eth0` and terminates TLS, then it
+forwards the traffic to the app via the `lo` device in plain text. For SSH
+traffic the same thing happens, but envoy won't terminate the SSH encryption.
+
+To avoid capturing the SSH traffic which contains the just captured bytes, thus
+indefinitely capturing the same content over and over again, cf-pcap will try to
+determine the correct port to filter out. This is done on a best-effort basis
+and failure to determine the correct filter will not stop the capture from
+happening.
+
+## Capturing traffic locally
+
+SSH into the app instance you would like to capture traffic from. To capture the
+plain-text HTTP traffic for an app listening on port 8080 you specify the
+loopback interface and filter for TCP traffic on port 8080. Once you've copied
+the file to your local machine you can print out the captured traffic your
+tcpdumps `-r` to read form a previous capture file:
+
+<!-- TODO: correct paths and output -->
+```
+$ cf ssh $app
+vcap@app $ cf-pcap -interface lo -filter "tcp port 8080" > capture01.pcap
+...
+^C
+...
+vcap@app $ ^D
+$ scp $app:capture01.pcap .
+$ tcpdump -Xr capture01.pcap
+```
+
+## Capturing traffic remotely
+
+Similar to the local capture, you start cf-pcap with the details on what you
+want to capture but this time wrap it in the `-c` of cf-ssh. This will send the
+pcap output to stdout and you can redirect the content to a local file.
+Afterwards, you can again use tcpdumps `-r` to inspect the content:
+
+```
+cf ssh $app -c "cf-pcap -interface lo -filter 'tcp port 8080'" > capture02.pcap
+...
+^C
+...
+$ tcpdump -Xr capture02.pcap
+```
diff --git a/jobs/rep/spec b/jobs/rep/spec
@@ -245,6 +245,10 @@ properties:
     description: "Maximum container capacity per rep"
     default: 250
 
+  diego.rep.enable_cf_pcap:
+    description: "Allow app developers to capture tcpdumps within their app containers."
+    default: false
+
   enable_healthcheck_metrics:
     description: "When set, enables the rep to emit healtcheck failure metrics."
     default: false
diff --git a/jobs/rep/templates/bpm.yml.erb b/jobs/rep/templates/bpm.yml.erb
@@ -2,6 +2,11 @@
 processes:
 - name: rep
   executable: /var/vcap/jobs/rep/bin/rep
+<% if p("diego.rep.enable_cf_pcap", false) -%>
+  # This is required to preserve the file capabilities of cf-pcap to allow the vcap user to capture
+  # packets inside the app container without root.
+  capabilities: [ "SETFCAP" ]
+<% end -%>
   limits:
     open_files: 100000
   hooks:
diff --git a/packages/buildpack_app_lifecycle/packaging b/packages/buildpack_app_lifecycle/packaging
@@ -27,12 +27,16 @@ ldd $DEST/launcher && echo "launcher must be statically linked" && false
 cp /var/vcap/packages/diego-sshd/diego-sshd ${DEST}/diego-sshd
 cp /var/vcap/packages/diego-sshd/*.exe ${DEST}
 cp /var/vcap/packages/diego-sshd/winpty.dll ${DEST}/winpty.dll
+cp /var/vcap/packages/cf-pcap/cf-pcap ${DEST}/cf-pcap
 cp /var/vcap/packages/healthcheck/healthcheck ${DEST}/healthcheck
 cp /var/vcap/packages/healthcheck/healthcheck.exe ${DEST}/healthcheck.exe
 
+setcap cap_net_raw+ep ${DEST}/cf-pcap
+
 tar -czf ${BOSH_INSTALL_TARGET}/buildpack_app_lifecycle.tgz \
+  --xattrs --xattrs-include='*' \
   -C ${DEST} \
-  builder launcher shell healthcheck diego-sshd  \
+  builder launcher shell healthcheck diego-sshd cf-pcap  \
   builder.exe launcher.exe getenv.exe healthcheck.exe diego-sshd.exe \
   winpty-agent.exe winpty.dll
 
diff --git a/packages/buildpack_app_lifecycle/spec b/packages/buildpack_app_lifecycle/spec
@@ -5,6 +5,7 @@ dependencies:
   - golang-1.25-linux
   - healthcheck
   - diego-sshd
+  - cf-pcap
 
 files:
   - code.cloudfoundry.org/go.mod
diff --git a/packages/cf-pcap/packaging b/packages/cf-pcap/packaging
@@ -0,0 +1,22 @@
+set -e
+
+tar -xzf libpcap/libpcap-*.tar.gz
+
+LIBPCAP_DIR=$(ls -d libpcap-*)
+
+pushd ${LIBPCAP_DIR}
+  ./configure
+  make
+popd
+
+source /var/vcap/packages/golang-*-linux/bosh/compile.env
+
+pushd code.cloudfoundry.org
+  # -I adds the local libpcap to the search path for the compiler
+  export CGO_CFLAGS="-I${BOSH_COMPILE_TARGET}/${LIBPCAP_DIR}"
+  # -L adds the local libpcap to the search path for the linker
+  # -linkmode external: use an external linker: https://cs.opensource.google/go/go/+/refs/tags/go1.18:src/cmd/cgo/doc.go;l=794
+  # -extldflags -static: pass -static to ld, see man page for details
+  export CGO_LDFLAGS="-L${BOSH_COMPILE_TARGET}/${LIBPCAP_DIR} -static"
+  go build -ldflags '-linkmode external' -o ${BOSH_INSTALL_TARGET}/cf-pcap -a -installsuffix static code.cloudfoundry.org/cf-pcap
+popd
diff --git a/packages/cf-pcap/spec b/packages/cf-pcap/spec
@@ -0,0 +1,20 @@
+---
+name: cf-pcap
+dependencies:
+  - golang-1.25-linux
+
+files:
+  - libpcap/libpcap-*.tar.xz
+  - code.cloudfoundry.org/go.mod
+  - code.cloudfoundry.org/go.sum
+  - code.cloudfoundry.org/vendor/modules.txt
+  - code.cloudfoundry.org/cf-pcap/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/endian/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/layers/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/pcap/*.go # gosub
+  - code.cloudfoundry.org/vendor/github.com/gopacket/gopacket/pcapgo/*.go # gosub
+  - code.cloudfoundry.org/vendor/golang.org/x/net/bpf/*.go # gosub
+  - code.cloudfoundry.org/vendor/golang.org/x/sys/unix/*.go # gosub
+  - code.cloudfoundry.org/vendor/golang.org/x/sys/unix/*.s # gosub
+  - code.cloudfoundry.org/vendor/golang.org/x/sys/windows/*.go # gosub
diff --git a/packages/cnb_app_lifecycle/packaging b/packages/cnb_app_lifecycle/packaging
@@ -20,8 +20,12 @@ ldd $DEST/builder && echo "builder must be statically linked" && false
 ldd $DEST/launcher && echo "launcher must be statically linked" && false
 
 cp /var/vcap/packages/diego-sshd/diego-sshd ${DEST}/diego-sshd
+cp /var/vcap/packages/cf-pcap/cf-pcap ${DEST}/cf-pcap
 cp /var/vcap/packages/healthcheck/healthcheck ${DEST}/healthcheck
 
+setcap cap_net_raw+ep ${DEST}/cf-pcap
+
 tar -czf ${BOSH_INSTALL_TARGET}/cnb_app_lifecycle.tgz \
+  --xattrs --xattrs-include='*' \
   -C ${DEST} \
-  builder launcher healthcheck diego-sshd
+  builder launcher healthcheck diego-sshd cf-pcap
diff --git a/packages/cnb_app_lifecycle/spec b/packages/cnb_app_lifecycle/spec
@@ -5,6 +5,7 @@ dependencies:
   - golang-1.25-linux
   - healthcheck
   - diego-sshd
+  - cf-pcap
 
 files:
   - cnbapplifecycle/go.mod
diff --git a/packages/docker_app_lifecycle/packaging b/packages/docker_app_lifecycle/packaging
@@ -19,6 +19,11 @@ ldd ${DEST}/builder && echo "builder must be statically linked" && false
 ldd ${DEST}/launcher && echo "launcher must be statically linked" && false
 
 cp /var/vcap/packages/diego-sshd/diego-sshd ${DEST}/diego-sshd
+cp /var/vcap/packages/cf-pcap/cf-pcap ${DEST}/cf-pcap
 cp /var/vcap/packages/healthcheck/healthcheck ${DEST}/healthcheck
 
-tar -czf ${BOSH_INSTALL_TARGET}/docker_app_lifecycle.tgz -C ${DEST} builder launcher healthcheck diego-sshd
+setcap cap_net_raw+ep ${DEST}/cf-pcap
+
+tar -czf ${BOSH_INSTALL_TARGET}/docker_app_lifecycle.tgz \
+  --xattrs --xattrs-include='*' \
+  -C ${DEST} builder launcher healthcheck diego-sshd cf-pcap
diff --git a/packages/docker_app_lifecycle/spec b/packages/docker_app_lifecycle/spec
@@ -5,6 +5,7 @@ dependencies:
   - golang-1.25-linux
   - healthcheck
   - diego-sshd
+  - cf-pcap
 
 files:
   - code.cloudfoundry.org/go.mod
diff --git a/src/code.cloudfoundry.org/cf-pcap/pcap.go b/src/code.cloudfoundry.org/cf-pcap/pcap.go
@@ -0,0 +1,190 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/gopacket/gopacket"
+	"github.com/gopacket/gopacket/pcap"
+	"github.com/gopacket/gopacket/pcapgo"
+)
+
+var (
+	interfaceName = flag.String("interface", "", "Network interface to capture from (e.g. eth0, any)")
+	snaplen       = flag.Int("snaplen", 65535, "Snapshot length - max bytes to capture per packet")
+	filter        = flag.String("filter", "", "BPF filter expression (e.g. 'tcp port 80')")
+	verbose       = flag.Bool("v", false, "Verbose output")
+)
+
+var (
+	logLevel = &slog.LevelVar{}
+)
+
+func init() {
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})))
+}
+
+func main() {
+	os.Exit(Main())
+}
+
+func Main() int {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	flag.Parse()
+
+	if *interfaceName == "" {
+		slog.Error("interface flag is required")
+		return 1
+	}
+
+	if *verbose {
+		logLevel.Set(slog.LevelDebug)
+	}
+
+	slog.Debug("parsed flags",
+		"interface", *interfaceName,
+		"snaplen", *snaplen,
+		"filter", *filter,
+	)
+
+	errC := capture(ctx, *interfaceName, *snaplen, *filter)
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	select {
+	case err := <-errC:
+		if err != nil {
+			slog.Error("capture failed", "error", err)
+			return 1
+		}
+	case sig := <-sigChan:
+		slog.Info("received signal, stopping capture", "signal", sig.String())
+		cancel()
+	}
+
+	// drain channel to ensure clean shutdown
+	for range errC {
+	}
+
+	return 0
+}
+
+func capture(ctx context.Context, interfaceName string, snaplen int, filter string) <-chan error {
+	errC := make(chan error, 1)
+
+	handle, err := pcap.OpenLive(interfaceName, int32(snaplen), true, pcap.BlockForever)
+	if err != nil {
+		errC <- fmt.Errorf("failed to open device %s: %w", interfaceName, err)
+		return errC
+	}
+
+	defaultFilter := genDefaultFilter(interfaceName)
+	switch {
+	case filter != "" && defaultFilter != "":
+		filter = fmt.Sprintf("(%s) and (%s)", defaultFilter, filter)
+		slog.Info("adjusted filter to avoid capturing SSH session", "filter", filter)
+	case filter == "" && defaultFilter != "":
+		filter = defaultFilter
+		slog.Info("set filter to avoid capturing SSH session", "filter", filter)
+	case defaultFilter == "": // && filter != ""
+		slog.Warn("failed to determine default filter, capture may include SSH session")
+	}
+
+	err = handle.SetBPFFilter(filter)
+	if err != nil {
+		handle.Close()
+		errC <- fmt.Errorf("failed to set BPF filter '%s': %w", filter, err)
+		return errC
+	}
+
+	pcapWriter := pcapgo.NewWriter(os.Stdout)
+	err = pcapWriter.WriteFileHeader(uint32(snaplen), handle.LinkType())
+	if err != nil {
+		handle.Close()
+		errC <- fmt.Errorf("failed to write pcap header: %w", err)
+		return errC
+	}
+
+	go func(ctx context.Context, h *pcap.Handle, w *pcapgo.Writer, errC chan error) {
+		defer h.Close()
+		s := gopacket.NewPacketSource(h, h.LinkType())
+
+		var err error
+	packetLoop:
+		for {
+			select {
+			case <-ctx.Done():
+				h.Close()
+			case packet, ok := <-s.Packets():
+				if !ok {
+					break packetLoop
+				} else if packet == nil {
+					continue
+				}
+
+				err = w.WritePacket(packet.Metadata().CaptureInfo, packet.Data())
+				if err != nil {
+					break packetLoop
+				}
+			}
+		}
+
+		errC <- err
+	}(ctx, handle, pcapWriter, errC)
+
+	return errC
+}
+
+func genDefaultFilter(iface string) string {
+	var port int
+	switch iface {
+	case "lo":
+		port = 2222
+		return "not tcp port 2222"
+	case "eth0":
+		port = envoySSHPort()
+	default:
+		port = -1
+	}
+	if port < 0 {
+		return ""
+	}
+
+	return fmt.Sprintf("not tcp port %d", port)
+}
+
+func envoySSHPort() int {
+	instancePortsJSON, ok := os.LookupEnv("CF_INSTANCE_PORTS")
+	if !ok {
+		return -1
+	}
+
+	var instancePorts []struct {
+		InternalTLSProxy int `json:"internal_tls_proxy"`
+		Internal         int `json:"internal"`
+		// ExternalTlsProxy int `json:"external_tls_proxy"`
+		// External         int `json:"external"`
+	}
+	err := json.Unmarshal([]byte(instancePortsJSON), &instancePorts)
+	if err != nil {
+		slog.Warn("failed to parse CF_INSTANCE_PORTS", "error", err)
+		return -1
+	}
+
+	for _, p := range instancePorts {
+		if p.Internal == 2222 {
+			return p.InternalTLSProxy
+		}
+	}
+
+	return -1
+}
