Skip to content

Commit 6be5b58

Browse files
committed
Add agent-managed firewall with nftables
Move firewall management from stemcell scripts to bosh-agent to properly handle Jammy containers running on Noble hosts (cgroup v2 inherited from host). - Create platform/firewall package with nftables implementation using github.com/google/nftables library (pure Go, no C deps) - Add SetupFirewall() to Platform interface, called during bootstrap - Add 'bosh-agent firewall-allow <service>' CLI command for monit wrapper - Detect cgroup version at runtime from /proc/self/cgroup - NATS firewall only enabled for Jammy (Noble uses ephemeral credentials) - Gracefully handle missing base firewall (warn, don't fail)
1 parent 0761dfb commit 6be5b58

124 files changed

Lines changed: 15059 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

agent/bootstrap.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ func (boot bootstrap) Run() (err error) { //nolint:gocyclo
100100
return bosherr.WrapError(err, "Setting up networking")
101101
}
102102

103+
if err = boot.platform.SetupFirewall(settings.GetMbusURL()); err != nil {
104+
return bosherr.WrapError(err, "Setting up firewall")
105+
}
106+
103107
if err = boot.platform.SetupRawEphemeralDisks(settings.RawEphemeralDiskSettings()); err != nil {
104108
return bosherr.WrapError(err, "Setting up raw ephemeral disk")
105109
}

go.mod

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ require (
1515
github.com/coreos/go-iptables v0.8.0
1616
github.com/gofrs/uuid v4.4.0+incompatible
1717
github.com/golang/mock v1.6.0
18+
github.com/google/nftables v0.2.0
1819
github.com/google/uuid v1.6.0
1920
github.com/kevinburke/ssh_config v1.4.0
2021
github.com/masterzen/winrm v0.0.0-20250927112105-5f8e6c707321
@@ -67,9 +68,12 @@ require (
6768
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
6869
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
6970
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
71+
github.com/josharian/native v1.1.0 // indirect
7072
github.com/jpillora/backoff v1.0.0 // indirect
7173
github.com/klauspost/compress v1.18.3 // indirect
7274
github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786 // indirect
75+
github.com/mdlayher/netlink v1.7.2 // indirect
76+
github.com/mdlayher/socket v0.5.0 // indirect
7377
github.com/moby/sys/userns v0.1.0 // indirect
7478
github.com/nats-io/nkeys v0.4.14 // indirect
7579
github.com/nats-io/nuid v1.0.1 // indirect

go.sum

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
109109
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
110110
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
111111
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
112+
github.com/google/nftables v0.2.0 h1:PbJwaBmbVLzpeldoeUKGkE2RjstrjPKMl6oLrfEJ6/8=
113+
github.com/google/nftables v0.2.0/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4=
112114
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc=
113115
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
114116
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -135,6 +137,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
135137
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
136138
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
137139
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
140+
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
141+
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
138142
github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
139143
github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
140144
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
@@ -157,6 +161,10 @@ github.com/masterzen/winrm v0.0.0-20250927112105-5f8e6c707321 h1:AKIJL2PfBX2uie0
157161
github.com/masterzen/winrm v0.0.0-20250927112105-5f8e6c707321/go.mod h1:JajVhkiG2bYSNYYPYuWG7WZHr42CTjMTcCjfInRNCqc=
158162
github.com/maxbrunsfeld/counterfeiter/v6 v6.12.1 h1:D4O2wLxB384TS3ohBJMfolnxb4qGmoZ1PnWNtit8LYo=
159163
github.com/maxbrunsfeld/counterfeiter/v6 v6.12.1/go.mod h1:RuJdxo0oI6dClIaMzdl3hewq3a065RH65dofJP03h8I=
164+
github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
165+
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
166+
github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI=
167+
github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI=
160168
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
161169
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
162170
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -220,6 +228,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
220228
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
221229
github.com/tidwall/transform v0.0.0-20201103190739-32f242e2dbde h1:AMNpJRc7P+GTwVbl8DkK2I9I8BBUzNiHuH/tlxrpan0=
222230
github.com/tidwall/transform v0.0.0-20201103190739-32f242e2dbde/go.mod h1:MvrEmduDUz4ST5pGZ7CABCnOU5f3ZiOAZzT6b1A6nX8=
231+
github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
232+
github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
223233
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
224234
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
225235
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=

main/agent.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
boshapp "github.com/cloudfoundry/bosh-agent/v2/app"
1414
"github.com/cloudfoundry/bosh-agent/v2/infrastructure/agentlogger"
1515
"github.com/cloudfoundry/bosh-agent/v2/platform"
16+
boshfirewall "github.com/cloudfoundry/bosh-agent/v2/platform/firewall"
1617
)
1718

1819
const mainLogTag = "main"
@@ -81,6 +82,9 @@ func main() {
8182
case "compile":
8283
compileTarball(cmd, os.Args[2:])
8384
return
85+
case "firewall-allow":
86+
handleFirewallAllow(os.Args[2:])
87+
return
8488
}
8589
}
8690
asyncLog := logger.NewAsyncWriterLogger(logger.LevelDebug, os.Stderr)
@@ -103,3 +107,35 @@ func newSignalableLogger(logger logger.Logger) logger.Logger {
103107
signalableLogger, _ := agentlogger.NewSignalableLogger(logger, c)
104108
return signalableLogger
105109
}
110+
111+
// handleFirewallAllow handles the "bosh-agent firewall-allow <service>" CLI command.
112+
// This is called by processes (like monit) that need firewall access to local services.
113+
func handleFirewallAllow(args []string) {
114+
if len(args) < 1 {
115+
fmt.Fprintf(os.Stderr, "Usage: bosh-agent firewall-allow <service>\n")
116+
fmt.Fprintf(os.Stderr, "Allowed services: %v\n", boshfirewall.AllowedServices)
117+
os.Exit(1)
118+
}
119+
120+
service := boshfirewall.Service(args[0])
121+
122+
// Create minimal logger for CLI command
123+
log := logger.NewLogger(logger.LevelError)
124+
125+
// Create firewall manager
126+
firewallMgr, err := boshfirewall.NewNftablesFirewall(log)
127+
if err != nil {
128+
fmt.Fprintf(os.Stderr, "Error creating firewall manager: %s\n", err)
129+
os.Exit(1)
130+
}
131+
132+
// Get parent PID (the process that called us)
133+
callerPID := os.Getppid()
134+
135+
if err := firewallMgr.AllowService(service, callerPID); err != nil {
136+
fmt.Fprintf(os.Stderr, "Error allowing service: %s\n", err)
137+
os.Exit(1)
138+
}
139+
140+
fmt.Printf("Firewall exception added for service: %s (caller PID: %d)\n", service, callerPID)
141+
}

platform/dummy_platform.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,10 @@ func (p dummyPlatform) SetupRecordsJSONPermission(path string) error {
562562
return nil
563563
}
564564

565+
func (p dummyPlatform) SetupFirewall(mbusURL string) error {
566+
return nil
567+
}
568+
565569
func (p dummyPlatform) Shutdown() error {
566570
return nil
567571
}

platform/firewall/cgroup_linux.go

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
//go:build linux
2+
3+
package firewall
4+
5+
import (
6+
"fmt"
7+
"os"
8+
"strings"
9+
10+
cgroups "github.com/containerd/cgroups/v3"
11+
)
12+
13+
// DetectCgroupVersion detects the cgroup version at runtime by checking
14+
// whether the system is using unified (v2) or legacy (v1) cgroup hierarchy.
15+
// This correctly handles:
16+
// - Jammy VM on Jammy host: Detects cgroup v1
17+
// - Jammy container on Noble host: Detects cgroup v2 (inherits from host!)
18+
// - Noble anywhere: Detects cgroup v2
19+
func DetectCgroupVersion() (CgroupVersion, error) {
20+
if cgroups.Mode() == cgroups.Unified {
21+
return CgroupV2, nil
22+
}
23+
return CgroupV1, nil
24+
}
25+
26+
// GetProcessCgroup gets the cgroup identity for a process by reading /proc/<pid>/cgroup
27+
func GetProcessCgroup(pid int, version CgroupVersion) (ProcessCgroup, error) {
28+
cgroupFile := fmt.Sprintf("/proc/%d/cgroup", pid)
29+
data, err := os.ReadFile(cgroupFile)
30+
if err != nil {
31+
return ProcessCgroup{}, fmt.Errorf("reading %s: %w", cgroupFile, err)
32+
}
33+
34+
if version == CgroupV2 {
35+
return parseCgroupV2(string(data))
36+
}
37+
return parseCgroupV1(string(data))
38+
}
39+
40+
// parseCgroupV2 extracts the cgroup path from /proc/<pid>/cgroup for cgroup v2
41+
// Format: "0::/system.slice/bosh-agent.service"
42+
func parseCgroupV2(data string) (ProcessCgroup, error) {
43+
for _, line := range strings.Split(data, "\n") {
44+
line = strings.TrimSpace(line)
45+
if strings.HasPrefix(line, "0::") {
46+
path := strings.TrimPrefix(line, "0::")
47+
return ProcessCgroup{
48+
Version: CgroupV2,
49+
Path: path,
50+
}, nil
51+
}
52+
}
53+
return ProcessCgroup{}, fmt.Errorf("cgroup v2 path not found in /proc/self/cgroup")
54+
}
55+
56+
// parseCgroupV1 extracts the cgroup info from /proc/<pid>/cgroup for cgroup v1
57+
// Format: "12:net_cls,net_prio:/system.slice/bosh-agent.service"
58+
func parseCgroupV1(data string) (ProcessCgroup, error) {
59+
// Look for net_cls controller which is used for firewall matching
60+
for _, line := range strings.Split(data, "\n") {
61+
line = strings.TrimSpace(line)
62+
if strings.Contains(line, "net_cls") {
63+
parts := strings.SplitN(line, ":", 3)
64+
if len(parts) >= 3 {
65+
return ProcessCgroup{
66+
Version: CgroupV1,
67+
Path: parts[2],
68+
// ClassID will be set when the process is added to the cgroup
69+
}, nil
70+
}
71+
}
72+
}
73+
74+
// Fallback: return empty path, will use classid-based matching
75+
return ProcessCgroup{
76+
Version: CgroupV1,
77+
}, nil
78+
}
79+
80+
// ReadOperatingSystem reads the operating system name from the BOSH-managed file
81+
func ReadOperatingSystem() (string, error) {
82+
data, err := os.ReadFile("/var/vcap/bosh/etc/operating_system")
83+
if err != nil {
84+
return "", err
85+
}
86+
return strings.TrimSpace(string(data)), nil
87+
}

platform/firewall/cgroup_other.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
//go:build !linux
2+
3+
package firewall
4+
5+
import "fmt"
6+
7+
// DetectCgroupVersion is not supported on non-Linux platforms
8+
func DetectCgroupVersion() (CgroupVersion, error) {
9+
return CgroupV1, fmt.Errorf("cgroup detection not supported on this platform")
10+
}
11+
12+
// GetProcessCgroup is not supported on non-Linux platforms
13+
func GetProcessCgroup(pid int, version CgroupVersion) (ProcessCgroup, error) {
14+
return ProcessCgroup{}, fmt.Errorf("cgroup not supported on this platform")
15+
}
16+
17+
// ReadOperatingSystem is not supported on non-Linux platforms
18+
func ReadOperatingSystem() (string, error) {
19+
return "", fmt.Errorf("operating system detection not supported on this platform")
20+
}

platform/firewall/firewall.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package firewall
2+
3+
// Service represents a local service that can be protected by firewall
4+
type Service string
5+
6+
const (
7+
ServiceMonit Service = "monit"
8+
// Future services can be added here
9+
)
10+
11+
// AllowedServices is the list of services that can be requested via CLI
12+
var AllowedServices = []Service{ServiceMonit}
13+
14+
// CgroupVersion represents the cgroup hierarchy version
15+
type CgroupVersion int
16+
17+
const (
18+
CgroupV1 CgroupVersion = 1
19+
CgroupV2 CgroupVersion = 2
20+
)
21+
22+
// ProcessCgroup represents a process's cgroup identity
23+
type ProcessCgroup struct {
24+
Version CgroupVersion
25+
Path string // For cgroup v2: full path like "/system.slice/bosh-agent.service"
26+
ClassID uint32 // For cgroup v1: net_cls classid
27+
}
28+
29+
//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
30+
//counterfeiter:generate . Manager
31+
32+
// Manager manages firewall rules for local service access
33+
type Manager interface {
34+
// SetupAgentRules sets up the agent's own firewall exceptions during bootstrap.
35+
// Called once during agent bootstrap after networking is configured.
36+
// mbusURL is the NATS URL for setting up NATS firewall rules (Jammy only).
37+
SetupAgentRules(mbusURL string) error
38+
39+
// AllowService opens firewall for the calling process to access a service.
40+
// Returns error if service is not in AllowedServices.
41+
// Called by external processes via "bosh-agent firewall-allow <service>".
42+
AllowService(service Service, callerPID int) error
43+
44+
// Cleanup removes all agent-managed firewall rules.
45+
// Called during agent shutdown (optional).
46+
Cleanup() error
47+
}
48+
49+
// IsAllowedService checks if a service is in the allowed list
50+
func IsAllowedService(s Service) bool {
51+
for _, allowed := range AllowedServices {
52+
if s == allowed {
53+
return true
54+
}
55+
}
56+
return false
57+
}

0 commit comments

Comments
 (0)