Skip to content

Commit 862f8b2

Browse files
drewmalindrewmalin
authored
fix(BRE2-657): Update iptables to Allow Docker Egress, Docker Bridge Communication (#86)
* docker egress and tests * add tests to lifecycle validation * use network * fmt * see if tests fail * explicit pull * see if shadeform test fails * error message
1 parent ceb2fc2 commit 862f8b2

4 files changed

Lines changed: 186 additions & 4 deletions

File tree

‎internal/validation/suite.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,16 @@ func RunInstanceLifecycleValidation(t *testing.T, config ProviderConfig) {
129129
require.NoError(t, err, "ValidateDockerFirewallBlocksPort should pass - docker port should be blocked by iptables")
130130
})
131131

132+
t.Run("ValidateDockerFirewallAllowsEgress", func(t *testing.T) {
133+
err := v1.ValidateDockerFirewallAllowsEgress(ctx, client, instance, ssh.GetTestPrivateKey())
134+
require.NoError(t, err, "ValidateDockerFirewallAllowsEgress should pass - egress should be allowed")
135+
})
136+
137+
t.Run("ValidateDockerFirewallAllowsContainerToContainerCommunication", func(t *testing.T) {
138+
err := v1.ValidateDockerFirewallAllowsContainerToContainerCommunication(ctx, client, instance, ssh.GetTestPrivateKey())
139+
require.NoError(t, err, "ValidateDockerFirewallAllowsContainerToContainerCommunication should pass - container to container communication should be allowed")
140+
})
141+
132142
if capabilities.IsCapable(v1.CapabilityStopStartInstance) && instance.Stoppable {
133143
t.Run("ValidateStopStartInstance", func(t *testing.T) {
134144
err := v1.ValidateStopStartInstance(ctx, client, instance)
@@ -322,6 +332,16 @@ func RunFirewallValidation(t *testing.T, config ProviderConfig, opts FirewallVal
322332
require.NoError(t, err, "ValidateDockerFirewallBlocksPort should pass - docker port should be blocked")
323333
})
324334

335+
t.Run("ValidateDockerFirewallAllowsEgress", func(t *testing.T) {
336+
err := v1.ValidateDockerFirewallAllowsEgress(ctx, client, instance, ssh.GetTestPrivateKey())
337+
require.NoError(t, err, "ValidateDockerFirewallAllowsEgress should pass - egress should be allowed")
338+
})
339+
340+
t.Run("ValidateDockerFirewallAllowsContainerToContainerCommunication", func(t *testing.T) {
341+
err := v1.ValidateDockerFirewallAllowsContainerToContainerCommunication(ctx, client, instance, ssh.GetTestPrivateKey())
342+
require.NoError(t, err, "ValidateDockerFirewallAllowsContainerToContainerCommunication should pass - container to container communication should be allowed")
343+
})
344+
325345
// Test that SSH port is accessible (sanity check)
326346
t.Run("ValidateSSHPortAccessible", func(t *testing.T) {
327347
err := v1.ValidateFirewallAllowsPort(ctx, client, instance, ssh.GetTestPrivateKey(), instance.SSHPort)

‎v1/networking_validation.go‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"fmt"
77
"net"
88
"net/http"
9+
"strings"
910
"time"
1011

1112
"github.com/brevdev/cloud/internal/ssh"
@@ -141,6 +142,144 @@ func ValidateDockerFirewallBlocksPort(ctx context.Context, client CloudInstanceR
141142
return nil
142143
}
143144

145+
func ValidateDockerFirewallAllowsEgress(ctx context.Context, client CloudInstanceReader, instance *Instance, privateKey string) error {
146+
var err error
147+
instance, err = WaitForInstanceLifecycleStatus(ctx, client, instance, LifecycleStatusRunning, PendingToRunningTimeout)
148+
if err != nil {
149+
return fmt.Errorf("failed to wait for instance running: %w", err)
150+
}
151+
152+
publicIP := instance.PublicIP
153+
if publicIP == "" {
154+
return fmt.Errorf("public IP is not available for instance %s", instance.CloudID)
155+
}
156+
157+
sshClient, err := ssh.ConnectToHost(ctx, ssh.ConnectionConfig{
158+
User: instance.SSHUser,
159+
HostPort: fmt.Sprintf("%s:%d", publicIP, instance.SSHPort),
160+
PrivKey: privateKey,
161+
})
162+
if err != nil {
163+
return fmt.Errorf("failed to SSH into instance: %w", err)
164+
}
165+
defer func() { _ = sshClient.Close() }()
166+
167+
dockerCmd, err := setupDockerCommand(ctx, sshClient, instance.CloudID)
168+
if err != nil {
169+
return err
170+
}
171+
172+
// Pull the alpine image
173+
cmd := fmt.Sprintf(
174+
"%s pull alpine",
175+
dockerCmd,
176+
)
177+
_, stderr, err := sshClient.RunCommand(ctx, cmd)
178+
if err != nil {
179+
return fmt.Errorf("failed to pull alpine image: %w, stderr: %s", err, stderr)
180+
}
181+
182+
// Start a Docker container to ping Google's DNS server
183+
cmd = fmt.Sprintf(
184+
"%s run --rm alpine ping -c 3 8.8.8.8",
185+
dockerCmd,
186+
)
187+
stdout, stderr, err := sshClient.RunCommand(ctx, cmd)
188+
if err != nil {
189+
return fmt.Errorf("failed to connect to Google's DNS server: %w, stderr: %s", err, stderr)
190+
}
191+
if !strings.Contains(stdout, "3 packets transmitted, 3 packets received") {
192+
return fmt.Errorf("expected successful ping, got: %s", stdout)
193+
}
194+
195+
return nil
196+
}
197+
198+
func ValidateDockerFirewallAllowsContainerToContainerCommunication(ctx context.Context, client CloudInstanceReader, instance *Instance, privateKey string) error {
199+
var err error
200+
instance, err = WaitForInstanceLifecycleStatus(ctx, client, instance, LifecycleStatusRunning, PendingToRunningTimeout)
201+
if err != nil {
202+
return fmt.Errorf("failed to wait for instance running: %w", err)
203+
}
204+
205+
publicIP := instance.PublicIP
206+
if publicIP == "" {
207+
return fmt.Errorf("public IP is not available for instance %s", instance.CloudID)
208+
}
209+
210+
sshClient, err := ssh.ConnectToHost(ctx, ssh.ConnectionConfig{
211+
User: instance.SSHUser,
212+
HostPort: fmt.Sprintf("%s:%d", publicIP, instance.SSHPort),
213+
PrivKey: privateKey,
214+
})
215+
if err != nil {
216+
return fmt.Errorf("failed to SSH into instance: %w", err)
217+
}
218+
defer func() { _ = sshClient.Close() }()
219+
220+
dockerCmd, err := setupDockerCommand(ctx, sshClient, instance.CloudID)
221+
if err != nil {
222+
return err
223+
}
224+
225+
// Create a docker network
226+
networkName := "firewall-test-network"
227+
cmd := fmt.Sprintf(
228+
"%s network create %s",
229+
dockerCmd, networkName,
230+
)
231+
_, stderr, err := sshClient.RunCommand(ctx, cmd)
232+
if err != nil {
233+
return fmt.Errorf("failed to create docker network: %w, stderr: %s", err, stderr)
234+
}
235+
236+
// Pull the alpine image
237+
cmd = fmt.Sprintf(
238+
"%s pull alpine",
239+
dockerCmd,
240+
)
241+
_, stderr, err = sshClient.RunCommand(ctx, cmd)
242+
if err != nil {
243+
return fmt.Errorf("failed to pull alpine image: %w, stderr: %s", err, stderr)
244+
}
245+
246+
// Pull the nginx image
247+
cmd = fmt.Sprintf(
248+
"%s pull nginx:alpine",
249+
dockerCmd,
250+
)
251+
_, stderr, err = sshClient.RunCommand(ctx, cmd)
252+
if err != nil {
253+
return fmt.Errorf("failed to pull nginx image: %w, stderr: %s", err, stderr)
254+
}
255+
256+
// Start a Docker container in the background
257+
containerName := "firewall-test-container-to-container"
258+
cmd = fmt.Sprintf(
259+
"%s run -d --name %s --network %s nginx:alpine",
260+
dockerCmd, containerName, networkName,
261+
)
262+
_, stderr, err = sshClient.RunCommand(ctx, cmd)
263+
if err != nil {
264+
return fmt.Errorf("failed to start docker container: %w, stderr: %s", err, stderr)
265+
}
266+
267+
// Start a second Docker container to connect to the first container
268+
cmd = fmt.Sprintf(
269+
"%s run --network %s --rm alpine wget -q -O- http://%s",
270+
dockerCmd, networkName, containerName,
271+
)
272+
stdout, stderr, err := sshClient.RunCommand(ctx, cmd)
273+
if err != nil {
274+
return fmt.Errorf("failed to connect to nginx container: %w, stderr: %s", err, stderr)
275+
}
276+
277+
if !strings.Contains(stdout, "Welcome to nginx") {
278+
return fmt.Errorf("expected successful wget, got: %s", stdout)
279+
}
280+
return nil
281+
}
282+
144283
// setupDockerCommand ensures Docker is available and returns the command to use (always with sudo)
145284
func setupDockerCommand(ctx context.Context, sshClient *ssh.Client, instanceID CloudProviderInstanceID) (string, error) {
146285
// Check if Docker is available

‎v1/providers/nebius/instance.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1810,6 +1810,10 @@ func generateIPTablesCommands() []string {
18101810
commands := []string{
18111811
"iptables -F DOCKER-USER",
18121812
"iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT",
1813+
"iptables -A DOCKER-USER -i docker0 ! -o docker0 -j ACCEPT",
1814+
"iptables -A DOCKER-USER -i br+ ! -o br+ -j ACCEPT",
1815+
"iptables -A DOCKER-USER -i docker0 -o docker0 -j ACCEPT",
1816+
"iptables -A DOCKER-USER -i br+ -o br+ -j ACCEPT",
18131817
"iptables -A DOCKER-USER -i lo -j ACCEPT",
18141818
"iptables -A DOCKER-USER -j DROP",
18151819
"iptables -A DOCKER-USER -j RETURN", // Expected by Docker

‎v1/providers/shadeform/firewall.go‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,26 @@ const (
1515
ufwDefaultAllowPort2222 = "ufw allow 2222/tcp"
1616
ufwForceEnable = "ufw --force enable"
1717

18-
ipTablesResetDockerUserChain = "iptables -F DOCKER-USER"
19-
ipTablesAllowDockerUserOutbound = "iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT"
18+
// Clear DOCKER-USER policy.
19+
ipTablesResetDockerUserChain = "iptables -F DOCKER-USER"
20+
21+
// Allow return traffic.
22+
ipTablesAllowDockerUserOutbound = "iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT"
23+
24+
// Allow containers to initiate outbound traffic (default bridge + user-defined bridges).
25+
ipTablesAllowDockerUserOutboundInit0 = "iptables -A DOCKER-USER -i docker0 ! -o docker0 -j ACCEPT"
26+
ipTablesAllowDockerUserOutboundInit1 = "iptables -A DOCKER-USER -i br+ ! -o br+ -j ACCEPT"
27+
28+
// Allow container-to-container on the same bridge.
29+
ipTablesAllowDockerUserDockerToDocker0 = "iptables -A DOCKER-USER -i docker0 -o docker0 -j ACCEPT"
30+
ipTablesAllowDockerUserDockerToDocker1 = "iptables -A DOCKER-USER -i br+ -o br+ -j ACCEPT"
31+
32+
// Allow inbound traffic on the loopback interface.
2033
ipTablesAllowDockerUserInpboundLoopback = "iptables -A DOCKER-USER -i lo -j ACCEPT"
21-
ipTablesDropDockerUserInbound = "iptables -A DOCKER-USER -j DROP"
22-
ipTablesReturnDockerUser = "iptables -A DOCKER-USER -j RETURN"
34+
35+
// Drop everything else.
36+
ipTablesDropDockerUserInbound = "iptables -A DOCKER-USER -j DROP"
37+
ipTablesReturnDockerUser = "iptables -A DOCKER-USER -j RETURN"
2338
)
2439

2540
func (c *ShadeformClient) GenerateFirewallScript(firewallRules v1.FirewallRules) (string, error) {
@@ -63,6 +78,10 @@ func (c *ShadeformClient) getIPTablesCommands() []string {
6378
commands := []string{
6479
ipTablesResetDockerUserChain,
6580
ipTablesAllowDockerUserOutbound,
81+
ipTablesAllowDockerUserOutboundInit0,
82+
ipTablesAllowDockerUserOutboundInit1,
83+
ipTablesAllowDockerUserDockerToDocker0,
84+
ipTablesAllowDockerUserDockerToDocker1,
6685
ipTablesAllowDockerUserInpboundLoopback,
6786
ipTablesDropDockerUserInbound,
6887
ipTablesReturnDockerUser, // Expected by Docker

0 commit comments

Comments
 (0)