|
20 | 20 | # with remote DNS (socks5h). Without that wiring in sandd, this test hangs at |
21 | 21 | # "daemon failed to connect"; with it, the daemon connects and exec works. |
22 | 22 | # |
23 | | -# ASYMMETRY (mirrors Nebula, deliberately): |
24 | | -# - controller: NET_ADMIN + /dev/net/tun. It is the INFRA side (in-cluster), and |
25 | | -# accepting INBOUND mesh connections on a normal listening socket needs a real |
26 | | -# TUN. This is not the constrained side, so privilege here is fine. |
27 | | -# - daemon: UNPRIVILEGED, userspace-networking. This is the TENANT side (the GPU |
28 | | -# workload container) — exactly what must work without NET_ADMIN/TUN. |
| 23 | +# BOTH SIDES RUN USERSPACE-NETWORKING (Server(connect="tunnel") and sandd --tunnel |
| 24 | +# each start `tailscaled --tun=userspace-networking`). The controller keeps |
| 25 | +# NET_ADMIN + /dev/net/tun only as a harmless fallback / to mirror the infra side |
| 26 | +# of Nebula; in userspace mode tailscaled forwards inbound mesh TCP to the local |
| 27 | +# :8765 listener, so no kernel TUN is actually required. The daemon is deliberately |
| 28 | +# UNPRIVILEGED (no NET_ADMIN/TUN) — that is the TENANT side (the GPU workload |
| 29 | +# container) and the exact constraint this test proves works. |
29 | 30 | # |
30 | 31 | # Orchestrated by python/tests/test_e2e_tunnel.py (mints the auth key between |
31 | 32 | # `up headscale` and `up controller daemon`). Not part of the default e2e run. |
@@ -58,27 +59,29 @@ services: |
58 | 59 | - -c |
59 | 60 | - | |
60 | 61 | set -e |
61 | | - echo "[ctrl] starting tailscaled (kernel TUN, infra side)" |
62 | | - tailscaled --state=/var/lib/tailscale/tailscaled.state & |
63 | | - sleep 3 |
64 | | - tailscale up \ |
65 | | - --authkey="${SANDD_TUNNEL_AUTH_KEY:-}" \ |
66 | | - --login-server=http://headscale:8080 \ |
67 | | - --hostname=controller \ |
68 | | - --accept-routes |
69 | | - echo "[ctrl] mesh IP: $$(tailscale ip -4)" |
| 62 | + # Do NOT run tailscaled/`tailscale up` here: Server(connect="tunnel") |
| 63 | + # brings up its own tailscaled (--tun=userspace-networking) and runs |
| 64 | + # `tailscale up` internally (server/src/lib.rs setup_tunnel_controller). |
| 65 | + # A manual `tailscale up` first would set --hostname, and the Server's |
| 66 | + # second `up` (which omits it) then fails tailscale's "must mention all |
| 67 | + # non-default flags" guard. The tailnet hostname comes from the OS |
| 68 | + # hostname instead, which compose sets via `hostname: controller` below |
| 69 | + # -> MagicDNS name controller.sandd.local. |
70 | 70 | python3 -u << 'PYEOF' |
71 | 71 | import os, time |
72 | 72 | from sandd import Server, TunnelConfig |
73 | 73 | cfg = TunnelConfig(authkey=os.environ["SANDD_TUNNEL_AUTH_KEY"], |
74 | 74 | server="http://headscale:8080") |
75 | | - # The server joins the mesh via the running tailscaled/TUN above and |
76 | | - # listens on :8765 across it. |
| 75 | + # Server joins the mesh (userspace-networking) and listens on :8765; |
| 76 | + # tailscale proxies inbound mesh connections to this local listener. |
77 | 77 | server = Server(host="0.0.0.0", port=8765, connect="tunnel", tunnel_config=cfg) |
78 | 78 | print("[ctrl] controller ready, waiting for daemons", flush=True) |
79 | 79 | seen = set() |
80 | 80 | while True: |
81 | | - for did in server.list_daemons(): |
| 81 | + # list_daemons() returns DaemonInfo objects; exec() and the log |
| 82 | + # markers want the plain id string (d.id), not the object repr. |
| 83 | + for d in server.list_daemons(): |
| 84 | + did = d.id |
82 | 85 | if did not in seen: |
83 | 86 | seen.add(did) |
84 | 87 | print(f"[ctrl] DAEMON_CONNECTED {did}", flush=True) |
|
0 commit comments