diff --git a/AGENTS.md b/AGENTS.md index e2228a47..5392f8b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ other two where the boundary matters: | Document | Covers | Start here when you're touching... | | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| [docs/agents/ARCHITECTURE-CNI.md](docs/agents/ARCHITECTURE-CNI.md) | The CNI attach chain: `galactic-cni` (installer), `galactic-veth`, `galactic-tap`, `galactic-ipam`, `galactic-bgp`, `galactic-route`, `vmtap-cni` | Pod/VM attach or detach behavior, CNI config fields/conflists, IPAM, the SRv6 uSID TC-BPF datapath's CNI-side registration, or anything under `cmd/galactic-{cni,veth,tap,ipam,bgp,route}`/`cmd/vmtap-cni`, `internal/cni*`, `internal/installer`, `internal/vmtap` | +| [docs/agents/ARCHITECTURE-CNI.md](docs/agents/ARCHITECTURE-CNI.md) | The CNI attach chain: `galactic-cni` (installer), `galactic-veth`, `galactic-tap`, `galactic-ipam`, `galactic-bgp`, `galactic-route` | Pod/VM attach or detach behavior, CNI config fields/conflists, IPAM, the SRv6 uSID TC-BPF datapath's CNI-side registration, or anything under `cmd/galactic-{cni,veth,tap,ipam,bgp,route}`, `internal/cni*`, `internal/installer` | | [docs/agents/ARCHITECTURE-ROUTER.md](docs/agents/ARCHITECTURE-ROUTER.md) | The BGP/EVPN control plane: `galactic-router` (BGPRouter/BGPPeer/BGPAdvertisement/BGPPolicy/BGPVRFInstance reconcilers, embedded GoBGP, GC) | BGP CRD reconciliation, GoBGP runtime behavior, EVPN path construction, orphaned-CRD/VRF garbage collection, or anything under `cmd/galactic-router`, `internal/reconcile`, `internal/runtime`, `internal/gc`, `internal/model`, `internal/hash` | | [docs/agents/ARCHITECTURE-GATEWAY.md](docs/agents/ARCHITECTURE-GATEWAY.md) | The edge XDP NAT+LB gateway: `galactic-gateway`, `NetworkGateway`/`NetworkRule` reconcilers, the edge XDP datapath | Ingress load-balancing/NAT, `NetworkGateway`/`NetworkRule` CRDs, Active-Active BGP placement, or anything under `cmd/galactic-gateway`, `internal/gateway`, `internal/plumbing/ebpf/edge{prog,map,attach}`, the `NetworkGateway`/`NetworkRule`-related code in `internal/controller` | diff --git a/README.md b/README.md index 0d2bb8f0..a4b0db16 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ See the [galactic DevContainer](./.devcontainer/galactic/) for development envir ### Production Deployment -Manifests for a real cluster live under [`config/`](./config/), composed with [Kustomize](https://kustomize.io). One command deploys the `galactic-system` namespace (labeled `pod-security.kubernetes.io/enforce: privileged` — every DaemonSet here needs it, for hostPath volumes, hostNetwork, and elevated capabilities), the `galactic-cni` DaemonSet, both `galactic-router` roles — `tenant` (per-node, runs everywhere except control-plane nodes) and `tenant-control` (BGP route reflector, opt-in — stays at zero replicas until nodes are labeled `galactic.datumapis.com/node: control`) — and the `vmtap-cni` DaemonSet (a second, standalone CNI plugin that gives Unikraft microVMs managed by `kraftlet` access to the pod's Cilium-assigned identity; see [`docs/vmtap-cni/configuration.md`](./docs/vmtap-cni/configuration.md)): +Manifests for a real cluster live under [`config/`](./config/), composed with [Kustomize](https://kustomize.io). One command deploys the `galactic-system` namespace (labeled `pod-security.kubernetes.io/enforce: privileged` — every DaemonSet here needs it, for hostPath volumes, hostNetwork, and elevated capabilities), the `galactic-cni` DaemonSet, and both `galactic-router` roles — `tenant` (per-node, runs everywhere except control-plane nodes) and `tenant-control` (BGP route reflector, opt-in — stays at zero replicas until nodes are labeled `galactic.datumapis.com/node: control`): ```bash kubectl apply -k config/ @@ -122,7 +122,7 @@ task # list available tasks ```bash task build # produces bin/{galactic-cni,galactic-veth,galactic-tap,galactic-ipam, - # galactic-bgp,galactic-route,galactic-router,galactic-gateway,vmtap-cni} + # galactic-bgp,galactic-route,galactic-router,galactic-gateway} task lint # golangci-lint + yamlfmt; lint-fix applies safe auto-fixes task ci # full pipeline: lint → build → test:unit → test:e2e ``` diff --git a/Taskfile.yaml b/Taskfile.yaml index b49f4dd4..e845c93f 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -143,7 +143,6 @@ tasks: - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-router ./cmd/galactic-router - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-gateway ./cmd/galactic-gateway - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-nat66 ./cmd/galactic-nat66 - - go build -ldflags "{{.LDFLAGS}}" -o bin/vmtap-cni ./cmd/vmtap-cni - GOBIN={{.LOCALBIN}} go install github.com/containernetworking/plugins/plugins/main/host-device@v1.9.1 ## diff --git a/cmd/galactic-tap/main.go b/cmd/galactic-tap/main.go index 3e08e7eb..7cd955ab 100644 --- a/cmd/galactic-tap/main.go +++ b/cmd/galactic-tap/main.go @@ -25,8 +25,7 @@ const ( The tap master plugin in the galactic CNI chain, for VM-based workloads (Kata, Firecracker, kraftlet/Unikraft) attaching directly to a galactic VPC - network. Unrelated to vmtap-cni, which is chained after Cilium's own CNI - plugin for a different purpose entirely (see its own doc comment). + network. Find more information at: https://www.datum.net/docs` ) diff --git a/cmd/vmtap-cni/main.go b/cmd/vmtap-cni/main.go deleted file mode 100644 index 4dc93985..00000000 --- a/cmd/vmtap-cni/main.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package main - -import ( - "fmt" - "log" - "os" - "os/signal" - "strings" - "syscall" - "time" - - "github.com/containernetworking/cni/pkg/version" - "github.com/spf13/cobra" - "golang.org/x/term" - - "go.datum.net/galactic/internal/metadata" - "go.datum.net/galactic/internal/vmtap" -) - -const ( - appName = "vmtap-cni" - - appDesc = `vmtap CNI Plugin - - Chained CNI plugin giving a Unikraft/kraftlet microVM access to the pod's - real Cilium-assigned identity via a tap device and tc-redirect. Must be - chained after Cilium's own CNI plugin in the pod's primary conflist. - - Find more information at: https://www.datum.net/docs` -) - -func newRootCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: appName, - Short: strings.Split(appDesc, "\n")[0], - Long: appDesc, - RunE: func(cmd *cobra.Command, _ []string) error { - if ok, _ := cmd.Flags().GetBool("build-info"); ok { - fmt.Println(metadata.BuildInfo(appName)) - return nil - } - if ok, _ := cmd.Flags().GetBool("version"); ok { - fmt.Printf("%s version %s\n", appName, metadata.Version) - return nil - } - if os.Getenv("CNI_COMMAND") == "VERSION" { - return version.All.Encode(os.Stdout) - } - - // Real CNI runtimes always pipe the network config JSON on - // stdin and close it. If stdin is an interactive terminal - // instead, no config will ever arrive and skel's blocking - // stdin read would hang forever — print version info instead. - if term.IsTerminal(int(os.Stdin.Fd())) { - fmt.Printf("%s version %s\n", appName, metadata.Version) - fmt.Printf("CNI protocol versions supported: %s\n", strings.Join(version.All.SupportedVersions(), ", ")) - return nil - } - - vmtap.RunPlugin() - return nil - }, - } - - cmd.Flags().Bool("build-info", false, "Print build information and exit") - cmd.Flags().BoolP("version", "V", false, "Print version and exit") - cmd.AddCommand(newPatchConflistCommand()) - return cmd -} - -// newPatchConflistCommand runs the conflist-chaining installer step: see -// vmtap.RunPatchLoop for why this loops instead of running once. It is -// meant to run as a long-lived container (not an init container) in the -// vmtap-cni DaemonSet — see config/vmtap-cni/daemonset.yaml. -func newPatchConflistCommand() *cobra.Command { - var ( - cniNetDir string - ciliumGlob string - pollInterval time.Duration - ) - - cmd := &cobra.Command{ - Use: "patch-conflist", - Short: "Chain vmtap-cni into Cilium's conflist and keep re-patching it", - RunE: func(cmd *cobra.Command, _ []string) error { - ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) - defer stop() - vmtap.RunPatchLoop(ctx, cniNetDir, ciliumGlob, pollInterval) - return nil - }, - } - cmd.Flags().StringVar(&cniNetDir, "cni-net-dir", "/host/etc/cni/net.d", "Host CNI conflist directory to patch") - cmd.Flags().StringVar(&ciliumGlob, "cilium-glob", "*cilium*.conflist", - "Glob (within --cni-net-dir) matching Cilium's own conflist file") - cmd.Flags().DurationVar(&pollInterval, "poll-interval", 10*time.Second, - "How often to re-check the conflist for the vmtap-cni entry") - return cmd -} - -func main() { - if err := newRootCommand().Execute(); err != nil { - log.Fatalf("error: %v", err) - } -} diff --git a/config/fabric-router/daemonset.yaml b/config/fabric-router/daemonset.yaml index 8b7eaa8e..7829ccd0 100644 --- a/config/fabric-router/daemonset.yaml +++ b/config/fabric-router/daemonset.yaml @@ -10,7 +10,7 @@ spec: matchLabels: app.kubernetes.io/name: fabric-router # Unlike every other DaemonSet in config/ (galactic-cni, galactic-router, - # galactic-gateway, vmtap-cni), this one owns the eBGP session to the + # galactic-gateway), this one owns the eBGP session to the # physical fabric switch, not an in-cluster control-plane relationship. # Bouncing FRR here drops the node's only underlay path, so every overlay # SRv6/EVPN path riding on it blackholes until the eBGP session @@ -46,8 +46,8 @@ spec: # depends on before it can start — so this must tolerate NotReady the # same way a CNI plugin does, or it never gets scheduled early enough. # Blanket-tolerate all taints, same as every other DaemonSet in - # config/ (galactic-cni, galactic-router, galactic-gateway, - # vmtap-cni). Placement is controlled entirely by the node affinity + # config/ (galactic-cni, galactic-router, galactic-gateway). + # Placement is controlled entirely by the node affinity # below (edge/control/gateway roles) — tolerations don't need to be # scoped, and scoping them risks silently blocking fabric-router if # an unenumerated taint (a new role taint, a maintenance taint, etc.) diff --git a/config/kustomization.yaml b/config/kustomization.yaml index f144f47c..a29ea4c9 100644 --- a/config/kustomization.yaml +++ b/config/kustomization.yaml @@ -2,4 +2,3 @@ resources: - galactic-system - galactic-cni - galactic-router - - vmtap-cni diff --git a/config/vmtap-cni/daemonset.yaml b/config/vmtap-cni/daemonset.yaml deleted file mode 100644 index a88e7634..00000000 --- a/config/vmtap-cni/daemonset.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: vmtap-cni - namespace: galactic-system - labels: - app.kubernetes.io/name: vmtap-cni -spec: - selector: - matchLabels: - app.kubernetes.io/name: vmtap-cni - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - app.kubernetes.io/name: vmtap-cni - spec: - serviceAccountName: vmtap-cni - # There's no rbac.yaml for vmtap-cni at all (internal/vmtap has no - # k8s client — it only patches a local conflist file and manages - # TAP/TC state), so the default ServiceAccount token would grant this - # pod zero capability while still being mounted for no reason. - automountServiceAccountToken: false - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - priorityClassName: system-node-critical - securityContext: - seccompProfile: - type: RuntimeDefault - tolerations: - - operator: Exists - # Placeholder node signal: only nodes actually running kraftlet-managed - # Unikraft workloads need this plugin chained into their conflist. - # "galactic.datumapis.com/node: kraftlet" is not an established label - # anywhere else in this repo — it stands in for whatever pod/node-level - # signal gets decided per the open item in - # .local/kraftlet-cilium-tap-plan.md section 7 ("Decide the pod-level - # signal (annotation vs. RuntimeClass)..."). Update or remove this - # affinity block once that decision is made. - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: DoesNotExist - - key: galactic.datumapis.com/node - operator: In - values: - - kraftlet - initContainers: - - name: install-cni-bin - # ":latest" is a placeholder only, matching the convention in - # config/galactic-cni/daemonset.yaml — no such tag is ever pushed to GHCR. - # vmtap-cni ships inside the galactic-cni image (see - # containers/galactic-cni/Dockerfile) rather than a separate one. - image: ghcr.io/datum-cloud/galactic-cni:latest - command: ["cp", "/vmtap-cni", "/host/opt/cni/bin/vmtap-cni"] - securityContext: - runAsUser: 0 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - memory: 64Mi - volumeMounts: - - name: cni-bin-dir - mountPath: /host/opt/cni/bin - containers: - # patch-conflist chains vmtap-cni into Cilium's conflist and keeps - # re-checking it — see internal/vmtap.RunPatchLoop and the - # conflist-chaining open item in - # .local/kraftlet-cilium-tap-plan.md section 7. This is a - # best-effort convergence loop, not a verified solution: nothing - # here guarantees the patch lands before Cilium starts servicing - # ADD calls on a freshly booted node. - - name: patch-conflist - image: ghcr.io/datum-cloud/galactic-cni:latest - command: ["/vmtap-cni", "patch-conflist"] - securityContext: - runAsUser: 0 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - resources: - requests: - cpu: 5m - memory: 16Mi - limits: - memory: 32Mi - volumeMounts: - - name: cni-net-dir - mountPath: /host/etc/cni/net.d - volumes: - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate diff --git a/config/vmtap-cni/kustomization.yaml b/config/vmtap-cni/kustomization.yaml deleted file mode 100644 index e37e13c8..00000000 --- a/config/vmtap-cni/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - serviceaccount.yaml - - daemonset.yaml diff --git a/config/vmtap-cni/serviceaccount.yaml b/config/vmtap-cni/serviceaccount.yaml deleted file mode 100644 index ad30767d..00000000 --- a/config/vmtap-cni/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vmtap-cni - namespace: galactic-system diff --git a/containers/galactic-cni/Dockerfile b/containers/galactic-cni/Dockerfile index 6406dcf2..c56a48c6 100644 --- a/containers/galactic-cni/Dockerfile +++ b/containers/galactic-cni/Dockerfile @@ -133,20 +133,6 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ -X go.datum.net/galactic/internal/metadata.GitURL=${GIT_URL}" \ -o galactic-route cmd/galactic-route/main.go -# Build vmtap-cni. It ships in this image rather than one of its own so the -# vmtap DaemonSet (config/vmtap-cni/) can reference the same published -# ghcr.io/datum-cloud/galactic-cni image instead of a second, separately -# published image. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ - -ldflags "-s -w \ - -X go.datum.net/galactic/internal/metadata.Version=${VERSION} \ - -X go.datum.net/galactic/internal/metadata.GitCommit=${GIT_COMMIT} \ - -X go.datum.net/galactic/internal/metadata.GitTreeState=${GIT_TREE_STATE} \ - -X go.datum.net/galactic/internal/metadata.BuildDate=${BUILD_DATE} \ - -X go.datum.net/galactic/internal/metadata.SPDXLicense=${SPDX_LICENSE} \ - -X go.datum.net/galactic/internal/metadata.GitURL=${GIT_URL}" \ - -o vmtap-cni cmd/vmtap-cni/main.go - # Build the host-device CNI plugin binary, used by galactic-veth to move # the veth endpoint into the container network namespace. GOBIN can't be set # for a cross-compiled `go install` (the toolchain rejects it outright when @@ -172,7 +158,6 @@ COPY --from=builder /workspace/galactic-tap /galactic-tap COPY --from=builder /workspace/galactic-ipam /galactic-ipam COPY --from=builder /workspace/galactic-bgp /galactic-bgp COPY --from=builder /workspace/galactic-route /galactic-route -COPY --from=builder /workspace/vmtap-cni /vmtap-cni COPY --from=builder /workspace/host-device /host-device COPY --from=builder /var/run/galactic-cni /var/run/galactic-cni @@ -189,7 +174,6 @@ COPY --from=production /galactic-tap /galactic-tap COPY --from=production /galactic-ipam /galactic-ipam COPY --from=production /galactic-bgp /galactic-bgp COPY --from=production /galactic-route /galactic-route -COPY --from=production /vmtap-cni /vmtap-cni COPY --from=production /host-device /host-device COPY --from=production /var/run/galactic-cni /var/run/galactic-cni diff --git a/docs/agents/ARCHITECTURE-CNI.md b/docs/agents/ARCHITECTURE-CNI.md index 779151fd..679e7c2c 100644 --- a/docs/agents/ARCHITECTURE-CNI.md +++ b/docs/agents/ARCHITECTURE-CNI.md @@ -33,7 +33,7 @@ the BGP/SRv6/eBPF state. IPAM is delegated to a separate `galactic-ipam` binary via the standard CNI IPAM delegation protocol, not chained. None of these five binaries are ever installed by hand: `galactic-cni`, a sixth binary that is never itself a CNI plugin (no NAD ever names it in a `"type"` -field), stages the other five (plus `vmtap-cni`, `host-device`) onto the +field), stages the other five (plus `host-device`) onto the host from its own `init`/`run` DaemonSet containers. All six ship in the same `ghcr.io/datum-cloud/galactic-cni` image — see [Repository Layout](#repository-layout) and @@ -92,8 +92,7 @@ galactic/ │ ├── galactic-tap/ # tap master plugin binary (VM workloads) │ ├── galactic-ipam/ # delegated CNI IPAM plugin binary │ ├── galactic-bgp/ # BGP/SRv6/eBPF publish plugin binary -│ ├── galactic-route/ # termination-route plugin binary (optional chain stage) -│ └── vmtap-cni/ # Cilium chain-conflist patcher for VM tap interfaces +│ └── galactic-route/ # termination-route plugin binary (optional chain stage) ├── internal/ │ ├── hostconf/ # Shared static-conflist HostConf schema/loader, │ │ # read by every binary in the chain @@ -121,8 +120,6 @@ galactic/ │ │ # prevResult alone, zero kernel-interface access │ ├── cniroute/ # galactic-route: termination-route plugin; │ │ # zero Kubernetes dependency -│ ├── vmtap/ # vmtap-cni: patches Cilium's own chain conflist -│ │ # to add a tap-interface stage for VM workloads │ ├── installer/ # galactic-cni DaemonSet init/run logic: binary │ │ # staging (all binaries above, one init │ │ # container/image), conflist templating, @@ -140,18 +137,16 @@ galactic/ │ └── vrf/ # Linux VRF create/delete/lookup ├── config/ │ ├── system/ # galactic-system namespace (shared by every component) -│ ├── cni/ # hostNetwork DaemonSet: `init` container stages -│ │ # every chain binary + host-device into -│ │ # /opt/cni/bin and writes the conflist + -│ │ # kubeconfig; `run` container refreshes -│ │ # credentials, manages the eBPF datapath, and -│ │ # serves gRPC health checks -│ └── vmtap/ # vmtap-cni DaemonSet: stages vmtap-cni and patches -│ # Cilium's chain conflist +│ └── cni/ # hostNetwork DaemonSet: `init` container stages +│ # every chain binary + host-device into +│ # /opt/cni/bin and writes the conflist + +│ # kubeconfig; `run` container refreshes +│ # credentials, manages the eBPF datapath, and +│ # serves gRPC health checks ├── deploy/ │ └── containerlab/ # ContainerLab lab topology and scripts └── containers/ - └── galactic-cni/ # galactic-veth/galactic-tap/-ipam/-bgp/-route + vmtap-cni + └── galactic-cni/ # galactic-veth/galactic-tap/-ipam/-bgp/-route # + host-device image (e2e test and production publish) ``` @@ -174,7 +169,6 @@ See [docs/cni-cmd-sequence.md](../cni/cni-cmd-sequence.md) for the full CNI ADD/ | `internal/cniipam` | `galactic-ipam` | Delegated CNI IPAM plugin (no k8s dependency) | | `internal/cnibgp` | `galactic-bgp` | BGP/SRv6/eBPF publish plugin (zero kernel-interface dependency) | | `internal/cniroute` | `galactic-route` | Termination-route plugin (no k8s dependency) | -| `internal/vmtap` | `vmtap-cni` | Cilium chain-conflist patcher for VM tap interfaces | | `internal/installer` | `galactic-cni` | DaemonSet `init`/`run` logic: binary staging (every chain binary), conflist/kubeconfig templating, credential refresh, gRPC health server | | `internal/plumbing/intf` | every CNI-chain binary | Interface naming, base62↔hex encoding | | `internal/plumbing/ebpf` | `galactic-cni` (attach/metrics via `run`), `galactic-bgp` (registration) | TC-BPF uSID datapath: preflight, uformat, prog, attach, usidmap, metrics | @@ -424,7 +418,6 @@ any shared, per-attachment kernel/CRD state — see the `cmdDel` note in | `internal/cniipam` | galactic-ipam | CNI IPAM delegation protocol (`cmdAdd`/`cmdDel`/`cmdCheck`/`cmdStatus`); explicit `"ipam"`-block contract; no k8s dependency | No | | `internal/cnibgp` | galactic-bgp | BGP/SRv6/eBPF publish: SID/Argument allocation + collision detection, `registerEBPFDatapath`/`unregisterEBPFDatapath`, `BGPVRFInstance`/`BGPAdvertisement` CRUD with retry; learns everything from `prevResult` | No | | `internal/cniroute` | galactic-route | Termination-route plugin: installs/rolls-back VRF-table routes; no k8s dependency | No | -| `internal/vmtap` | vmtap-cni | Patches Cilium's own chain conflist to add a tap-interface stage for VM workloads | No | | `internal/installer` | galactic-cni | DaemonSet `init`/`run` support: binary staging (every chain binary), node-identity check, conflist/kubeconfig templating, credential refresh ticker, log rotation, eBPF datapath lifecycle, gRPC health server | No | | `internal/plumbing/intf` | every CNI-chain binary | Deterministic interface naming (`G{vpc9}{att3}V/H/G`); base62↔hex encoding | No | | `internal/plumbing/ebpf` | galactic-cni (attach/metrics via `run`), galactic-bgp (registration) | TC-BPF uSID datapath: kernel preflight, uFMT bit-layout codec, compiled program + bindings, load/pin/attach lifecycle, map read/write API, Prometheus metrics | Yes (pinned BPF maps) | @@ -486,7 +479,7 @@ auto-detect directly (see Configuration above). **Publish pipeline:** `.github/workflows/publish.yaml`'s `publish-galactic-cni-image` job builds and pushes `ghcr.io/datum-cloud/galactic-cni`; `publish-kustomize-bundles` (which `needs` every per-binary image job, including the router and gateway ones) stamps that real tag into `config/galactic-cni` when it pushes `config/` as the `ghcr.io/datum-cloud/galactic-kustomize` OCI Kustomize bundle. **Container image:** -- `containers/galactic-cni/Dockerfile` — multi-stage build (golang builder → distroless → final Alpine stage for `iproute2`/`nsenter`); builds `galactic-cni` (the installer), `galactic-veth`, `galactic-tap`, `galactic-ipam`, `galactic-bgp`, `galactic-route`, `vmtap-cni`, and the delegated `host-device` CNI plugin binary, `ENTRYPOINT ["/galactic-cni"]`. Used both by `task test:e2e` (`scripts/ci.sh e2etest` builds it, tags `galactic-cni:e2e`, `kind load`s it into the ephemeral e2e cluster) and by `publish.yaml` (pushed as `ghcr.io/datum-cloud/galactic-cni`). Both the init container (`/galactic-cni init`) and the long-running container (`/galactic-cni run`) run this same image; the DaemonSet no longer shells out to an `install.sh` script, so the Alpine/`iproute2` final stage exists purely for e2e test needs (kernel `ip`/`nsenter` operations exercised via `task test:e2e`) rather than anything the installer subcommands require. Reusing the e2e-tested artifact for publish is preferred over maintaining a second, untested variant. +- `containers/galactic-cni/Dockerfile` — multi-stage build (golang builder → distroless → final Alpine stage for `iproute2`/`nsenter`); builds `galactic-cni` (the installer), `galactic-veth`, `galactic-tap`, `galactic-ipam`, `galactic-bgp`, `galactic-route`, and the delegated `host-device` CNI plugin binary, `ENTRYPOINT ["/galactic-cni"]`. Used both by `task test:e2e` (`scripts/ci.sh e2etest` builds it, tags `galactic-cni:e2e`, `kind load`s it into the ephemeral e2e cluster) and by `publish.yaml` (pushed as `ghcr.io/datum-cloud/galactic-cni`). Both the init container (`/galactic-cni init`) and the long-running container (`/galactic-cni run`) run this same image; the DaemonSet no longer shells out to an `install.sh` script, so the Alpine/`iproute2` final stage exists purely for e2e test needs (kernel `ip`/`nsenter` operations exercised via `task test:e2e`) rather than anything the installer subcommands require. Reusing the e2e-tested artifact for publish is preferred over maintaining a second, untested variant. **History:** the original `.github/workflows/release.yaml` built and pushed a single `ghcr.io/datum-cloud/galactic:{version,major.minor,major,sha}` image from a shared `containers/galactic/Dockerfile`, but that image only ever built `galactic-cni` while `config/galactic-router/base/daemonset.yaml` ran `command: [/galactic-router]` against it — the image advertised a binary it never built. Both were removed; per-binary Dockerfiles and `publish.yaml` (see [ARCHITECTURE-ROUTER.md](ARCHITECTURE-ROUTER.md#cicd)) fix this. @@ -497,7 +490,6 @@ auto-detect directly (see Configuration above). - **No binary's `cmdDel` tears down shared kernel/CRD state.** By design (see Key Design Decisions above) — cleanup of the VRF, routes, the eBPF `vrf_table` entry, and BGP CRDs is deferred to `galactic-router`'s asynchronous GC controller, not performed synchronously in any chain binary's `cmdDel`. The per-attachment host veth/tap interface is the one exception: it is private, not shared, so `cmdDel` deletes it directly instead of deferring it. - **`internal/plumbing/vrf` and `internal/cni/route` have no unit tests.** `vrf` requires `CAP_NET_ADMIN` and a real kernel; `route` (wrapped by `internal/cniroute`, which does have its own tests) was never backfilled with tests of its own when the CNI plugin-chain split moved its caller out of `internal/cni`. `internal/plumbing/intf` is fully unit-testable (pure functions only). Kernel-path coverage otherwise comes from the e2e suite (`task test:e2e`). - **The e2e suite doesn't cover `galactic-route` or `galactic-bgp`.** `TestCNITapInterface` (`tests/e2e/e2e_test.go`) only drives `galactic-tap`'s own ADD — verifying BGP/SRv6/eBPF publish end-to-end would need a `BGPRouter` CRD fixture and additional RBAC the test doesn't set up. This gap predates the CNI plugin-chain split too: the monolithic `galactic-cni` this replaced was never e2e-verified past its own CNI result shape either. -- **`vmtap-cni`/`internal/vmtap`** (a separate, Cilium-chain-conflist-patching binary for VM tap interfaces, unrelated to the `galactic-veth`/`galactic-tap`/`galactic-ipam`/`galactic-bgp`/`galactic-route` chain) has its own doc at `docs/vmtap-cni/configuration.md` — cross-referenced from [Repository Layout](#repository-layout) and the [Module / Package Reference](#module--package-reference) table above, but not otherwise elaborated on in this document. - **`galactic-cni`'s install DaemonSet is a Go installer, not a shell script.** `config/galactic-cni/configmap.yaml`/`install.sh` were deleted; `config/galactic-cni/daemonset.yaml` now runs `hostNetwork: true` with an `install-cni` init container (`command: ["/galactic-cni", "init"]`, calling `installer.Bootstrap`) and a `credential-refresh` main container (`command: ["/galactic-cni", "run"]`, calling `installer.Run`), both on the same image (see CI/CD above). `Bootstrap` writes every binary in the CNI chain to `/opt/cni/bin`, the static conflist to `/etc/cni/net.d/10-galactic.conflist`, and `ca.crt`/kubeconfig to `/var/lib/galactic` (chosen over `/etc/galactic` specifically so it lands under `/var`, the one path immutable-root distros like Talos allow hostPath writes to without a host-level `extraMounts` entry); `Run` refreshes the kubeconfig token every 300s and rotates the CNI log once it exceeds 10MB. `/opt/cni/bin` is fixed by the CNI/kubelet plugin-discovery convention and can't be relocated by this DaemonSet alone — on Talos it needs its own `extraMounts` entry in the machine config if it isn't writable by default. The `run` container also serves gRPC health checks on port `5180` (`livenessProbe`/`readinessProbe` in the DaemonSet spec), and `config/galactic-cni/rbac.yaml` grants `get` on `nodes` for `Bootstrap`'s node-identity check. - **The uSID TC-BPF datapath (`internal/plumbing/ebpf/prog/usid.c`) doesn't generate PMTUD ICMPv6 errors.** When `bpf_fib_lookup()` returns `BPF_FIB_LKUP_RET_FRAG_NEEDED` (egress route's MTU is smaller than the inner packet), the program counts `DROP_REASON_FIB_FRAG_NEEDED` and silently drops (`TC_ACT_SHOT`) rather than sending an ICMPv6 Packet Too Big back to the original sender — unlike the static-route `SEG6_LOCAL_ACTION_END_DT46` path this datapath replaces, where the kernel's own IPv6 stack emits that ICMP message. Accepted as a known cost of the TC-BPF cutover for this milestone; generating ICMPv6 PTB from the datapath itself is unscheduled future work, not planned for a specific milestone yet. The edge gateway's own XDP program has the identical gap for its own FIB lookup — see [ARCHITECTURE-GATEWAY.md](ARCHITECTURE-GATEWAY.md#known-constraints). @@ -529,4 +521,4 @@ auto-detect directly (see Configuration above). - The master plugin's ADD result is the runtime's authoritative CNI result for the whole chain — `galactic-route`/`galactic-bgp` (chained after it) both pass `prevResult` straight through unchanged. A successful ADD response does not by itself guarantee `galactic-bgp` has even run yet, let alone that the BGP CRDs exist. What it does guarantee: the master plugin (`galactic-veth`/`galactic-tap`) fetches its own attachment's `NetworkAttachmentDefinition` before creating any kernel state and fails ADD if `galactic-bgp` is missing from its conflist's `plugins` list (`internal/hostconf.VerifyChainIncludes`, `internal/nadpatch.VerifyChainComplete`) — a stale/hand-edited conflist that drops the entry entirely can no longer attach successfully with no path to its VPC ([#331](https://github.com/datum-cloud/galactic/issues/331)). This is presence-only, not position-aware: it does not confirm `galactic-bgp` actually ran, only that the conflist names it. - `types.PluginConf.PrevResult` (from `containernetworking/cni/pkg/types`) has JSON tag `"-"` and is **never populated** by plain `json.Unmarshal`; only the sibling `RawPrevResult map[string]interface{}` field actually receives the previous plugin's result. A pre-existing quirk of that library, not specific to this codebase — every CNI-chain package that reads prevResult (`internal/cni/ops_check.go`, `internal/cnibgp/prevresult.go`, `internal/cniroute/ops_add.go`) reads `RawPrevResult` for this reason. - No binary's `cmdDel` deletes the VRF, routes, the eBPF `vrf_table` entry, or `BGPAdvertisement`/`BGPVRFInstance` CRDs — each binary's own DEL only handles its own per-container bookkeeping (IPAM deallocation, guest-netns/host-device cleanup, and deleting this attachment's own private host veth/tap interface outright). Shared-resource cleanup is entirely the GC controller's job (`internal/gc`, part of `galactic-router` — see [ARCHITECTURE-ROUTER.md](ARCHITECTURE-ROUTER.md)), to avoid racing a concurrent ADD during pod restarts. -- Production images are published by `.github/workflows/publish.yaml` as separate per-binary images, not one shared image — see CI/CD above. `galactic-cni`'s own image carries all five CNI-chain binaries plus `vmtap-cni`/`host-device`, not just `galactic-cni` itself. +- Production images are published by `.github/workflows/publish.yaml` as separate per-binary images, not one shared image — see CI/CD above. `galactic-cni`'s own image carries all five CNI-chain binaries plus `host-device`, not just `galactic-cni` itself. diff --git a/docs/agents/ARCHITECTURE.md b/docs/agents/ARCHITECTURE.md index 956ba4e0..995399c8 100644 --- a/docs/agents/ARCHITECTURE.md +++ b/docs/agents/ARCHITECTURE.md @@ -11,7 +11,7 @@ gateway-role nodes), each with its own architecture document: | Document | Covers | | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | -| [ARCHITECTURE-CNI.md](ARCHITECTURE-CNI.md) | The CNI attach chain — `galactic-cni` (installer), `galactic-veth`, `galactic-tap`, `galactic-ipam`, `galactic-bgp`, `galactic-route`, `vmtap-cni` | +| [ARCHITECTURE-CNI.md](ARCHITECTURE-CNI.md) | The CNI attach chain — `galactic-cni` (installer), `galactic-veth`, `galactic-tap`, `galactic-ipam`, `galactic-bgp`, `galactic-route` | | [ARCHITECTURE-ROUTER.md](ARCHITECTURE-ROUTER.md) | The BGP/EVPN control plane — `galactic-router` | | [ARCHITECTURE-GATEWAY.md](ARCHITECTURE-GATEWAY.md) | The edge XDP NAT+LB gateway — `galactic-gateway`, `NetworkGateway`/`NetworkRule` | diff --git a/docs/architecture/containers.puml b/docs/architecture/containers.puml index 2b9e4169..807431ad 100644 --- a/docs/architecture/containers.puml +++ b/docs/architecture/containers.puml @@ -9,7 +9,7 @@ Person(tenant, "Tenant / Developer", "Deploys pods/VMs; authors NetworkRule spec Person_Ext(client, "External Client") System_Boundary(galactic, "Galactic") { - Container(cnichain, "galactic-veth CNI chain", "Go binaries: galactic-cni (installer), galactic-veth/galactic-tap (master plugins), galactic-ipam, galactic-bgp, galactic-route, vmtap-cni", "Wires pods/VMs into VPC networks: VRF, veth/tap, IPAM, SRv6 registration; writes BGP CRDs") + Container(cnichain, "galactic-veth CNI chain", "Go binaries: galactic-cni (installer), galactic-veth/galactic-tap (master plugins), galactic-ipam, galactic-bgp, galactic-route", "Wires pods/VMs into VPC networks: VRF, veth/tap, IPAM, SRv6 registration; writes BGP CRDs") Container(usidbpf, "SRv6 uSID datapath", "TC-BPF (cilium/ebpf)", "Kernel decap/encap for VPC traffic; loaded by galactic-cni, map entries registered by galactic-bgp") Container(router, "galactic-router", "Go, controller-runtime", "Reconciles BGPRouter/BGPPeer/BGPAdvertisement/BGPPolicy/BGPVRFInstance CRDs; runs orphaned-CRD/VRF GC") diff --git a/docs/vmtap-cni/configuration.md b/docs/vmtap-cni/configuration.md deleted file mode 100644 index 7d13651b..00000000 --- a/docs/vmtap-cni/configuration.md +++ /dev/null @@ -1,141 +0,0 @@ -# vmtap-cni Configuration - -`vmtap-cni` is a standalone CNI plugin, separate from `galactic-veth`, that gives a -Unikraft microVM managed by `kraftlet` access to the pod's real Cilium-assigned -identity. It has no VPC/VPCAttachment configuration and no Kubernetes API -dependency — it never creates a `BGPAdvertisement`, never touches a VRF, and -these pods have no `vpc`/`vpcattachment`. See -[.local/kraftlet-cilium-tap-plan.md](../../.local/kraftlet-cilium-tap-plan.md) -for the full design. - -> This plugin has not yet been validated against a real Cilium-managed cluster -> — see [Open items / unvalidated caveats](#open-items--unvalidated-caveats) below -> before relying on it in production. - -## How it works - -`vmtap-cni` must be **chained** after Cilium's own CNI plugin inside the pod's -primary conflist (e.g. appended into whatever `/etc/cni/net.d/*cilium*.conflist` -Cilium installs) — it is not delivered via a Multus `NetworkAttachmentDefinition`, -and it does not run standalone. Because it runs as part of the same `cmdAdd` -invocation that creates the pod's `eth0`, it receives Cilium's `prevResult` -(interface name, MAC, MTU, IPs) directly. - -On `ADD`, inside the pod's network namespace: - -1. Reads `eth0`'s state from `prevResult` (MAC, link MTU) and the kernel's - routing table (route MTU) — see [MTU handling](#mtu-handling) below. - **`eth0` itself is never modified.** -2. Creates a tap device (default name `tap0`) with its MTU set to the - resolved route MTU. -3. Installs a `clsact`-equivalent `ingress` qdisc and a `tc-mirred` redirect - filter on both `eth0` and the tap in each direction, so every packet - arriving on one is stolen and re-injected as egress on the other. -4. Returns a CNI result that copies `prevResult` forward (this plugin is the - last one in the chain, so its result is what the runtime hands back) and - appends the tap as a new interface entry, with `sandbox` set to - `CNI_CONTAINERID` instead of a netns path — see - [How kraftlet reads the result](#how-kraftlet-reads-the-result) below. - -`DEL` removes the redirect filters and the tap device; both `DEL` and `CHECK` -are idempotent per the CNI spec. Opening the tap's fd, handing it to the VMM, -and configuring the Unikraft guest's network stack (no runtime DHCP by -default) are **not** this plugin's job — that is `kraftlet`'s responsibility -once it reads this plugin's `ADD` result. - -## CNI Configuration JSON - -All fields are optional; the plugin runs against sane defaults when the -config block is empty (`{"type": "vmtap-cni"}` is a valid entry). - -| Field | Type | Default | Description | -| ----------------- | -------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | `bool` | `true` | Set to `false` to no-op this conflist entry (passes `prevResult` through unchanged) without removing it from the chain. | -| `tap_name` | `string` | `"tap0"` | Name of the tap device created inside the pod netns. | -| `owner_uid` | `int` | `0` (root) | Tap device owner uid, so a non-root VMM process can open its fd without `CAP_NET_ADMIN`. | -| `owner_gid` | `int` | `0` (root) | Tap device owner gid. | -| `filter_priority` | `int` | `1` | tc filter priority for the mirred redirect filters. Override if it collides with Cilium's own hooks — see the caveat below. | - -Standard CNI fields (`cniVersion`, `name`, `prevResult`) are inherited from the -conflist envelope, as for any chained plugin. - -### Example conflist entry - -```json -{ - "cniVersion": "1.0.0", - "name": "cilium", - "plugins": [ - { "type": "cilium-cni" }, - { "type": "vmtap-cni" } - ] -} -``` - -## MTU handling - -Cilium adjusts the pod's *route* MTU for overlay/tunnel overhead, which can -differ from `eth0`'s *link* MTU. `vmtap-cni` reads the actual route MTU from -the kernel routing table (the route's `MTU` metric, via netlink `RouteList`) -rather than copying the link MTU, and sets the tap's own MTU to that -resolved value — so the tap's `mtu` field in the CNI result already carries -the corrected number kraftlet needs, without a separate schema field. If no -route carries an explicit MTU metric, it falls back to `eth0`'s link MTU. - -## How kraftlet reads the result - -`vmtap-cni`'s `cmdAdd` runs at pod-sandbox-creation time, invoked by whatever -calls the chained conflist (typically containerd's CNI integration) — not by -`kraftlet` directly. `kraftlet`'s own VM-start code runs later, as a separate -step, and needs to read the same CNI result back out. - -This plugin's convention: the tap's interface entry in the CNI result sets -`sandbox` to `CNI_CONTAINERID` (the CRI sandbox ID) rather than a netns path, -since there is no netns for the VM the way there is for a container. -**This convention has not been confirmed against the actual kraftlet/CRI -integration** — see the open item below. - -## Cilium-specific caveats (unvalidated) - -These are called out explicitly because they are correctness risks this -plugin's code cannot self-verify — they require a real Cilium-managed -cluster to confirm: - -- **tc/bpf hook ordering**: the default `filter_priority` (`1`) has not been - checked against any specific Cilium version/datapath mode (veth vs. - netkit, tunnel vs. native routing) for a collision with Cilium's own - `clsact` bpf programs on `eth0`. -- **socketLB / kube-proxy replacement**: Cilium's socket-level load balancing - intercepts `connect()`/`sendmsg()` in the *host* kernel, which the guest's - own kernel never reaches. This needs a **cluster-wide** Cilium config - change (`socketLB.hostNamespaceOnly=true`), not anything this plugin can - set per-pod. -- **Envoy sidecar interaction**: unverified whether Envoy/Cilium's own - interception of traffic on `eth0` still works correctly once that traffic - is also being mirrored to/from the tap. -- **NetworkPolicy, Service routing, Hubble attribution**: all assumed to - keep working because `eth0`'s identity is untouched, but not yet tested - end-to-end. - -## Open items - -- **Conflist chaining.** `vmtap-cni` ships with a `patch-conflist` subcommand - (`internal/vmtap.RunPatchLoop`, wired into - [`config/vmtap-cni/daemonset.yaml`](../../config/vmtap-cni/daemonset.yaml)) that - polls for a `*cilium*.conflist` file and appends a `{"type": "vmtap-cni"}` - entry to it if missing. This is a best-effort convergence loop, **not** a - verified solution — it does not guarantee the patch lands before Cilium - starts servicing `ADD` calls on a freshly booted node, and it does not - remove its own entry on uninstall (manually edit Cilium's conflist back - out if needed). -- **Pod-level trigger signal.** Which pods get this conflist entry at all - (RuntimeClass vs. annotation) is not decided; `config/vmtap-cni/daemonset.yaml` - currently gates the whole DaemonSet on a placeholder node label - (`galactic.datumapis.com/node: kraftlet`) that does not exist anywhere else - in this repo. -- **kraftlet hand-off convention.** The `sandbox: CNI_CONTAINERID` convention - described above is this plugin's own choice, not something confirmed - against kraftlet's actual CRI/containerd integration. - -See [.local/kraftlet-cilium-tap-plan.md](../../.local/kraftlet-cilium-tap-plan.md) -section 7 for the full list. diff --git a/internal/config/cni.go b/internal/config/cni.go index 41a0735c..a14b9a93 100644 --- a/internal/config/cni.go +++ b/internal/config/cni.go @@ -32,10 +32,9 @@ const ( // own tc/bpf programs on the same native device ingress hook these // interfaces use, and this package's priority-1 default has not been // validated against every Cilium version/datapath mode for a - // collision -- see the same class of caveat internal/vmtap/config.go - // documents for its own filter_priority knob. Override here if a - // deployment's Cilium install needs this filter at a different - // priority to run in the intended order relative to Cilium's own. + // collision. Override here if a deployment's Cilium install needs + // this filter at a different priority to run in the intended order + // relative to Cilium's own. EnvCNIEBPFFilterPriority = "GALACTIC_CNI_EBPF_FILTER_PRIORITY" // EnvCNINAT66ShardSIDs is a comma-separated list of every live diff --git a/internal/vmtap/config.go b/internal/vmtap/config.go deleted file mode 100644 index 3611a3b4..00000000 --- a/internal/vmtap/config.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "encoding/json" - "errors" - "fmt" - - "github.com/containernetworking/cni/pkg/types" - "github.com/containernetworking/cni/pkg/version" -) - -// defaultTapName is the tap device name used when PluginConf.TapName is unset. -const defaultTapName = "tap0" - -// defaultFilterPriority is the tc filter priority used for the mirred -// redirect filters when PluginConf.FilterPriority is unset. Priority 1 is -// the highest (lowest-numbered) priority tc allows; this has not been -// validated against Cilium's own clsact priority on any specific -// version/datapath mode — see the tc/bpf hook ordering caveat in -// docs/vmtap-cni/configuration.md. Override via filter_priority if it -// collides. -const defaultFilterPriority = 1 - -// errNoPrevResult is returned when the plugin config carries no prevResult. -// vmtap-cni only makes sense chained after a plugin (Cilium) that has -// already configured the pod's primary interface — mirrors -// awslabs/tc-redirect-tap's NoPreviousResultError. -var errNoPrevResult = errors.New("vmtap-cni must be chained after a plugin that sets prevResult (e.g. cilium-cni)") - -// parseConf unmarshals the CNI configuration from stdin data, defaults -// optional fields, and parses prevResult into conf.PrevResult. Returns -// errNoPrevResult if no prevResult is present. -func parseConf(data []byte) (*PluginConf, error) { - conf := &PluginConf{} - if err := json.Unmarshal(data, conf); err != nil { - return nil, &types.Error{Code: 7, Msg: "invalid CNI config", Details: err.Error()} - } - - if conf.TapName == "" { - conf.TapName = defaultTapName - } - if conf.FilterPriority == 0 { - conf.FilterPriority = defaultFilterPriority - } - - if conf.RawPrevResult == nil { - return nil, &types.Error{Code: 7, Msg: errNoPrevResult.Error()} - } - if err := version.ParsePrevResult(&conf.PluginConf); err != nil { - return nil, &types.Error{Code: 6, Msg: fmt.Sprintf("parse prevResult: %v", err)} - } - if conf.PrevResult == nil { - return nil, &types.Error{Code: 7, Msg: errNoPrevResult.Error()} - } - - return conf, nil -} diff --git a/internal/vmtap/config_test.go b/internal/vmtap/config_test.go deleted file mode 100644 index ce2902c0..00000000 --- a/internal/vmtap/config_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "strings" - "testing" -) - -const validStdin = `{ - "cniVersion": "1.0.0", - "name": "cilium", - "type": "vmtap-cni", - "prevResult": { - "cniVersion": "1.0.0", - "interfaces": [{"name": "eth0", "mac": "aa:bb:cc:dd:ee:ff", "mtu": 1500}], - "ips": [{"address": "10.0.0.5/24", "gateway": "10.0.0.1", "interface": 0}] - } -}` - -func TestParseConfMissingPrevResult(t *testing.T) { - _, err := parseConf([]byte(`{"cniVersion": "1.0.0", "name": "cilium", "type": "vmtap-cni"}`)) - if err == nil { - t.Fatal("expected error for missing prevResult, got nil") - } - if !strings.Contains(err.Error(), "chained") { - t.Errorf("unexpected error message: %v", err) - } -} - -func TestParseConfInvalidJSON(t *testing.T) { - _, err := parseConf([]byte(`not json`)) - if err == nil { - t.Fatal("expected error for invalid JSON, got nil") - } -} - -func TestParseConfDefaults(t *testing.T) { - conf, err := parseConf([]byte(validStdin)) - if err != nil { - t.Fatalf("parseConf() = %v, want nil", err) - } - if conf.TapName != defaultTapName { - t.Errorf("TapName = %q, want %q", conf.TapName, defaultTapName) - } - if conf.FilterPriority != defaultFilterPriority { - t.Errorf("FilterPriority = %d, want %d", conf.FilterPriority, defaultFilterPriority) - } - if !conf.enabled() { - t.Error("enabled() = false, want true (default)") - } - if conf.PrevResult == nil { - t.Fatal("PrevResult = nil, want parsed result") - } -} - -func TestParseConfExplicitFields(t *testing.T) { - stdin := `{ - "cniVersion": "1.0.0", - "name": "cilium", - "type": "vmtap-cni", - "enabled": false, - "tap_name": "vmtap1", - "filter_priority": 42, - "prevResult": { - "cniVersion": "1.0.0", - "interfaces": [{"name": "eth0", "mac": "aa:bb:cc:dd:ee:ff", "mtu": 1500}] - } - }` - - conf, err := parseConf([]byte(stdin)) - if err != nil { - t.Fatalf("parseConf() = %v, want nil", err) - } - if conf.TapName != "vmtap1" { - t.Errorf("TapName = %q, want %q", conf.TapName, "vmtap1") - } - if conf.FilterPriority != 42 { - t.Errorf("FilterPriority = %d, want %d", conf.FilterPriority, 42) - } - if conf.enabled() { - t.Error("enabled() = true, want false (explicitly disabled)") - } -} diff --git a/internal/vmtap/conflist.go b/internal/vmtap/conflist.go deleted file mode 100644 index 98bdc3db..00000000 --- a/internal/vmtap/conflist.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" -) - -// PluginType is the "type" field vmtap-cni registers under in a conflist's -// "plugins" array, and the value PatchConflistDir looks for to decide -// whether a conflist has already been patched. -const PluginType = "vmtap-cni" - -// conflistEnvelope is the standard CNI conflist JSON structure. Plugins is -// kept as raw JSON so patching only ever appends a new element — existing -// plugin entries (Cilium's own, and any others already chained) are never -// re-marshaled and so cannot be reformatted or reordered by a patch. -type conflistEnvelope struct { - CNIVersion string `json:"cniVersion"` - Name string `json:"name"` - Plugins []json.RawMessage `json:"plugins"` -} - -// PatchConflistDir scans dir (non-recursively) for conflist files matching -// glob and appends a {"type": vmtap.PluginType} entry to each one's -// "plugins" array, unless already present. Returns the paths that were -// newly patched this call. -// -// This is a best-effort installer step, not a verified solution to the -// conflist-chaining problem flagged in -// .local/kraftlet-cilium-tap-plan.md section 7: it assumes Cilium's -// conflist already exists under dir by the time this runs (there is no -// cross-DaemonSet ordering guarantee for that — see RunPatchLoop, which -// polls specifically to tolerate arriving late) and it does not undo the -// patch on uninstall, so removing vmtap-cni from a cluster currently -// requires manually editing Cilium's conflist back out. Both are open -// items, not solved problems. -func PatchConflistDir(dir, glob string) ([]string, error) { - matches, err := filepath.Glob(filepath.Join(dir, glob)) - if err != nil { - return nil, fmt.Errorf("glob %q in %q: %w", glob, dir, err) - } - - var patched []string - for _, path := range matches { - changed, err := patchConflistFile(path) - if err != nil { - return patched, fmt.Errorf("patch %q: %w", path, err) - } - if changed { - patched = append(patched, path) - } - } - return patched, nil -} - -// patchConflistFile appends vmtap-cni's plugin entry to path's "plugins" -// array if not already present. Returns true if the file was modified. -func patchConflistFile(path string) (bool, error) { - data, err := os.ReadFile(path) - if err != nil { - return false, fmt.Errorf("read: %w", err) - } - - var env conflistEnvelope - if err := json.Unmarshal(data, &env); err != nil { - return false, fmt.Errorf("parse conflist: %w", err) - } - - for _, raw := range env.Plugins { - var meta struct { - Type string `json:"type"` - } - if err := json.Unmarshal(raw, &meta); err != nil { - continue - } - if meta.Type == PluginType { - return false, nil // already patched - } - } - - entry, err := json.Marshal(map[string]string{"type": PluginType}) - if err != nil { - return false, fmt.Errorf("marshal plugin entry: %w", err) - } - env.Plugins = append(env.Plugins, entry) - - out, err := json.MarshalIndent(env, "", " ") - if err != nil { - return false, fmt.Errorf("marshal patched conflist: %w", err) - } - - return true, writeFileAtomic(path, out) -} - -// writeFileAtomic writes data to path via a temp-file-plus-rename, so a -// CNI runtime reading the conflist concurrently never observes a partially -// written file — conflists are read on every pod ADD, including -// potentially while this patch is in flight. -func writeFileAtomic(path string, data []byte) error { - info, err := os.Stat(path) - if err != nil { - return fmt.Errorf("stat: %w", err) - } - - tmp := path + ".vmtap-cni.tmp" - if err := os.WriteFile(tmp, data, info.Mode()); err != nil { - return fmt.Errorf("write temp file: %w", err) - } - if err := os.Rename(tmp, path); err != nil { - _ = os.Remove(tmp) - return fmt.Errorf("rename temp file into place: %w", err) - } - return nil -} diff --git a/internal/vmtap/conflist_run.go b/internal/vmtap/conflist_run.go deleted file mode 100644 index dc580bb8..00000000 --- a/internal/vmtap/conflist_run.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "context" - "log/slog" - "time" -) - -// RunPatchLoop periodically re-applies PatchConflistDir until ctx is -// canceled. Running as a loop rather than a one-shot init container is a -// deliberate hedge against two ordering problems neither Kubernetes nor -// this plugin can otherwise guarantee: Cilium's own DaemonSet may not have -// written its conflist yet the first time this container starts, and -// Cilium can rewrite its conflist later (e.g. on an upgrade), which would -// silently drop vmtap-cni back out of the chain until the next patch. -// -// It is still not a complete answer to the conflist-chaining problem in -// .local/kraftlet-cilium-tap-plan.md section 7 — in particular, nothing -// here guarantees this patch lands before Cilium's own CNI starts -// servicing ADD calls on a freshly booted node, only that it eventually -// converges. -func RunPatchLoop(ctx context.Context, dir, glob string, interval time.Duration) { - patchOnce := func() { - patched, err := PatchConflistDir(dir, glob) - if err != nil { - slog.Error("conflist patch: failed", "dir", dir, "glob", glob, "err", err) - return - } - for _, path := range patched { - slog.Info("conflist patch: appended vmtap-cni to plugin chain", "path", path) - } - } - - patchOnce() - - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - patchOnce() - } - } -} diff --git a/internal/vmtap/conflist_test.go b/internal/vmtap/conflist_test.go deleted file mode 100644 index 05c18cbb..00000000 --- a/internal/vmtap/conflist_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" -) - -const ciliumConflist = `{ - "cniVersion": "1.0.0", - "name": "cilium", - "plugins": [ - {"type": "cilium-cni"} - ] -}` - -func TestPatchConflistDirAppendsEntry(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "05-cilium.conflist") - if err := os.WriteFile(path, []byte(ciliumConflist), 0644); err != nil { - t.Fatalf("write fixture: %v", err) - } - - patched, err := PatchConflistDir(dir, "*cilium*.conflist") - if err != nil { - t.Fatalf("PatchConflistDir() = %v, want nil", err) - } - if len(patched) != 1 || patched[0] != path { - t.Fatalf("patched = %v, want [%s]", patched, path) - } - - var env conflistEnvelope - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read patched file: %v", err) - } - if err := json.Unmarshal(data, &env); err != nil { - t.Fatalf("unmarshal patched file: %v", err) - } - if len(env.Plugins) != 2 { - t.Fatalf("plugins count = %d, want 2", len(env.Plugins)) - } - var last struct { - Type string `json:"type"` - } - if err := json.Unmarshal(env.Plugins[1], &last); err != nil { - t.Fatalf("unmarshal last plugin entry: %v", err) - } - if last.Type != PluginType { - t.Errorf("last plugin type = %q, want %q", last.Type, PluginType) - } -} - -func TestPatchConflistDirIdempotent(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "05-cilium.conflist") - if err := os.WriteFile(path, []byte(ciliumConflist), 0644); err != nil { - t.Fatalf("write fixture: %v", err) - } - - if _, err := PatchConflistDir(dir, "*cilium*.conflist"); err != nil { - t.Fatalf("first PatchConflistDir() = %v, want nil", err) - } - patched, err := PatchConflistDir(dir, "*cilium*.conflist") - if err != nil { - t.Fatalf("second PatchConflistDir() = %v, want nil", err) - } - if len(patched) != 0 { - t.Errorf("second call patched = %v, want empty (already patched)", patched) - } -} - -func TestPatchConflistDirNoMatch(t *testing.T) { - dir := t.TempDir() - - patched, err := PatchConflistDir(dir, "*cilium*.conflist") - if err != nil { - t.Fatalf("PatchConflistDir() = %v, want nil", err) - } - if len(patched) != 0 { - t.Errorf("patched = %v, want empty (no matching files)", patched) - } -} diff --git a/internal/vmtap/doc.go b/internal/vmtap/doc.go deleted file mode 100644 index a183477b..00000000 --- a/internal/vmtap/doc.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -// Package vmtap implements a chained CNI plugin that gives a Unikraft -// microVM (managed by kraftlet) access to the pod's real Cilium-assigned -// identity via a tap device, using the tc-redirect pattern Kata Containers -// uses for its "tcfilter" endpoint type. -// -// vmtap-cni is unrelated to the VPC/SRv6 dataplane implemented by -// internal/cni — it never touches a VRF, never creates a BGPAdvertisement, -// and these pods have no vpc/vpcattachment. It must be chained after -// Cilium's own CNI plugin in the pod's primary conflist and requires a -// prevResult describing the interface Cilium already configured (typically -// eth0). On ADD it creates a tap device in the same network namespace, adds -// mirred redirect tc filters between the two links in both directions, and -// reports the tap as a synthetic VM-facing interface in the CNI result — -// eth0 itself is never modified. DEL removes the filters and the tap -// device; both DEL and CHECK are idempotent per the CNI spec. -// -// See .local/kraftlet-cilium-tap-plan.md for the design this package -// implements, including caveats around route-MTU, Cilium's socketLB, and -// tc/bpf hook ordering that still require empirical validation on a real -// cluster. -package vmtap diff --git a/internal/vmtap/ops_add.go b/internal/vmtap/ops_add.go deleted file mode 100644 index ce6402c8..00000000 --- a/internal/vmtap/ops_add.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "fmt" - "log/slog" - - "github.com/containernetworking/cni/pkg/skel" - "github.com/containernetworking/cni/pkg/types" - type100 "github.com/containernetworking/cni/pkg/types/100" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/vishvananda/netlink" -) - -// cmdAdd creates a tap device in the pod's network namespace and installs -// bidirectional tc-mirred redirects between it and the interface the -// preceding plugin (Cilium) already configured, without ever modifying that -// interface. See doc.go and .local/kraftlet-cilium-tap-plan.md section 3. -func cmdAdd(args *skel.CmdArgs) error { - conf, err := parseConf(args.StdinData) - if err != nil { - return err - } - - slog.Info("ADD: starting", "containerID", args.ContainerID, "netns", args.Netns, "ifName", args.IfName) - - if !conf.enabled() { - slog.Info("ADD: plugin disabled via config, passing prevResult through unchanged", - "containerID", args.ContainerID) - return types.PrintResult(conf.PrevResult, conf.CNIVersion) - } - - var result *type100.Result - err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { - info, err := resolveRedirectInterface(conf, args.IfName) - if err != nil { - return fmt.Errorf("resolve redirect interface %q: %w", args.IfName, err) - } - - redirectLink, err := netlink.LinkByName(args.IfName) - if err != nil { - return fmt.Errorf("find redirect interface %q: %w", args.IfName, err) - } - - // Tap MTU is the resolved route MTU, not eth0's link MTU — this is - // what fixes the Cilium/Kata MTU mismatch caveat (section 4 of the - // plan) without needing a dedicated schema field: the tap's own Mtu - // in the CNI result is already the corrected value. - tapLink, err := addTap(conf.TapName, info.routeMTU, conf.OwnerUID, conf.OwnerGID) - if err != nil { - return fmt.Errorf("create tap %q: %w", conf.TapName, err) - } - - if err := addRedirect(redirectLink, tapLink, conf.FilterPriority); err != nil { - return fmt.Errorf("add redirect %s->%s: %w", args.IfName, conf.TapName, err) - } - if err := addRedirect(tapLink, redirectLink, conf.FilterPriority); err != nil { - return fmt.Errorf("add redirect %s->%s: %w", conf.TapName, args.IfName, err) - } - slog.Debug("ADD: redirect installed", "containerID", args.ContainerID, - "redirect", args.IfName, "tap", conf.TapName, "routeMTU", info.routeMTU) - - result, err = buildResult(conf, tapLink, args.ContainerID) - return err - }) - if err != nil { - return fmt.Errorf("ADD in netns %q: %w", args.Netns, err) - } - - slog.Info("ADD: tap and redirect ready", "containerID", args.ContainerID, "tap", conf.TapName) - return types.PrintResult(result, conf.CNIVersion) -} diff --git a/internal/vmtap/ops_check.go b/internal/vmtap/ops_check.go deleted file mode 100644 index e68daa8a..00000000 --- a/internal/vmtap/ops_check.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "errors" - "fmt" - "log/slog" - - "github.com/containernetworking/cni/pkg/skel" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/vishvananda/netlink" -) - -// cmdCheck validates that the tap device and both redirect filters are -// still present, and that the redirect interface (eth0) has not drifted -// from what prevResult recorded — the strongest guarantee this plugin can -// give that it is still leaving Cilium's interface alone. -func cmdCheck(args *skel.CmdArgs) error { - conf, err := parseConf(args.StdinData) - if err != nil { - return err - } - slog.Info("CHECK: starting", "containerID", args.ContainerID) - - if !conf.enabled() { - slog.Info("CHECK: plugin disabled via config, nothing to check", "containerID", args.ContainerID) - return nil - } - - var errs []error - if nsErr := ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { - checkInNetns(conf, args, &errs) - return nil - }); nsErr != nil { - errs = append(errs, fmt.Errorf("enter netns %q: %w", args.Netns, nsErr)) - } - - if len(errs) > 0 { - err := fmt.Errorf("CHECK failed: %w", errors.Join(errs...)) - slog.Error("CHECK: failed", "err", err, "containerID", args.ContainerID) - return err - } - slog.Info("CHECK: passed", "containerID", args.ContainerID) - return nil -} - -// checkInNetns runs the actual state checks once inside the pod netns, -// appending any failures to errs so cmdCheck can report all of them at once -// rather than stopping at the first. -func checkInNetns(conf *PluginConf, args *skel.CmdArgs, errs *[]error) { - redirectLink, err := netlink.LinkByName(args.IfName) - if err != nil { - *errs = append(*errs, fmt.Errorf("redirect interface %q: %w", args.IfName, err)) - return - } - - tapLink, err := netlink.LinkByName(conf.TapName) - if err != nil { - *errs = append(*errs, fmt.Errorf("tap %q: %w", conf.TapName, err)) - return - } - - if ok, err := hasRedirectFilter(redirectLink, conf.FilterPriority); err != nil { - *errs = append(*errs, fmt.Errorf("check redirect filter on %q: %w", args.IfName, err)) - } else if !ok { - *errs = append(*errs, fmt.Errorf("missing redirect filter on %q at priority %d", args.IfName, conf.FilterPriority)) - } - - if ok, err := hasRedirectFilter(tapLink, conf.FilterPriority); err != nil { - *errs = append(*errs, fmt.Errorf("check redirect filter on %q: %w", conf.TapName, err)) - } else if !ok { - *errs = append(*errs, fmt.Errorf("missing redirect filter on %q at priority %d", conf.TapName, conf.FilterPriority)) - } - - info, err := resolveRedirectInterface(conf, args.IfName) - if err != nil { - *errs = append(*errs, fmt.Errorf("resolve prevResult interface %q: %w", args.IfName, err)) - return - } - if info.mac != "" && redirectLink.Attrs().HardwareAddr.String() != info.mac { - *errs = append(*errs, fmt.Errorf("interface %q MAC changed: expected %q, got %q", - args.IfName, info.mac, redirectLink.Attrs().HardwareAddr.String())) - } - if info.mtu > 0 && redirectLink.Attrs().MTU != info.mtu { - *errs = append(*errs, fmt.Errorf("interface %q MTU changed: expected %d, got %d", - args.IfName, info.mtu, redirectLink.Attrs().MTU)) - } -} diff --git a/internal/vmtap/ops_del.go b/internal/vmtap/ops_del.go deleted file mode 100644 index fb166ee6..00000000 --- a/internal/vmtap/ops_del.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "fmt" - "log/slog" - - "github.com/containernetworking/cni/pkg/skel" - "github.com/containernetworking/cni/pkg/types" - type100 "github.com/containernetworking/cni/pkg/types/100" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/vishvananda/netlink" -) - -// cmdDel removes the redirect filter and tap device this plugin created. -// DEL is idempotent per the CNI spec: a missing netns, tap, or filter is not -// an error. The redirect interface (eth0) is never touched — it was never -// modified by ADD, so there is nothing to restore on it. -func cmdDel(args *skel.CmdArgs) error { - slog.Info("DEL: starting", "containerID", args.ContainerID, "netns", args.Netns) - - conf, err := parseConf(args.StdinData) - if err != nil { - // Config didn't parse (or carries no prevResult) — nothing to - // identify which tap/priority to clean up. Still return success: - // DEL must be idempotent even against garbage state. - slog.Warn("DEL: failed to parse config, skipping cleanup", "err", err, "containerID", args.ContainerID) - return types.PrintResult(&type100.Result{}, "1.0.0") - } - - if err := ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { - if link, linkErr := netlink.LinkByName(args.IfName); linkErr == nil { - if err := deleteRedirect(link, conf.FilterPriority); err != nil { - return fmt.Errorf("remove redirect on %q: %w", args.IfName, err) - } - } - return deleteTap(conf.TapName) - }); err != nil { - // Netns is commonly already gone by the time DEL runs (the runtime - // tears it down before or concurrently with calling DEL) — that is - // the expected, non-error case, not a real cleanup failure. - if _, statErr := ns.GetNS(args.Netns); statErr != nil { - slog.Debug("DEL: netns already gone, nothing to clean up", "containerID", args.ContainerID, "netns", args.Netns) - return types.PrintResult(&type100.Result{}, conf.CNIVersion) - } - return fmt.Errorf("DEL in netns %q: %w", args.Netns, err) - } - - slog.Info("DEL: done", "containerID", args.ContainerID) - return types.PrintResult(&type100.Result{}, conf.CNIVersion) -} diff --git a/internal/vmtap/prevresult.go b/internal/vmtap/prevresult.go deleted file mode 100644 index 7a299d44..00000000 --- a/internal/vmtap/prevresult.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "fmt" - - type100 "github.com/containernetworking/cni/pkg/types/100" - "github.com/vishvananda/netlink" -) - -// resolveRedirectInterface reads ifName's state out of pluginConf's already- -// parsed prevResult (MAC, link MTU) and, for the route MTU Cilium may have -// adjusted independently of the link MTU, out of the kernel routing table. -// Callers must already be running inside the target network namespace. -// -// Per .local/kraftlet-cilium-tap-plan.md section 1, this never mutates -// ifName — it is read-only. -func resolveRedirectInterface(conf *PluginConf, ifName string) (*redirectInterfaceInfo, error) { - prevResult, err := type100.GetResult(conf.PrevResult) - if err != nil { - return nil, fmt.Errorf("get prevResult: %w", err) - } - - var iface *type100.Interface - for _, i := range prevResult.Interfaces { - if i.Name == ifName { - iface = i - break - } - } - if iface == nil { - return nil, fmt.Errorf("prevResult does not describe an interface named %q", ifName) - } - - link, err := netlink.LinkByName(ifName) - if err != nil { - return nil, fmt.Errorf("find interface %q: %w", ifName, err) - } - - routeMTU, err := findRouteMTU(link) - if err != nil { - return nil, fmt.Errorf("resolve route MTU for %q: %w", ifName, err) - } - if routeMTU == 0 { - // No route carries an explicit MTU metric — fall back to the link - // MTU. This is the same fallback awslabs/tc-redirect-tap always - // takes (it never reads route MTU at all); see the MTU caveat in - // docs/vmtap-cni/configuration.md for when this fallback is wrong. - routeMTU = iface.Mtu - } - - return &redirectInterfaceInfo{ - name: ifName, - mac: iface.Mac, - mtu: iface.Mtu, - routeMTU: routeMTU, - }, nil -} - -// findRouteMTU returns the largest MTU metric set on any route bound to -// link, across both address families. Returns 0 if no route carries an -// explicit MTU (the common case when Cilium hasn't needed to shrink it for -// overlay/tunnel overhead). -func findRouteMTU(link netlink.Link) (int, error) { - var maxMTU int - for _, family := range []int{netlink.FAMILY_V4, netlink.FAMILY_V6} { - routes, err := netlink.RouteList(link, family) - if err != nil { - return 0, fmt.Errorf("list routes: %w", err) - } - for _, r := range routes { - if r.MTU > maxMTU { - maxMTU = r.MTU - } - } - } - return maxMTU, nil -} diff --git a/internal/vmtap/result.go b/internal/vmtap/result.go deleted file mode 100644 index c7c9c1c7..00000000 --- a/internal/vmtap/result.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "fmt" - - "github.com/containernetworking/cni/pkg/types" - type100 "github.com/containernetworking/cni/pkg/types/100" - "github.com/vishvananda/netlink" -) - -// buildResult copies prevResult forward (interfaces, IPs, routes — vmtap-cni -// is the last plugin in the chain, so its own result is what the CNI runtime -// hands back to the caller) and appends the tap device as a new interface -// entry. -// -// The tap's Sandbox field is set to containerID rather than a netns path — -// there is no netns for the VM the way there is for a container, so this -// plugin uses containerID (the CRI sandbox ID passed as CNI_CONTAINERID) as -// the hand-off key kraftlet is expected to read the CNI ADD result back out -// by. This convention is not yet confirmed against the actual kraftlet -// integration — see the "how kraftlet reads this plugin's result" open item -// in .local/kraftlet-cilium-tap-plan.md section 7. -// -// tapLink's MTU is expected to already equal the resolved route MTU (not -// eth0's link MTU) — see resolveRedirectInterface and the MTU caveat in -// docs/vmtap-cni/configuration.md — so no separate route-MTU field is -// needed: the tap's own Mtu communicates the corrected value. -func buildResult(conf *PluginConf, tapLink netlink.Link, containerID string) (*type100.Result, error) { - prevResult, err := type100.GetResult(conf.PrevResult) - if err != nil { - return nil, fmt.Errorf("get prevResult: %w", err) - } - - result := &type100.Result{ - CNIVersion: conf.CNIVersion, - Interfaces: append([]*type100.Interface{}, prevResult.Interfaces...), - IPs: append([]*type100.IPConfig{}, prevResult.IPs...), - Routes: append([]*types.Route{}, prevResult.Routes...), - DNS: prevResult.DNS, - } - - attrs := tapLink.Attrs() - result.Interfaces = append(result.Interfaces, &type100.Interface{ - Name: attrs.Name, - Mac: attrs.HardwareAddr.String(), - Mtu: attrs.MTU, - Sandbox: containerID, - }) - - return result, nil -} diff --git a/internal/vmtap/tap.go b/internal/vmtap/tap.go deleted file mode 100644 index 0c4507de..00000000 --- a/internal/vmtap/tap.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "fmt" - "log/slog" - - "github.com/vishvananda/netlink" -) - -// addTap creates a tap device with the given name, MTU, and owner uid/gid. -// Callers must already be running inside the target network namespace (see -// ns.WithNetNSPath in ops_add.go) — addTap never enters a namespace itself. -// Idempotent: if a tap device with this name already exists (crash-recovery -// or a retried ADD), its attributes are left as-is rather than recreated, -// mirroring internal/cni/tap.Add's repair behavior for a simpler case (no -// VRF enslavement here). -func addTap(name string, mtu int, ownerUID, ownerGID uint32) (netlink.Link, error) { - if existing, err := netlink.LinkByName(name); err == nil { - slog.Warn("vmtap: found existing tap from a previous ADD attempt, reusing", "tap", name) - return existing, nil - } - - tap := &netlink.Tuntap{ - LinkAttrs: netlink.LinkAttrs{ - Name: name, - MTU: mtu, - }, - Mode: netlink.TUNTAP_MODE_TAP, - Flags: netlink.TUNTAP_ONE_QUEUE | netlink.TUNTAP_VNET_HDR, - Owner: ownerUID, - Group: ownerGID, - } - - if err := netlink.LinkAdd(tap); err != nil { - return nil, fmt.Errorf("create tap %q: %w", name, err) - } - slog.Debug("vmtap: tap created", "tap", name, "mtu", mtu, "ownerUID", ownerUID, "ownerGID", ownerGID) - - link, err := netlink.LinkByName(name) - if err != nil { - return nil, fmt.Errorf("find created tap %q: %w", name, err) - } - - if err := netlink.LinkSetUp(link); err != nil { - return nil, fmt.Errorf("bring up tap %q: %w", name, err) - } - - return link, nil -} - -// deleteTap removes the named tap device. Idempotent — a missing device is -// not an error, per the CNI spec's DEL idempotency requirement. -func deleteTap(name string) error { - link, err := netlink.LinkByName(name) - if err != nil { - slog.Debug("vmtap: tap already gone, nothing to delete", "tap", name) - return nil - } - if err := netlink.LinkDel(link); err != nil { - return fmt.Errorf("delete tap %q: %w", name, err) - } - slog.Debug("vmtap: tap deleted", "tap", name) - return nil -} diff --git a/internal/vmtap/tc.go b/internal/vmtap/tc.go deleted file mode 100644 index 818a0801..00000000 --- a/internal/vmtap/tc.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "fmt" - "log/slog" - - "github.com/vishvananda/netlink" - "golang.org/x/sys/unix" -) - -// addRedirect wires bidirectional tc-mirred redirects between fromLink and -// toLink: every packet ingressing on fromLink is stolen and re-injected as -// an egress packet on toLink. Calling it twice, once with (eth0, tap0) and -// once with (tap0, eth0), gives the full bidirectional pattern described in -// .local/kraftlet-cilium-tap-plan.md section 3 — same mechanism -// awslabs/tc-redirect-tap uses for Firecracker. -// -// Callers must already be running inside the target network namespace. -// priority must not collide with Cilium's own clsact hooks on fromLink — -// see the tc/bpf hook ordering caveat in docs/vmtap-cni/configuration.md; -// this is unvalidated against any specific Cilium version/datapath mode. -func addRedirect(fromLink, toLink netlink.Link, priority uint16) error { - if err := ensureIngressQdisc(fromLink); err != nil { - return fmt.Errorf("ensure ingress qdisc on %q: %w", fromLink.Attrs().Name, err) - } - - if err := removeRedirectFilter(fromLink, priority); err != nil { - return fmt.Errorf("clear stale redirect filter on %q: %w", fromLink.Attrs().Name, err) - } - - filter := &netlink.U32{ - FilterAttrs: netlink.FilterAttrs{ - LinkIndex: fromLink.Attrs().Index, - Parent: netlink.HANDLE_MIN_INGRESS, - Priority: priority, - Protocol: unix.ETH_P_ALL, - }, - Actions: []netlink.Action{ - &netlink.MirredAction{ - ActionAttrs: netlink.ActionAttrs{Action: netlink.TC_ACT_STOLEN}, - MirredAction: netlink.TCA_EGRESS_REDIR, - Ifindex: toLink.Attrs().Index, - }, - }, - } - if err := netlink.FilterAdd(filter); err != nil { - return fmt.Errorf("add redirect filter %s->%s: %w", fromLink.Attrs().Name, toLink.Attrs().Name, err) - } - slog.Debug("vmtap: redirect filter installed", - "from", fromLink.Attrs().Name, "to", toLink.Attrs().Name, "priority", priority) - return nil -} - -// ensureIngressQdisc adds an ingress qdisc to link if one is not already -// present. Idempotent, and tolerant of Cilium (or anything else) having -// already attached its own qdisc — an ingress qdisc is shared per-link, so -// finding one already there (of any recognized type) is not an error; only -// a failed add when none exists is. -func ensureIngressQdisc(link netlink.Link) error { - qdiscs, err := netlink.QdiscList(link) - if err != nil { - return fmt.Errorf("list qdiscs: %w", err) - } - for _, q := range qdiscs { - if q.Attrs().Parent == netlink.HANDLE_INGRESS { - return nil - } - } - - qdisc := &netlink.Ingress{ - QdiscAttrs: netlink.QdiscAttrs{ - LinkIndex: link.Attrs().Index, - Parent: netlink.HANDLE_INGRESS, - }, - } - if err := netlink.QdiscAdd(qdisc); err != nil { - return fmt.Errorf("add ingress qdisc: %w", err) - } - return nil -} - -// hasRedirectFilter reports whether a filter at the given priority exists on -// link's ingress qdisc — used by cmdCheck to verify ADD's filters are still -// in place. -func hasRedirectFilter(link netlink.Link, priority uint16) (bool, error) { - filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_INGRESS) - if err != nil { - return false, fmt.Errorf("list filters: %w", err) - } - for _, f := range filters { - if f.Attrs().Priority == priority { - return true, nil - } - } - return false, nil -} - -// removeRedirectFilter deletes the redirect filter this plugin owns at the -// given priority on link's ingress qdisc, if present. Idempotent — a -// missing filter is not an error. -func removeRedirectFilter(link netlink.Link, priority uint16) error { - filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_INGRESS) - if err != nil { - return fmt.Errorf("list filters: %w", err) - } - for _, f := range filters { - if f.Attrs().Priority == priority { - if err := netlink.FilterDel(f); err != nil { - return fmt.Errorf("delete filter at priority %d: %w", priority, err) - } - } - } - return nil -} - -// deleteRedirect removes the redirect filter this plugin installed on link. -// It intentionally leaves the ingress qdisc itself in place — on eth0 that -// qdisc may be shared with Cilium's own hooks, and deleting it out from -// under a still-running pod interface would be far riskier than leaving an -// unused qdisc behind. Idempotent. -func deleteRedirect(link netlink.Link, priority uint16) error { - if err := removeRedirectFilter(link, priority); err != nil { - return fmt.Errorf("remove redirect filter on %q: %w", link.Attrs().Name, err) - } - return nil -} diff --git a/internal/vmtap/tc_test.go b/internal/vmtap/tc_test.go deleted file mode 100644 index 8b04e8d4..00000000 --- a/internal/vmtap/tc_test.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "fmt" - "os" - "testing" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/vishvananda/netlink" -) - -// requireRoot skips the test when not running as root. tap and tc -// operations require CAP_NET_ADMIN/CAP_SYS_ADMIN — mirrors the pattern in -// internal/cni/netns_test.go and internal/cni/tap/tap_test.go. -func requireRoot(t *testing.T) { - t.Helper() - if os.Getuid() != 0 { - t.Skip("skipping: requires root (CAP_NET_ADMIN/CAP_SYS_ADMIN)") - } -} - -// inTestNetns creates a fresh network namespace with a dummy "eth0" link -// standing in for Cilium's real interface, runs fn inside it, and tears the -// namespace down afterward. Isolating each test in its own netns keeps -// tap/tc state from leaking onto the host running the test. -func inTestNetns(t *testing.T, fn func()) { - t.Helper() - requireRoot(t) - - nsObj, err := ns.TempNetNS() - if err != nil { - t.Fatalf("create test netns: %v", err) - } - defer nsObj.Close() //nolint:errcheck // best-effort cleanup - - err = nsObj.Do(func(_ ns.NetNS) error { - dummy := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "eth0"}} - if err := netlink.LinkAdd(dummy); err != nil { - return fmt.Errorf("add dummy eth0: %w", err) - } - link, err := netlink.LinkByName("eth0") - if err != nil { - return err - } - if err := netlink.LinkSetUp(link); err != nil { - return fmt.Errorf("set eth0 up: %w", err) - } - fn() - return nil - }) - if err != nil { - t.Fatalf("in test netns: %v", err) - } -} - -func TestAddTapCreatesLink(t *testing.T) { - inTestNetns(t, func() { - link, err := addTap("tap0", 1500, 0, 0) - if err != nil { - t.Fatalf("addTap() = %v, want nil", err) - } - if link.Type() != "tuntap" && link.Attrs().Name != "tap0" { - t.Errorf("addTap() link = %+v, want a tap0 link", link.Attrs()) - } - if link.Attrs().MTU != 1500 { - t.Errorf("MTU = %d, want 1500", link.Attrs().MTU) - } - }) -} - -func TestAddTapIdempotent(t *testing.T) { - inTestNetns(t, func() { - if _, err := addTap("tap0", 1500, 0, 0); err != nil { - t.Fatalf("first addTap() = %v, want nil", err) - } - if _, err := addTap("tap0", 1500, 0, 0); err != nil { - t.Fatalf("second addTap() = %v, want nil (idempotent)", err) - } - }) -} - -func TestDeleteTapRemovesLink(t *testing.T) { - inTestNetns(t, func() { - if _, err := addTap("tap0", 1500, 0, 0); err != nil { - t.Fatalf("addTap() = %v, want nil", err) - } - if err := deleteTap("tap0"); err != nil { - t.Fatalf("deleteTap() = %v, want nil", err) - } - if _, err := netlink.LinkByName("tap0"); err == nil { - t.Error("tap0 still exists after deleteTap") - } - }) -} - -func TestDeleteTapNonExistentIsNoop(t *testing.T) { - inTestNetns(t, func() { - if err := deleteTap("zzz-nonexistent"); err != nil { - t.Errorf("deleteTap(nonexistent) = %v, want nil", err) - } - }) -} - -func TestAddRedirectAndCheck(t *testing.T) { - inTestNetns(t, func() { - tapLink, err := addTap("tap0", 1500, 0, 0) - if err != nil { - t.Fatalf("addTap() = %v, want nil", err) - } - eth0, err := netlink.LinkByName("eth0") - if err != nil { - t.Fatalf("find eth0: %v", err) - } - - const priority = 7 - if err := addRedirect(eth0, tapLink, priority); err != nil { - t.Fatalf("addRedirect(eth0->tap0) = %v, want nil", err) - } - if err := addRedirect(tapLink, eth0, priority); err != nil { - t.Fatalf("addRedirect(tap0->eth0) = %v, want nil", err) - } - - if ok, err := hasRedirectFilter(eth0, priority); err != nil || !ok { - t.Errorf("hasRedirectFilter(eth0) = %v, %v, want true, nil", ok, err) - } - if ok, err := hasRedirectFilter(tapLink, priority); err != nil || !ok { - t.Errorf("hasRedirectFilter(tap0) = %v, %v, want true, nil", ok, err) - } - - if err := deleteRedirect(eth0, priority); err != nil { - t.Fatalf("deleteRedirect(eth0) = %v, want nil", err) - } - if ok, err := hasRedirectFilter(eth0, priority); err != nil || ok { - t.Errorf("hasRedirectFilter(eth0) after delete = %v, %v, want false, nil", ok, err) - } - }) -} - -func TestAddRedirectIdempotent(t *testing.T) { - inTestNetns(t, func() { - tapLink, err := addTap("tap0", 1500, 0, 0) - if err != nil { - t.Fatalf("addTap() = %v, want nil", err) - } - eth0, err := netlink.LinkByName("eth0") - if err != nil { - t.Fatalf("find eth0: %v", err) - } - - const priority = 3 - if err := addRedirect(eth0, tapLink, priority); err != nil { - t.Fatalf("first addRedirect() = %v, want nil", err) - } - if err := addRedirect(eth0, tapLink, priority); err != nil { - t.Fatalf("second addRedirect() = %v, want nil (idempotent)", err) - } - - filters, err := netlink.FilterList(eth0, netlink.HANDLE_MIN_INGRESS) - if err != nil { - t.Fatalf("FilterList() = %v, want nil", err) - } - count := 0 - for _, f := range filters { - if f.Attrs().Priority == priority { - count++ - } - } - if count != 1 { - t.Errorf("filter count at priority %d = %d, want 1 (no duplicate)", priority, count) - } - }) -} diff --git a/internal/vmtap/types.go b/internal/vmtap/types.go deleted file mode 100644 index 9d982c83..00000000 --- a/internal/vmtap/types.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "github.com/containernetworking/cni/pkg/types" -) - -// PluginConf is the vmtap-cni configuration passed via stdin on each -// invocation, as a chained entry inside the pod's primary conflist (e.g. -// appended to whatever Cilium installs at /etc/cni/net.d/05-cilium.conflist). -// It carries no VPC/VPCAttachment identifiers — those belong exclusively to -// galactic-veth. -type PluginConf struct { - types.PluginConf - - // Enabled gates whether this invocation does anything. Defaults to true. - // Set to false to no-op the plugin for a given conflist entry without - // removing it from the chain — a cheap kill switch independent of - // whichever pod-level signal (annotation vs RuntimeClass) ultimately - // decides which pods get this conflist entry at all (see - // .local/kraftlet-cilium-tap-plan.md section 7). - Enabled *bool `json:"enabled,omitempty"` - - // TapName overrides the default tap interface name ("tap0"). - TapName string `json:"tap_name,omitempty"` - - // OwnerUID and OwnerGID set the tap device's owner so kraftlet can open - // its fd without running as root or holding CAP_NET_ADMIN. Zero (root) - // if unset. - OwnerUID uint32 `json:"owner_uid,omitempty"` - OwnerGID uint32 `json:"owner_gid,omitempty"` - - // FilterPriority overrides the default tc filter priority used for the - // mirred redirect filters. Only needed if the default collides with - // Cilium's own bpf hooks on a given cluster/datapath mode — see the - // Cilium-specific caveats in docs/vmtap-cni/configuration.md. - FilterPriority uint16 `json:"filter_priority,omitempty"` -} - -// enabled reports whether the plugin should act on this invocation. -// Defaults to true when unset. -func (c *PluginConf) enabled() bool { - return c.Enabled == nil || *c.Enabled -} - -// redirectInterfaceInfo describes the interface Cilium already configured -// (typically eth0), read from prevResult and netlink. It is never mutated — -// only used to describe the tap's redirect target and to populate the CNI -// result the guest-side consumer (kraftlet) reads. -type redirectInterfaceInfo struct { - name string - mac string - mtu int // link MTU, from prevResult - - // routeMTU is the pod's route MTU (accounting for overlay/tunnel - // overhead), read from the kernel's routing table rather than copied - // from the link MTU. See the MTU caveat in - // .local/kraftlet-cilium-tap-plan.md section 4 — Cilium adjusts the - // route MTU independently of the interface's link MTU. - routeMTU int -} diff --git a/internal/vmtap/vmtap.go b/internal/vmtap/vmtap.go deleted file mode 100644 index ff999828..00000000 --- a/internal/vmtap/vmtap.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2026 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package vmtap - -import ( - "github.com/containernetworking/cni/pkg/skel" - "github.com/containernetworking/cni/pkg/version" - - "go.datum.net/galactic/internal/metadata" -) - -// pluginName is used in the "about" string skel prints when invoked with no -// CNI_COMMAND, and mirrors the "type" field this plugin is expected to be -// registered under in the conflist chain. -const pluginName = "vmtap-cni" - -// RunPlugin starts the CNI plugin, handling ADD, DEL, and CHECK operations. -// STATUS is intentionally omitted (optional per the CNI spec): unlike -// galactic-veth, vmtap-cni has no Kubernetes API dependency or node-level -// bootstrap state to probe readiness against. -func RunPlugin() { - skel.PluginMainFuncs( - skel.CNIFuncs{ - Add: cmdAdd, - Check: cmdCheck, - Del: cmdDel, - }, - version.All, - "CNI "+pluginName+" plugin "+metadata.Version, - ) -}