diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..ab42b998d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,11 +112,21 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default (`/tmp/.sock`) | +| `graceful_shutdown` | boolean | `false` | Optional. When `true`, on SIGTERM urunc asks the monitor to shut the guest down gracefully over its control socket (QEMU and Cloud Hypervisor on all architectures, Firecracker on x86 only) instead of killing it; the container manager still escalates to SIGKILL after its grace period. Defaults to `false` | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for Qemu's data files. +The `socket_path` option applies to the monitors that expose a control socket: +Firecracker (its API socket), Qemu (a QMP socket) and Cloud Hypervisor (its REST +API socket). It has no effect on the other monitors. The monitor creates the +socket inside its own (pivoted) rootfs; `urunc` creates the directory of a +custom `socket_path` there for you, so the path can be anywhere. It only fails +if the location is invalid, for example when a file already exists at one of the +directories in the path. + **Example:** ```toml @@ -130,6 +140,7 @@ data_path = "/usr/local/share/" default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" +socket_path = "/run/urunc/fc.sock" ``` ### Extra binaries Configuration diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor.go b/pkg/unikontainers/hypervisors/cloud_hypervisor.go index 606a3c02e..a442ae957 100644 --- a/pkg/unikontainers/hypervisors/cloud_hypervisor.go +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor.go @@ -16,6 +16,7 @@ package hypervisors import ( "fmt" + "net/http" "strings" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -44,6 +45,19 @@ func (ch *CloudHypervisor) Ok() error { return nil } +// SupportsGuestShutdown reports that Cloud Hypervisor can shut the guest down +// gracefully via its REST API power-button endpoint. +func (ch *CloudHypervisor) SupportsGuestShutdown() bool { + return true +} + +// RequestGuestShutdown asks Cloud Hypervisor to press the guest's power button +// over its REST API control socket. socketPath is the already-resolved, +// host-reachable path; it is dialed directly. A non-2xx response is an error. +func (ch *CloudHypervisor) RequestGuestShutdown(socketPath string) error { + return unixSocketRequest(socketPath, http.MethodPut, "/api/v1/vm.power-button", nil) +} + // UsesKVM returns true as Cloud Hypervisor is a KVM-based VMM func (ch *CloudHypervisor) UsesKVM() bool { return true @@ -70,6 +84,10 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike // Start building the command exArgs := []string{ch.binaryPath} + // Expose the REST API over a control socket so the runtime can talk to + // Cloud Hypervisor after boot (e.g. for graceful shutdown). + exArgs = append(exArgs, "--api-socket", "path="+ResolveSocketPath(args)) + // Memory configuration if args.Sharedfs.Type == "virtiofs" { exArgs = append(exArgs, "--memory", fmt.Sprintf("size=%sM,shared=on", chMem)) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 9588a45bf..65aa95a2f 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -17,8 +17,10 @@ package hypervisors import ( "encoding/json" "fmt" + "net/http" "os" "path/filepath" + "runtime" "strings" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -88,6 +90,21 @@ func (fc *Firecracker) Ok() error { return nil } +// SupportsGuestShutdown reports whether Firecracker can shut the guest down +// gracefully. Firecracker's SendCtrlAltDel action only exists on x86; on +// aarch64 the API rejects it, so guest shutdown is supported on amd64 only. +func (fc *Firecracker) SupportsGuestShutdown() bool { + return runtime.GOARCH == "amd64" +} + +// RequestGuestShutdown asks Firecracker to inject a Ctrl+Alt+Del into the +// guest over its REST API control socket. socketPath is the already-resolved, +// host-reachable path; it is dialed directly. A non-2xx response is an error. +func (fc *Firecracker) RequestGuestShutdown(socketPath string) error { + body := []byte(`{"action_type":"SendCtrlAltDel"}`) + return unixSocketRequest(socketPath, http.MethodPut, "/actions", body) +} + func (fc *Firecracker) UsesKVM() bool { return true } @@ -108,9 +125,13 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - cmdString := fc.Path() + " --no-api --config-file " + // Launch Firecracker with its API socket enabled (drop --no-api) while + // still booting the guest from the config file. This preserves today's + // boot behavior and additionally leaves the control socket open for use + // after the guest has started. + apiSockPath := ResolveSocketPath(args) JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) - cmdString += JSONConfigFile + cmdString := fc.Path() + " --api-sock " + apiSockPath + " --config-file " + JSONConfigFile if !args.Seccomp { cmdString += " --no-seccomp" } diff --git a/pkg/unikontainers/hypervisors/graceful_shutdown_test.go b/pkg/unikontainers/hypervisors/graceful_shutdown_test.go new file mode 100644 index 000000000..b46a4c6e8 --- /dev/null +++ b/pkg/unikontainers/hypervisors/graceful_shutdown_test.go @@ -0,0 +1,294 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "encoding/json" + "net" + "net/http" + "os" + "path/filepath" + "runtime" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// tempSocketPath returns a short path under the system temp dir for a unix +// socket, avoiding the ~108 byte sun_path limit that long t.TempDir()/subtest +// names can blow past. The directory is removed when the test finishes. +func tempSocketPath(t *testing.T, name string) string { + t.Helper() + dir, err := os.MkdirTemp("", "gs") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return filepath.Join(dir, name) +} + +// qmpFakeServer is a minimal line-JSON QMP server over a unix socket. It sends +// the greeting on accept, answers qmp_capabilities with {"return":{}}, and on +// system_powerdown emits an async {"event":"POWERDOWN"} followed by the +// command's own {"return":{}} (modelling the interleaving real QEMU does, +// fact G29). It records the commands it receives, in order. +type qmpFakeServer struct { + sockPath string + listener net.Listener + + mu sync.Mutex + commands []string + + // When powerdownError is true, the server answers system_powerdown with a + // QMP {"error":...} object instead of a return. + powerdownError bool +} + +func newQMPFakeServer(t *testing.T, powerdownError bool) *qmpFakeServer { + t.Helper() + sockPath := tempSocketPath(t, "qmp.sock") + ln, err := net.Listen("unix", sockPath) + require.NoError(t, err) + + s := &qmpFakeServer{ + sockPath: sockPath, + listener: ln, + powerdownError: powerdownError, + } + go s.serve() + t.Cleanup(func() { _ = ln.Close() }) + return s +} + +func (s *qmpFakeServer) serve() { + conn, err := s.listener.Accept() + if err != nil { + return + } + defer conn.Close() + + enc := json.NewEncoder(conn) + dec := json.NewDecoder(conn) + + // Real QEMU sends the greeting unprompted right after accept (fact G26). + _ = enc.Encode(map[string]any{ + "QMP": map[string]any{ + "version": map[string]any{}, + "capabilities": []string{}, + }, + }) + + for { + var cmd map[string]any + if err := dec.Decode(&cmd); err != nil { + return + } + execute, _ := cmd["execute"].(string) + + s.mu.Lock() + s.commands = append(s.commands, execute) + s.mu.Unlock() + + switch execute { + case "qmp_capabilities": + _ = enc.Encode(map[string]any{"return": map[string]any{}}) + case "system_powerdown": + if s.powerdownError { + _ = enc.Encode(map[string]any{ + "error": map[string]any{"class": "GenericError", "desc": "boom"}, + }) + continue + } + // Async event first, command return second (fact G29). A correct + // client must skip the event and keep reading until the return. + _ = enc.Encode(map[string]any{ + "timestamp": map[string]any{"seconds": 0, "microseconds": 0}, + "event": "POWERDOWN", + }) + _ = enc.Encode(map[string]any{"return": map[string]any{}}) + default: + _ = enc.Encode(map[string]any{"return": map[string]any{}}) + } + } +} + +func (s *qmpFakeServer) recordedCommands() []string { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]string, len(s.commands)) + copy(out, s.commands) + return out +} + +func TestQemuRequestGuestShutdown(t *testing.T) { + t.Run("negotiates capabilities then powers down, tolerating the event", func(t *testing.T) { + s := newQMPFakeServer(t, false) + + q := &Qemu{} + err := q.RequestGuestShutdown(s.sockPath) + assert.NoError(t, err) + + assert.Equal(t, + []string{"qmp_capabilities", "system_powerdown"}, + s.recordedCommands(), + "client must send qmp_capabilities before system_powerdown", + ) + }) + + t.Run("surfaces a QMP error object as an error", func(t *testing.T) { + s := newQMPFakeServer(t, true) + + q := &Qemu{} + err := q.RequestGuestShutdown(s.sockPath) + assert.Error(t, err) + }) +} + +// TestQemuClientDrainsPowerdownReturn proves the QMP client reads until it sees +// the command's own "return", consuming it off the wire, instead of stopping at +// the async POWERDOWN event. net.Pipe is fully synchronous: every server write +// blocks until the client reads it. A single-read client (one that treated the +// event as the reply and left the return unread) would leave the server's +// return write pending and deadlock the follow-up command; a correct +// read-until-return client drains it and lets the follow-up complete. The +// deadlines turn that deadlock into a clean, prompt failure rather than a hang. +func TestQemuGuestShutdownDrainsPowerdownReturn(t *testing.T) { + t.Parallel() + + clientConn, serverConn := net.Pipe() + defer serverConn.Close() + require.NoError(t, clientConn.SetDeadline(time.Now().Add(3*time.Second))) + require.NoError(t, serverConn.SetDeadline(time.Now().Add(3*time.Second))) + + done := make(chan error, 1) + go func() { + defer clientConn.Close() + enc := json.NewEncoder(clientConn) + dec := json.NewDecoder(clientConn) + if err := qmpCommand(enc, dec, "system_powerdown"); err != nil { + done <- err + return + } + // Reachable only if the powerdown "return" was drained; otherwise it + // deadlocks against the server's still-pending return write. + done <- qmpCommand(enc, dec, "query-status") + }() + + senc := json.NewEncoder(serverConn) + sdec := json.NewDecoder(serverConn) + + var cmd map[string]any + require.NoError(t, sdec.Decode(&cmd)) + assert.Equal(t, "system_powerdown", cmd["execute"]) + // Async event first, then the command's own return (fact G29). + require.NoError(t, senc.Encode(map[string]any{"event": "POWERDOWN"})) + require.NoError(t, senc.Encode(map[string]any{"return": map[string]any{}})) + + require.NoError(t, sdec.Decode(&cmd), + "follow-up command must arrive, proving the powerdown return was drained") + assert.Equal(t, "query-status", cmd["execute"]) + require.NoError(t, senc.Encode(map[string]any{"return": map[string]any{}})) + + select { + case err := <-done: + assert.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("qmp client did not drain the powerdown return; follow-up command deadlocked") + } +} + +// clhFakeServer is a fake Cloud Hypervisor REST server speaking HTTP over a +// unix socket. It records the method and path of every request and answers +// with a configured status code. +type clhFakeServer struct { + sockPath string + + mu sync.Mutex + requests [][2]string // {method, path} +} + +func newCLHFakeServer(t *testing.T, status int) *clhFakeServer { + t.Helper() + sockPath := tempSocketPath(t, "ch.sock") + ln, err := net.Listen("unix", sockPath) + require.NoError(t, err) + + s := &clhFakeServer{sockPath: sockPath} + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests = append(s.requests, [2]string{r.Method, r.URL.Path}) + s.mu.Unlock() + w.WriteHeader(status) + }), + ReadHeaderTimeout: 5 * time.Second, + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return s +} + +func (s *clhFakeServer) recordedRequests() [][2]string { + s.mu.Lock() + defer s.mu.Unlock() + out := make([][2]string, len(s.requests)) + copy(out, s.requests) + return out +} + +func TestCloudHypervisorRequestGuestShutdown(t *testing.T) { + tests := []struct { + name string + status int + wantErr bool + }{ + {"204 succeeds", http.StatusNoContent, false}, + {"500 fails", http.StatusInternalServerError, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newCLHFakeServer(t, tt.status) + + ch := &CloudHypervisor{} + err := ch.RequestGuestShutdown(s.sockPath) + + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + + assert.Equal(t, + [][2]string{{http.MethodPut, "/api/v1/vm.power-button"}}, + s.recordedRequests(), + "expected exactly one PUT /api/v1/vm.power-button", + ) + }) + } +} + +func TestSupportsGuestShutdown(t *testing.T) { + t.Parallel() + + assert.True(t, (&Qemu{}).SupportsGuestShutdown(), "qemu supports guest shutdown") + assert.True(t, (&CloudHypervisor{}).SupportsGuestShutdown(), "cloud hypervisor supports guest shutdown") + assert.False(t, (&HVT{}).SupportsGuestShutdown(), "hvt does not support guest shutdown") + assert.False(t, (&SPT{}).SupportsGuestShutdown(), "spt does not support guest shutdown") + assert.False(t, (&Hedge{}).SupportsGuestShutdown(), "hedge does not support guest shutdown") + assert.Equal(t, runtime.GOARCH == "amd64", (&Firecracker{}).SupportsGuestShutdown(), + "firecracker supports guest shutdown only on amd64") +} diff --git a/pkg/unikontainers/hypervisors/hedge.go b/pkg/unikontainers/hypervisors/hedge.go index 7051ba7f0..8db2e9e1e 100644 --- a/pkg/unikontainers/hypervisors/hedge.go +++ b/pkg/unikontainers/hypervisors/hedge.go @@ -42,6 +42,14 @@ func (h *Hedge) Stop(_ int) error { return fmt.Errorf("hedge not implemented yet") } +func (h *Hedge) SupportsGuestShutdown() bool { + return false +} + +func (h *Hedge) RequestGuestShutdown(_ string) error { + return fmt.Errorf("hedge not implemented yet") +} + func (h *Hedge) UsesKVM() bool { return true } diff --git a/pkg/unikontainers/hypervisors/http_unix.go b/pkg/unikontainers/hypervisors/http_unix.go new file mode 100644 index 000000000..0b3056c87 --- /dev/null +++ b/pkg/unikontainers/hypervisors/http_unix.go @@ -0,0 +1,76 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" +) + +// socketRequestTimeout bounds a single control-socket REST request so a stuck +// monitor cannot block the kill path indefinitely. +const socketRequestTimeout = 5 * time.Second + +// unixSocketRequest performs an HTTP request to a monitor's REST API served +// over a unix socket. The socket path is dialed directly; the URL host is a +// placeholder net/http requires. A non-2xx response is returned as an error. +// body may be nil for requests without a payload. +func unixSocketRequest(socketPath, method, urlPath string, body []byte) error { + client := &http.Client{ + Timeout: socketRequestTimeout, + Transport: &http.Transport{ + // A single one-shot request; keep no idle unix connection in the + // pool, so nothing lingers if this helper is ever called from a + // long-lived process. + DisableKeepAlives: true, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socketPath) + }, + }, + } + + var reqBody io.Reader + if body != nil { + reqBody = bytes.NewReader(body) + } + req, err := http.NewRequest(method, "http://unix"+urlPath, reqBody) + if err != nil { + return fmt.Errorf("failed to build %s %s request: %w", method, urlPath, err) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("%s %s over %q failed: %w", method, urlPath, socketPath, err) + } + defer resp.Body.Close() + + // The body is only used to enrich an error string, so cap the read. + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("%s %s returned status %d: %s", + method, urlPath, resp.StatusCode, strings.TrimSpace(string(respBody))) + } + return nil +} diff --git a/pkg/unikontainers/hypervisors/hvt.go b/pkg/unikontainers/hypervisors/hvt.go index 69cb12b91..b7b52d723 100644 --- a/pkg/unikontainers/hypervisors/hvt.go +++ b/pkg/unikontainers/hypervisors/hvt.go @@ -15,6 +15,7 @@ package hypervisors import ( + "fmt" "os/exec" "runtime" "strings" @@ -134,6 +135,17 @@ func (h *HVT) UsesKVM() bool { return true } +// SupportsGuestShutdown reports that HVT has no control socket to request a +// graceful guest shutdown over. +func (h *HVT) SupportsGuestShutdown() bool { + return false +} + +// RequestGuestShutdown is unsupported for HVT; it has no control socket. +func (h *HVT) RequestGuestShutdown(_ string) error { + return fmt.Errorf("guest shutdown not supported for hvt") +} + // SupportsSharedfs returns a bool value depending on the monitor support for shared-fs func (h *HVT) SupportsSharedfs(_ string) bool { return false diff --git a/pkg/unikontainers/hypervisors/qemu.go b/pkg/unikontainers/hypervisors/qemu.go index 1ac77f870..269077bd8 100644 --- a/pkg/unikontainers/hypervisors/qemu.go +++ b/pkg/unikontainers/hypervisors/qemu.go @@ -67,6 +67,10 @@ func (q *Qemu) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel) ([]str cmdString += " -cpu host" // Choose CPU cmdString += " -enable-kvm" // Enable KVM to use CPU virt extensions cmdString += " -display none -vga none -serial stdio -monitor null" // Disable graphic output + // Expose a QMP control socket so the runtime can talk to QEMU after boot + // (e.g. for graceful shutdown). server,nowait lets QEMU boot without + // waiting for a client to connect. + cmdString += " -qmp unix:" + ResolveSocketPath(args) + ",server,nowait" if args.VCPUs > 0 { cmdString += fmt.Sprintf(" -smp %d", args.VCPUs) diff --git a/pkg/unikontainers/hypervisors/qemu_qmp.go b/pkg/unikontainers/hypervisors/qemu_qmp.go new file mode 100644 index 000000000..4dbda5d14 --- /dev/null +++ b/pkg/unikontainers/hypervisors/qemu_qmp.go @@ -0,0 +1,99 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "encoding/json" + "fmt" + "net" + "time" +) + +// qmpDeadline bounds the whole QMP exchange so a stuck monitor cannot block +// the kill path indefinitely. +const qmpDeadline = 5 * time.Second + +// SupportsGuestShutdown reports that QEMU can shut the guest down gracefully +// via the QMP system_powerdown command. +func (q *Qemu) SupportsGuestShutdown() bool { + return true +} + +// RequestGuestShutdown connects to QEMU's QMP control socket and asks the +// guest to power down. socketPath is the already-resolved, host-reachable +// path; it is dialed directly. QMP requires a capabilities handshake before +// any command, and system_powerdown's reply is interleaved with an async +// POWERDOWN event, so each response is read until its own "return" arrives. +func (q *Qemu) RequestGuestShutdown(socketPath string) error { + // Bound the whole attempt (dial plus exchange) by a single absolute + // deadline, so the worst case stays within one qmpDeadline budget rather + // than dial timeout plus exchange timeout stacking on top of each other. + deadline := time.Now().Add(qmpDeadline) + + conn, err := net.DialTimeout("unix", socketPath, time.Until(deadline)) + if err != nil { + return fmt.Errorf("failed to connect to QMP socket %q: %w", socketPath, err) + } + defer conn.Close() + + if err := conn.SetDeadline(deadline); err != nil { + return fmt.Errorf("failed to set QMP socket deadline: %w", err) + } + + dec := json.NewDecoder(conn) + enc := json.NewEncoder(conn) + + // QEMU sends the QMP greeting unprompted right after connect. + var greeting map[string]json.RawMessage + if err := dec.Decode(&greeting); err != nil { + return fmt.Errorf("failed to read QMP greeting: %w", err) + } + + // Capabilities negotiation is mandatory before any other command. + if err := qmpCommand(enc, dec, "qmp_capabilities"); err != nil { + return err + } + + // Ask the guest to power down (ACPI power button). + return qmpCommand(enc, dec, "system_powerdown") +} + +// qmpCommand sends a QMP command with no arguments and waits for its return. +func qmpCommand(enc *json.Encoder, dec *json.Decoder, command string) error { + if err := enc.Encode(map[string]string{"execute": command}); err != nil { + return fmt.Errorf("failed to send QMP command %q: %w", command, err) + } + return qmpReadReturn(dec, command) +} + +// qmpReadReturn reads QMP messages until it sees the command's own reply. Any +// async "event" message is skipped; a "return" ends the wait successfully; an +// "error" object is surfaced as an error. +func qmpReadReturn(dec *json.Decoder, command string) error { + for { + var msg map[string]json.RawMessage + if err := dec.Decode(&msg); err != nil { + return fmt.Errorf("failed to read QMP response for %q: %w", command, err) + } + if errObj, ok := msg["error"]; ok { + return fmt.Errorf("QMP command %q failed: %s", command, string(errObj)) + } + if _, ok := msg["return"]; ok { + return nil + } + // Any other message (typically an async "event") is skipped until the + // command's own "return" arrives. + } +} diff --git a/pkg/unikontainers/hypervisors/qemu_test.go b/pkg/unikontainers/hypervisors/qemu_test.go index fff45244c..0d81ff52c 100644 --- a/pkg/unikontainers/hypervisors/qemu_test.go +++ b/pkg/unikontainers/hypervisors/qemu_test.go @@ -80,6 +80,7 @@ func TestQemuBuildExecCmd(t *testing.T) { "-vga none", "-serial stdio", "-monitor null", + "-qmp unix:/tmp/.sock,server,nowait", "-m 256M", "-kernel " + testKernelPath, "-nic none", @@ -94,6 +95,26 @@ func TestQemuBuildExecCmd(t *testing.T) { "vhost-vsock-pci", }, }, + { + name: "configured SocketPath renders -qmp on that path", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + SocketPath: "/run/urunc/q.sock", + }, + unikernel: &fakeUnikernel{}, + mustContain: []string{"-qmp unix:/run/urunc/q.sock,server,nowait"}, + }, + { + name: "default SocketPath uses the container-id path", + args: types.ExecArgs{ + UnikernelPath: testKernelPath, + Command: testCommand, + ContainerID: "abc123", + }, + unikernel: &fakeUnikernel{}, + mustContain: []string{"-qmp unix:/tmp/abc123.sock,server,nowait"}, + }, { name: "custom MemSizeB renders -m in MB", args: types.ExecArgs{ diff --git a/pkg/unikontainers/hypervisors/spt.go b/pkg/unikontainers/hypervisors/spt.go index 60b5401f6..886d18930 100644 --- a/pkg/unikontainers/hypervisors/spt.go +++ b/pkg/unikontainers/hypervisors/spt.go @@ -15,6 +15,7 @@ package hypervisors import ( + "fmt" "os/exec" "strings" @@ -46,6 +47,17 @@ func (s *SPT) UsesKVM() bool { return false } +// SupportsGuestShutdown reports that SPT has no control socket to request a +// graceful guest shutdown over. +func (s *SPT) SupportsGuestShutdown() bool { + return false +} + +// RequestGuestShutdown is unsupported for SPT; it has no control socket. +func (s *SPT) RequestGuestShutdown(_ string) error { + return fmt.Errorf("guest shutdown not supported for spt") +} + // SupportsSharedfs returns a bool value depending on the monitor support for shared-fs func (s *SPT) SupportsSharedfs(_ string) bool { return false diff --git a/pkg/unikontainers/hypervisors/utils.go b/pkg/unikontainers/hypervisors/utils.go index 1bee33104..1882dcb56 100644 --- a/pkg/unikontainers/hypervisors/utils.go +++ b/pkg/unikontainers/hypervisors/utils.go @@ -17,13 +17,30 @@ package hypervisors import ( "errors" "fmt" + "path/filepath" "runtime" "strconv" "time" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" ) +// DefaultSocketDir is the directory used for a monitor's control socket when +// no socket_path is configured. It always exists inside the monitor rootfs, +// so the default path needs no extra directory setup. +const DefaultSocketDir = "/tmp" + +// ResolveSocketPath returns the path for a monitor's control socket: the +// configured SocketPath when set, otherwise a per-container default under +// DefaultSocketDir. Shared by every monitor that exposes a control socket. +func ResolveSocketPath(args types.ExecArgs) string { + if args.SocketPath != "" { + return args.SocketPath + } + return filepath.Join(DefaultSocketDir, args.ContainerID+".sock") +} + func cpuArch() string { switch runtime.GOARCH { case "arm64": diff --git a/pkg/unikontainers/hypervisors/vmm.go b/pkg/unikontainers/hypervisors/vmm.go index c8600957b..9633c6617 100644 --- a/pkg/unikontainers/hypervisors/vmm.go +++ b/pkg/unikontainers/hypervisors/vmm.go @@ -27,6 +27,17 @@ const DefaultMemory uint64 = 256 // The default memory for every hypervisor: 256 type VmmType string +// UsesControlSocket reports whether a monitor exposes a control socket whose +// path (socket_path) urunc must make reachable before the monitor launches. +func UsesControlSocket(vmmType VmmType) bool { + switch vmmType { + case FirecrackerVmm, QemuVmm, CloudHypervisorVmm: + return true + default: + return false + } +} + var ErrVMMNotInstalled = errors.New("vmm not found") var vmmLog = logrus.WithField("subsystem", "monitors") diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..fd5d94a53 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -42,6 +42,14 @@ type VMM interface { UsesKVM() bool SupportsSharedfs(string) bool Ok() error + // SupportsGuestShutdown reports whether this monitor can inject a native + // guest-shutdown event over its control socket instead of being killed. + SupportsGuestShutdown() bool + // RequestGuestShutdown asks the monitor to inject its native guest-shutdown + // event over its control socket. socketPath is the already-resolved, + // host-reachable path (e.g. /proc//root/...); the monitor dials it + // directly and must not re-resolve or re-prefix it. + RequestGuestShutdown(socketPath string) error } type NetDevParams struct { @@ -106,6 +114,7 @@ type ExecArgs struct { VAccelType string // Specifies the vAccel acceleration type(e.g. vsock). When empty, vAccel is disabled VSockDevPath string // The host directory where the fc unix socket is created VSockDevID int // The guest-cid + SocketPath string // The path of the monitor's control socket (empty means the monitor's default) Net NetDevParams Sharedfs SharedfsParams } @@ -131,9 +140,11 @@ type ExtraBinConfig struct { // MonitorConfig struct is used to hold hypervisor specific configuration // that is parsed from the urunc config file or state.json annotations type MonitorConfig struct { - DefaultMemoryMB uint `toml:"default_memory_mb"` - DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + DefaultMemoryMB uint `toml:"default_memory_mb"` + DefaultVCPUs uint `toml:"default_vcpus"` + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (falls back to a per-container default) + GracefulShutdown bool `toml:"graceful_shutdown,omitempty"` // When true, urunc asks the monitor to shut the guest down gracefully on SIGTERM instead of killing it. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 84172aca4..8d2c37659 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -402,6 +402,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath // ExecArgs vmmArgs := types.ExecArgs{ @@ -411,6 +412,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { Seccomp: true, // Enable Seccomp by default MemSizeB: uint64(defaultMemSizeMB * 1024 * 1024), VCPUs: uint(defaultVCPUs), + SocketPath: socketPath, Environment: os.Environ(), } @@ -648,6 +650,19 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + // Ensure the monitor's control socket directory exists inside the monitor + // rootfs, so the monitor can bind its socket there. changeRoot has already + // made this process' root the monitor rootfs, so the socket path is + // created relative to it. The default (/tmp) already exists; a custom + // socket_path may point at a directory that does not, and MkdirAll fails + // if that location is invalid (e.g. a file already exists there). + if hypervisors.UsesControlSocket(hypervisors.VmmType(vmmType)) { + sockDir := filepath.Dir(hypervisors.ResolveSocketPath(vmmArgs)) + if err = os.MkdirAll(sockDir, 0o755); err != nil { + return fmt.Errorf("failed to create control socket directory %q: %w", sockDir, err) + } + } + // uid/gid // Setup uid, gid and additional groups for the monitor process err = setupUser(u.Spec.Process.User) @@ -737,6 +752,33 @@ func (u *Unikontainer) Signal(signal unix.Signal) error { return err } + // On SIGTERM, when graceful shutdown is enabled and the monitor exposes a + // control socket, ask the monitor to inject its native guest-shutdown event + // instead of killing it. The socket lives inside the monitor's rootfs, so + // it is reached through /proc//root. Any failure falls back to + // forwarding the signal exactly as before. + graceful := signal == unix.SIGTERM && + u.UruncCfg.Monitors[vmmType].GracefulShutdown && + vmm.SupportsGuestShutdown() + if graceful { + sockName := hypervisors.ResolveSocketPath(types.ExecArgs{ + SocketPath: u.UruncCfg.Monitors[vmmType].SocketPath, + ContainerID: u.State.ID, + }) + hostPath := fmt.Sprintf("/proc/%d/root%s", u.State.Pid, sockName) + if err := vmm.RequestGuestShutdown(hostPath); err != nil { + uniklog.WithError(err).Warn("graceful shutdown failed, forwarding signal") + } else { + uniklog.Debug("graceful shutdown requested via control socket") + return nil + } + } else if signal == unix.SIGTERM && u.UruncCfg.Monitors[vmmType].GracefulShutdown { + // SIGTERM with graceful shutdown enabled, yet graceful is false: the + // only remaining reason is that this monitor does not support it. Log it + // so the fall-through is observable, without changing control flow. + uniklog.Debug("graceful shutdown enabled but not supported by this monitor, forwarding signal") + } + return vmm.Signal(u.State.Pid, signal) } diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..e7518039b 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -145,6 +145,8 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"graceful_shutdown"] = strconv.FormatBool(hvCfg.GracefulShutdown) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -191,6 +193,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { hvCfg.BinaryPath = val case "data_path": hvCfg.DataPath = val + case "socket_path": + hvCfg.SocketPath = val case "vhost": boolVal, err := strconv.ParseBool(val) if err != nil { @@ -198,6 +202,13 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "graceful_shutdown": + boolVal, err := strconv.ParseBool(val) + if err != nil { + uniklog.Warnf("Invalid graceful_shutdown value '%s' for monitor '%s': %v. Using default (false).", val, hv, err) + } else { + hvCfg.GracefulShutdown = boolVal + } } cfg.Monitors[hv] = hvCfg }