-
Notifications
You must be signed in to change notification settings - Fork 23
Add authenticated host capabilities endpoint #338
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
rgarcia
wants to merge
11
commits into
main
Choose a base branch
from
oss/host-capabilities-safe-diagnostics
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
bd2c6e8
Add authenticated host capabilities endpoint
rgarcia f8a1ca5
Redesign capabilities endpoint around multi-runtime registry enumeration
rgarcia cbc919c
Make capability reporting launch-truthful and config-aware
rgarcia c7ed2fe
Address round-2 review: probe Rosetta availability, check firecracker…
rgarcia 610529c
Address round-3 review: config-seeded default fallback, full QEMU lau…
rgarcia f5a2d44
Address round-4 review: truthful VZ fork capability, registry lock, b…
rgarcia 2260f64
Kill the QEMU version probe's whole process group on timeout
rgarcia 66a7fa1
Kill the QEMU version probe's process group on every completion path
rgarcia 993d31e
Retry probe-script tests on the fork/exec ETXTBSY race
rgarcia 47b6e51
Deterministic probe tests: cancel on observed pids, retry exec ETXTBS…
rgarcia d7fba22
Tighten QEMU launch prerequisites
rgarcia File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,224 @@ | ||
| package api | ||
|
|
||
| import ( | ||
| "context" | ||
| "runtime" | ||
| "sync" | ||
|
|
||
| "github.com/kernel/hypeman/lib/hypervisor" | ||
| "github.com/kernel/hypeman/lib/images" | ||
| "github.com/kernel/hypeman/lib/logger" | ||
| "github.com/kernel/hypeman/lib/network" | ||
| "github.com/kernel/hypeman/lib/oapi" | ||
| ) | ||
|
|
||
| // Server-level feature IDs: API surfaces this server exposes regardless of | ||
| // which runtime backs an instance. Per-runtime feature IDs are owned by | ||
| // lib/hypervisor (hypervisor.Capabilities.FeatureIDs) so adding a runtime | ||
| // capability never requires touching this handler. | ||
| const ( | ||
| featureInstances = "instances" | ||
| featureImages = "images" | ||
| featureBuilds = "builds" | ||
| featureVolumes = "volumes" | ||
| featureIngress = "ingress" | ||
| featureExec = "exec" | ||
| featureLogs = "logs" | ||
| featureDevices = "devices" | ||
| featureRosettaEmulation = "rosetta-emulation" | ||
| ) | ||
|
|
||
| // apiVersion is the API contract version from the embedded OpenAPI document. | ||
| // The decoded spec is cached: decoding it per request is needlessly expensive. | ||
| var apiVersion = sync.OnceValue(func() string { | ||
| spec, err := oapi.GetSwagger() | ||
| if err != nil || spec.Info == nil { | ||
| return "unknown" | ||
| } | ||
| return spec.Info.Version | ||
| }) | ||
|
|
||
| // defaultHypervisorProvider is the narrow accessor this handler needs from | ||
| // the instance manager. The concrete instances manager implements it; it is | ||
| // type-asserted rather than added to instances.Manager so alternate Manager | ||
| // implementations (mocks, wrappers) compiled against the public module keep | ||
| // building without a new method. | ||
| type defaultHypervisorProvider interface { | ||
| DefaultHypervisor() hypervisor.Type | ||
| } | ||
|
|
||
| // GetCapabilities reports host, runtime, network, and image capabilities. | ||
| func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilitiesRequestObject) (oapi.GetCapabilitiesResponseObject, error) { | ||
| log := logger.FromContext(ctx) | ||
|
|
||
| // Resolve the default runtime the way launches do. Prefer the manager's | ||
| // own effective default via the optional accessor; when the manager does | ||
| // not expose it (a wrapper embedding instances.Manager hides the concrete | ||
| // manager's extra method), fall back to the configured default the manager | ||
| // was constructed from — launches still route through the wrapped manager, | ||
| // so a hardcoded fallback would misreport a Firecracker/QEMU default as | ||
| // cloud-hypervisor. Only an empty (unconfigured) value normalizes to the | ||
| // compile-time default, mirroring lib/instances.NewManagerWithConfigE. | ||
| defaultRuntime := hypervisor.Type(s.Config.Hypervisor.Default) | ||
| if defaultRuntime == "" { | ||
| defaultRuntime = hypervisor.TypeCloudHypervisor | ||
| } | ||
| if p, ok := s.InstanceManager.(defaultHypervisorProvider); ok { | ||
| defaultRuntime = p.DefaultHypervisor() | ||
| } | ||
|
|
||
| // The capability registry is platform-gated at registration time, so its | ||
| // contents are exactly the runtimes this build supports on this host — | ||
| // including ones added after this handler was written. Capabilities and | ||
| // launch prerequisites are resolved per request, so configuration applied | ||
| // after init (e.g. a pinned cloud-hypervisor version) and host state | ||
| // (e.g. an installed QEMU binary) are reflected without a restart. | ||
| registered := hypervisor.RegisteredRuntimes() | ||
| runtimes := make([]oapi.CapabilitiesRuntime, 0, len(registered)) | ||
| defaultAvailable := false | ||
| for _, rt := range registered { | ||
| available := rt.Available() | ||
| if rt.Type == defaultRuntime { | ||
| defaultAvailable = available | ||
| } | ||
| if !available { | ||
| log.WarnContext(ctx, "runtime is registered but missing launch prerequisites", | ||
| "runtime", string(rt.Type), "error", rt.LaunchErr) | ||
| } | ||
| runtimes = append(runtimes, oapi.CapabilitiesRuntime{ | ||
| Name: string(rt.Type), | ||
| Available: available, | ||
| Features: rt.Capabilities.FeatureIDs(), | ||
| }) | ||
| } | ||
| if !defaultAvailable { | ||
| // Ordinary launches use the default runtime and will fail on this | ||
| // host; surface that in logs as well as in the response. | ||
| log.WarnContext(ctx, "configured default runtime is not available on this host", | ||
| "runtime", string(defaultRuntime), "host_os", runtime.GOOS, "host_arch", runtime.GOARCH) | ||
| } | ||
|
|
||
| networkCaps, err := s.networkCapabilities(ctx) | ||
| if err != nil { | ||
| log.ErrorContext(ctx, "failed to resolve network capabilities", "error", err) | ||
| return oapi.GetCapabilities500JSONResponse{ | ||
| Code: "internal_error", | ||
| Message: "failed to resolve network capabilities", | ||
| }, nil | ||
| } | ||
|
|
||
| emulation := emulationAvailable(runtime.GOOS, runtime.GOARCH, rosettaInstalled()) | ||
|
|
||
| resp := oapi.Capabilities{ | ||
| Server: oapi.CapabilitiesServer{ | ||
| Version: s.Config.Version, | ||
| ApiVersion: apiVersion(), | ||
| }, | ||
| Host: oapi.CapabilitiesHost{ | ||
| Os: runtime.GOOS, | ||
| Arch: runtime.GOARCH, | ||
| }, | ||
| DefaultRuntime: oapi.CapabilitiesDefaultRuntime{ | ||
| Name: string(defaultRuntime), | ||
| Available: defaultAvailable, | ||
| }, | ||
| Runtimes: runtimes, | ||
| Network: *networkCaps, | ||
| Images: oapi.CapabilitiesImages{ | ||
| Platforms: imagePlatforms(runtime.GOARCH, emulation), | ||
| DefaultPlatform: images.HostPlatformString(), | ||
| }, | ||
| Features: serverFeatures(runtime.GOOS, emulation), | ||
| } | ||
|
|
||
| return oapi.GetCapabilities200JSONResponse(resp), nil | ||
| } | ||
|
|
||
| // networkCapabilities resolves the guest networking model and the | ||
| // guest-visible host gateway from the network manager's effective default | ||
| // network. Gateway and subnet are omitted (not serialized as empty strings) | ||
| // when no default network has been resolved. | ||
| func (s *ApiService) networkCapabilities(ctx context.Context) (*oapi.CapabilitiesNetwork, error) { | ||
| caps := &oapi.CapabilitiesNetwork{ | ||
| Model: oapiNetworkModel(network.NetworkModel()), | ||
| GuestToGuest: false, | ||
| } | ||
| if s.NetworkManager == nil { | ||
| return caps, nil | ||
| } | ||
| nw, err := s.NetworkManager.EffectiveDefaultNetwork() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if nw == nil { | ||
| return caps, nil | ||
| } | ||
| if nw.Gateway != "" { | ||
| gateway := nw.Gateway | ||
| caps.Gateway = &gateway | ||
| } | ||
| if nw.Subnet != "" { | ||
| subnet := nw.Subnet | ||
| caps.Subnet = &subnet | ||
| } | ||
| caps.GuestToGuest = network.GuestToGuestEnabled(nw) | ||
| return caps, nil | ||
| } | ||
|
|
||
| // oapiNetworkModel maps the network package's typed model onto the API enum, | ||
| // keeping lib/network free of oapi dependencies. | ||
| func oapiNetworkModel(m network.Model) oapi.CapabilitiesNetworkModel { | ||
| switch m { | ||
| case network.ModelBridge: | ||
| return oapi.Bridge | ||
| case network.ModelNAT: | ||
| return oapi.Nat | ||
| } | ||
| // A new network.Model must also be added to the OpenAPI enum; surface it | ||
| // verbatim rather than misreporting it as a known model. | ||
| return oapi.CapabilitiesNetworkModel(m) | ||
| } | ||
|
|
||
| // emulationAvailable reports whether the host can boot images built for the | ||
| // other CPU architecture right now. Only Apple Silicon macOS hosts qualify | ||
| // (vz with Rosetta), and only when the Rosetta availability probe — the same | ||
| // Virtualization.framework check the vz-shim enforces at launch — reports it | ||
| // installed. Platform eligibility alone (darwin/arm64) is deliberately not | ||
| // enough: a macOS 11/12 host or one without Rosetta installed would advertise | ||
| // launches that lib/hypervisor/vz rejects. | ||
| func emulationAvailable(goos, goarch string, rosettaInstalled bool) bool { | ||
| return goos == "darwin" && goarch == "arm64" && rosettaInstalled | ||
| } | ||
|
|
||
| // imagePlatforms returns the image platforms (os/arch) the host can run: the | ||
| // host-native Linux guest platform, plus Rosetta-emulated linux/amd64 on | ||
| // Apple Silicon with Rosetta installed. | ||
| func imagePlatforms(goarch string, emulation bool) []string { | ||
| platforms := []string{"linux/" + goarch} | ||
| if emulation { | ||
| platforms = append(platforms, "linux/amd64") | ||
| } | ||
| return platforms | ||
| } | ||
|
|
||
| // serverFeatures builds the server-level feature ID list: API surfaces that | ||
| // are always present plus host-platform conditionals. | ||
| func serverFeatures(goos string, emulation bool) []string { | ||
| features := []string{ | ||
| featureInstances, | ||
| featureImages, | ||
| featureBuilds, | ||
| featureVolumes, | ||
| featureIngress, | ||
| featureExec, | ||
| featureLogs, | ||
| } | ||
| // Device (GPU/PCI) passthrough management is only available on Linux hosts. | ||
| if goos == "linux" { | ||
| features = append(features, featureDevices) | ||
| } | ||
| if emulation { | ||
| features = append(features, featureRosettaEmulation) | ||
| } | ||
| return features | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| //go:build darwin | ||
|
|
||
| package api | ||
|
|
||
| import ( | ||
| "runtime" | ||
| "slices" | ||
| "testing" | ||
|
|
||
| "github.com/kernel/hypeman/lib/hypervisor" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // TestGetCapabilitiesRosettaTracksProbe pins that Rosetta emulation reporting | ||
| // follows the live Virtualization.framework availability probe on this host | ||
| // — the same check the vz-shim enforces at launch — rather than treating | ||
| // every Apple Silicon host as emulation-capable: a host without Rosetta | ||
| // installed (or on macOS < 13) must advertise neither the rosetta-emulation | ||
| // feature nor the linux/amd64 image platform. | ||
| func TestGetCapabilitiesRosettaTracksProbe(t *testing.T) { | ||
| t.Parallel() | ||
| if runtime.GOARCH != "arm64" { | ||
| t.Skipf("rosetta emulation exists only on Apple Silicon (GOARCH=%s)", runtime.GOARCH) | ||
| } | ||
| svc := newTestService(t) | ||
| svc.NetworkManager = &stubCapabilitiesNetworkManager{} | ||
|
|
||
| caps := getCapabilities(t, svc) | ||
|
|
||
| want := rosettaInstalled() | ||
| require.Equal(t, want, slices.Contains(caps.Features, "rosetta-emulation"), | ||
| "rosetta-emulation feature must track the launch-path availability probe") | ||
| require.Equal(t, want, slices.Contains(caps.Images.Platforms, "linux/amd64"), | ||
| "linux/amd64 image platform must track the launch-path availability probe") | ||
| } | ||
|
|
||
| // TestRegisteredRuntimesDarwin pins the macOS registration boundary: only vz | ||
| // can genuinely launch VMs on macOS (the Linux backends require KVM and | ||
| // kernel AF_VSOCK), so the capability registry contains exactly vz even | ||
| // though the cloud-hypervisor and qemu packages are linked into the binary. | ||
| func TestRegisteredRuntimesDarwin(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| registered := hypervisor.RegisteredRuntimes() | ||
| require.Len(t, registered, 1) | ||
| require.Equal(t, hypervisor.TypeVZ, registered[0].Type) | ||
|
|
||
| for _, linuxOnly := range []hypervisor.Type{ | ||
| hypervisor.TypeCloudHypervisor, | ||
| hypervisor.TypeFirecracker, | ||
| hypervisor.TypeQEMU, | ||
| hypervisor.TypeQEMUMicroVM, | ||
| } { | ||
| _, ok := hypervisor.CapabilitiesForType(linuxOnly) | ||
| require.False(t, ok, "%s must not register capabilities on macOS", linuxOnly) | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| //go:build linux | ||
|
|
||
| package api | ||
|
|
||
| import ( | ||
| "runtime" | ||
| "testing" | ||
|
|
||
| "github.com/kernel/hypeman/lib/hypervisor" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // TestRegisteredRuntimesLinux pins the Linux registration boundary: the | ||
| // capability registry contains exactly the runtimes launchable on this host, | ||
| // in deterministic sorted order. qemu-microvm appears only where its x86 | ||
| // board exists, and vz never registers off macOS. | ||
| func TestRegisteredRuntimesLinux(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| expected := []string{"cloud-hypervisor", "firecracker", "qemu"} | ||
| if runtime.GOARCH == "amd64" { | ||
| expected = append(expected, "qemu-microvm") | ||
| } | ||
|
|
||
| names := make([]string, 0, len(expected)) | ||
| for _, rt := range hypervisor.RegisteredRuntimes() { | ||
| names = append(names, string(rt.Type)) | ||
| } | ||
| require.Equal(t, expected, names) | ||
|
|
||
| _, vzRegistered := hypervisor.CapabilitiesForType(hypervisor.TypeVZ) | ||
| require.False(t, vzRegistered, "vz must not register capabilities on Linux") | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| //go:build darwin && arm64 | ||
|
|
||
| package api | ||
|
|
||
| import "github.com/Code-Hex/vz/v3" | ||
|
|
||
| // rosettaInstalled reports whether Rosetta translation for Linux guests is | ||
| // installed and usable right now, using the same Virtualization.framework | ||
| // probe the vz-shim enforces when a launch requests Rosetta | ||
| // (cmd/vz-shim/rosetta_arm64.go): NotInstalled (Rosetta missing) and | ||
| // NotSupported (macOS < 13) both fail launches, so capability reporting must | ||
| // not advertise emulation in either state. Evaluated per request, so | ||
| // installing Rosetta (softwareupdate --install-rosetta) is reflected without | ||
| // a restart. cmd/api already links Virtualization.framework on macOS | ||
| // (checkHypervisorAccess), so this adds no build or runtime requirement. | ||
| var rosettaInstalled = func() bool { | ||
| return vz.LinuxRosettaDirectoryShareAvailability() == vz.LinuxRosettaAvailabilityInstalled | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| //go:build !(darwin && arm64) | ||
|
|
||
| package api | ||
|
|
||
| // rosettaInstalled is never true off Apple Silicon macOS: Rosetta emulation | ||
| // exists only under vz. Kept as a var with the same shape as the | ||
| // darwin/arm64 probe so the handler code is identical on every platform. | ||
| var rosettaInstalled = func() bool { return false } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.