diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 38b7eadd..a6c2edcf 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -146,6 +146,7 @@ func ctxWithImage(svc *ApiService, name string) context.Context { // Returns the image name on success, or fails the test on error/timeout. func createAndWaitForImage(t *testing.T, svc *ApiService, imageName string, timeout time.Duration) string { t.Helper() + imageName = apiTestImageRef(t, imageName) t.Logf("Creating image %s...", imageName) imgResp, err := svc.CreateImage(ctx(), oapi.CreateImageRequestObject{ diff --git a/cmd/api/api/cp_test.go b/cmd/api/api/cp_test.go index 4a9d5646..7c6fca5b 100644 --- a/cmd/api/api/cp_test.go +++ b/cmd/api/api/cp_test.go @@ -28,6 +28,7 @@ func TestCpToAndFromInstance(t *testing.T) { } svc := newTestService(t) + imageName := "docker.io/library/nginx:alpine" // Ensure system files (kernel and initrd) are available t.Log("Ensuring system files...") @@ -37,7 +38,7 @@ func TestCpToAndFromInstance(t *testing.T) { t.Log("System files ready") // Create and wait for nginx image (has a long-running process) - createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 30*time.Second) + imageName = createAndWaitForImage(t, svc, imageName, 30*time.Second) // Create instance t.Log("Creating instance...") @@ -45,7 +46,7 @@ func TestCpToAndFromInstance(t *testing.T) { instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "cp-test", - Image: "docker.io/library/nginx:alpine", + Image: imageName, Network: &struct { BandwidthDownload *string `json:"bandwidth_download,omitempty"` BandwidthUpload *string `json:"bandwidth_upload,omitempty"` @@ -168,6 +169,7 @@ func TestCpDirectoryToInstance(t *testing.T) { } svc := newTestService(t) + imageName := "docker.io/library/nginx:alpine" // Ensure system files t.Log("Ensuring system files...") @@ -176,7 +178,7 @@ func TestCpDirectoryToInstance(t *testing.T) { require.NoError(t, err) // Create and wait for nginx image (has a long-running process) - createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 30*time.Second) + imageName = createAndWaitForImage(t, svc, imageName, 30*time.Second) // Create instance t.Log("Creating instance...") @@ -184,7 +186,7 @@ func TestCpDirectoryToInstance(t *testing.T) { instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "cp-dir-test", - Image: "docker.io/library/nginx:alpine", + Image: imageName, Network: &struct { BandwidthDownload *string `json:"bandwidth_download,omitempty"` BandwidthUpload *string `json:"bandwidth_upload,omitempty"` diff --git a/cmd/api/api/exec_test.go b/cmd/api/api/exec_test.go index 2f30b1e5..2202c9c4 100644 --- a/cmd/api/api/exec_test.go +++ b/cmd/api/api/exec_test.go @@ -29,6 +29,7 @@ func TestExecInstanceNonTTY(t *testing.T) { } svc := newTestService(t) + imageName := "docker.io/library/nginx:alpine" // Ensure system files (kernel and initrd) are available t.Log("Ensuring system files...") @@ -38,7 +39,7 @@ func TestExecInstanceNonTTY(t *testing.T) { t.Log("System files ready") // Create and wait for nginx image (has a proper long-running process) - createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 30*time.Second) + imageName = createAndWaitForImage(t, svc, imageName, 30*time.Second) // Create instance t.Log("Creating instance...") @@ -46,7 +47,7 @@ func TestExecInstanceNonTTY(t *testing.T) { instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "exec-test", - Image: "docker.io/library/nginx:alpine", + Image: imageName, Network: &struct { BandwidthDownload *string `json:"bandwidth_download,omitempty"` BandwidthUpload *string `json:"bandwidth_upload,omitempty"` @@ -170,6 +171,7 @@ func TestExecWithDebianMinimal(t *testing.T) { } svc := newTestService(t) + imageName := "docker.io/library/debian:12-slim" // Ensure system files (kernel and initrd) are available t.Log("Ensuring system files...") @@ -179,7 +181,7 @@ func TestExecWithDebianMinimal(t *testing.T) { t.Log("System files ready") // Create Debian 12 slim image (minimal, no iproute2) - createAndWaitForImage(t, svc, "docker.io/library/debian:12-slim", 60*time.Second) + imageName = createAndWaitForImage(t, svc, imageName, 60*time.Second) // Create instance with a long-running command so the VM stays alive for exec. // Debian's default CMD is "bash" which exits immediately (no stdin), @@ -190,7 +192,7 @@ func TestExecWithDebianMinimal(t *testing.T) { instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "debian-exec-test", - Image: "docker.io/library/debian:12-slim", + Image: imageName, Cmd: &cmdOverride, Network: &struct { BandwidthDownload *string `json:"bandwidth_download,omitempty"` diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 8a060da1..7b8b1062 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -2,7 +2,6 @@ package api import ( "fmt" - "strings" "testing" "time" @@ -42,8 +41,8 @@ func TestCreateImage_Async(t *testing.T) { // Create images before alpine to populate the queue t.Log("Creating image queue...") queueImages := []string{ - "docker.io/library/busybox:latest", - "docker.io/library/nginx:alpine", + apiTestImageRef(t, "docker.io/library/busybox:latest"), + apiTestImageRef(t, "docker.io/library/nginx:alpine"), } for _, name := range queueImages { _, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ @@ -54,9 +53,10 @@ func TestCreateImage_Async(t *testing.T) { // Create alpine (should be last in queue) t.Log("Creating alpine image (should be queued)...") + alpineName := apiTestImageRef(t, "docker.io/library/alpine:latest") createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ Body: &oapi.CreateImageRequest{ - Name: "docker.io/library/alpine:latest", + Name: alpineName, }, }) require.NoError(t, err) @@ -65,14 +65,16 @@ func TestCreateImage_Async(t *testing.T) { require.True(t, ok, "expected 202 accepted response") img := oapi.Image(acceptedResp) - require.Equal(t, "docker.io/library/alpine:latest", img.Name) + require.Equal(t, alpineName, img.Name) require.NotEmpty(t, img.Digest, "digest should be populated immediately") t.Logf("Image created: name=%s, digest=%s, initial_status=%s, queue_position=%v", img.Name, img.Digest, img.Status, img.QueuePosition) // Construct digest reference for polling: repository@digest // GetImage expects format like "docker.io/library/alpine@sha256:..." - digestRef := "docker.io/library/alpine@" + img.Digest + alpineRef, err := images.ParseNormalizedRef(alpineName) + require.NoError(t, err) + digestRef := alpineRef.Repository() + "@" + img.Digest t.Logf("Polling with digest reference: %s", digestRef) // Poll until ready using digest (tag symlink doesn't exist until status=ready) @@ -135,7 +137,7 @@ func TestCreateImage_InvalidTag(t *testing.T) { t.Log("Creating image with invalid tag...") createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ Body: &oapi.CreateImageRequest{ - Name: "docker.io/library/busybox:foobar", + Name: apiTestImageRef(t, "docker.io/library/busybox:foobar"), }, }) require.NoError(t, err) @@ -181,13 +183,13 @@ func TestCreateImage_Idempotent(t *testing.T) { ctx := ctx() // Create first image to occupy queue position 0 - t.Log("Creating first image (busybox) to occupy queue...") + t.Log("Creating first image (nginx) to occupy queue...") _, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ - Body: &oapi.CreateImageRequest{Name: "docker.io/library/busybox:latest"}, + Body: &oapi.CreateImageRequest{Name: apiTestImageRef(t, "docker.io/library/nginx:alpine")}, }) require.NoError(t, err) - imageName := "docker.io/library/alpine:3.18" + imageName := apiTestImageRef(t, "docker.io/library/alpine:latest") // First call - should create and queue at position 1 t.Log("First CreateImage call (alpine)...") @@ -245,9 +247,9 @@ func TestCreateImage_Idempotent(t *testing.T) { } // Construct digest reference: repository@digest - // Extract repository from imageName (strip tag part) - repository := strings.Split(imageName, ":")[0] - digestRef := repository + "@" + img1.Digest + imageRef, err := images.ParseNormalizedRef(imageName) + require.NoError(t, err) + digestRef := imageRef.Repository() + "@" + img1.Digest t.Logf("Polling with digest reference: %s", digestRef) // Wait for build to complete - poll by digest (tag symlink doesn't exist until status=ready) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index b771addf..8081ac2f 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -135,6 +135,25 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if request.Body.Network != nil && request.Body.Network.Enabled != nil { networkEnabled = *request.Body.Network.Enabled } + var egressProxyConfig *instances.EgressProxyConfig + if request.Body.EgressProxy != nil { + enabled := request.Body.EgressProxy.Enabled != nil && *request.Body.EgressProxy.Enabled + egressProxyConfig = &instances.EgressProxyConfig{Enabled: enabled} + if request.Body.EgressProxy.MockEnvVars != nil { + egressProxyConfig.MockEnvVars = append([]string(nil), (*request.Body.EgressProxy.MockEnvVars)...) + } + if request.Body.EgressProxy.MockEnvVarDomains != nil { + egressProxyConfig.MockEnvVarDomains = make(map[string][]string, len(*request.Body.EgressProxy.MockEnvVarDomains)) + for envVar, domains := range *request.Body.EgressProxy.MockEnvVarDomains { + egressProxyConfig.MockEnvVarDomains[envVar] = append([]string(nil), domains...) + } + } + if request.Body.EgressProxy.EnforcementMode != nil { + egressProxyConfig.EnforcementMode = instances.EgressProxyEnforcementMode(*request.Body.EgressProxy.EnforcementMode) + } else if enabled { + egressProxyConfig.EnforcementMode = instances.EgressProxyEnforcementModeAll + } + } // Parse network bandwidth limits (0 = auto) // Supports both bit-based (e.g., "1Gbps") and byte-based (e.g., "125MB/s") formats @@ -255,6 +274,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Env: env, Tags: resourceTags, NetworkEnabled: networkEnabled, + EgressProxy: egressProxyConfig, Devices: deviceRefs, Volumes: volumes, Hypervisor: hvType, diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 96d25a38..76afa611 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -50,7 +50,7 @@ func TestCreateInstance_ParsesHumanReadableSizes(t *testing.T) { svc := newTestService(t) // Create and wait for alpine image - createAndWaitForImage(t, svc, "docker.io/library/alpine:latest", 30*time.Second) + imageName := createAndWaitForImage(t, svc, "docker.io/library/alpine:latest", 30*time.Second) // Ensure system files (kernel and initramfs) are available t.Log("Ensuring system files (kernel and initramfs)...") @@ -69,7 +69,7 @@ func TestCreateInstance_ParsesHumanReadableSizes(t *testing.T) { resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "test-sizes", - Image: "docker.io/library/alpine:latest", + Image: imageName, Size: &size, HotplugSize: &hotplugSize, OverlaySize: &overlaySize, @@ -215,6 +215,96 @@ func TestCreateInstance_OmittedHotplugSizeDefaultsToZero(t *testing.T) { assert.Equal(t, int64(0), int64(hotplugBytes), "response should report zero hotplug_size when omitted") } +func TestCreateInstance_MapsEgressProxyMockEnvVars(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + enabled := true + mockEnvVars := []string{"OUTBOUND_OPENAI_KEY", "GITHUB_TOKEN"} + mockEnvVarDomains := map[string][]string{ + "OUTBOUND_OPENAI_KEY": []string{"api.openai.com", "*.openai.com"}, + } + env := map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-openai-key-123", + "GITHUB_TOKEN": "real-gh-token-456", + } + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-egress-proxy-mock-env-vars", + Image: "docker.io/library/alpine:latest", + Env: &env, + EgressProxy: &struct { + Enabled *bool `json:"enabled,omitempty"` + EnforcementMode *oapi.CreateInstanceRequestEgressProxyEnforcementMode `json:"enforcement_mode,omitempty"` + MockEnvVarDomains *map[string][]string `json:"mock_env_var_domains,omitempty"` + MockEnvVars *[]string `json:"mock_env_vars,omitempty"` + }{ + Enabled: &enabled, + MockEnvVarDomains: &mockEnvVarDomains, + MockEnvVars: &mockEnvVars, + }, + }, + }) + require.NoError(t, err) + _, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.EgressProxy) + assert.True(t, mockMgr.lastReq.EgressProxy.Enabled) + assert.Equal(t, instances.EgressProxyEnforcementModeAll, mockMgr.lastReq.EgressProxy.EnforcementMode) + assert.Equal(t, []string{"OUTBOUND_OPENAI_KEY", "GITHUB_TOKEN"}, mockMgr.lastReq.EgressProxy.MockEnvVars) + assert.Equal(t, map[string][]string{"OUTBOUND_OPENAI_KEY": []string{"api.openai.com", "*.openai.com"}}, mockMgr.lastReq.EgressProxy.MockEnvVarDomains) + assert.Equal(t, "real-openai-key-123", mockMgr.lastReq.Env["OUTBOUND_OPENAI_KEY"]) + assert.Equal(t, "real-gh-token-456", mockMgr.lastReq.Env["GITHUB_TOKEN"]) +} + +func TestCreateInstance_MapsEgressProxyEnforcementMode(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + enabled := true + mode := oapi.HttpHttpsOnly + env := map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-openai-key-123", + } + mockEnvVars := []string{"OUTBOUND_OPENAI_KEY"} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-egress-proxy-enforcement-mode", + Image: "docker.io/library/alpine:latest", + Env: &env, + EgressProxy: &struct { + Enabled *bool `json:"enabled,omitempty"` + EnforcementMode *oapi.CreateInstanceRequestEgressProxyEnforcementMode `json:"enforcement_mode,omitempty"` + MockEnvVarDomains *map[string][]string `json:"mock_env_var_domains,omitempty"` + MockEnvVars *[]string `json:"mock_env_vars,omitempty"` + }{ + Enabled: &enabled, + EnforcementMode: &mode, + MockEnvVars: &mockEnvVars, + }, + }, + }) + require.NoError(t, err) + _, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.EgressProxy) + assert.Equal(t, instances.EgressProxyEnforcementModeHTTPHTTPSOnly, mockMgr.lastReq.EgressProxy.EnforcementMode) +} + func TestForkInstance_Success(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -408,7 +498,7 @@ func TestInstanceLifecycle_StopStart(t *testing.T) { svc := newTestService(t) // Use nginx:alpine so the VM runs a real workload (not just exits immediately) - createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 60*time.Second) + imageName := createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 60*time.Second) // Ensure system files (kernel and initramfs) are available t.Log("Ensuring system files (kernel and initramfs)...") @@ -423,7 +513,7 @@ func TestInstanceLifecycle_StopStart(t *testing.T) { createResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "test-lifecycle", - Image: "docker.io/library/nginx:alpine", + Image: imageName, Network: &struct { BandwidthDownload *string `json:"bandwidth_download,omitempty"` BandwidthUpload *string `json:"bandwidth_upload,omitempty"` diff --git a/cmd/api/api/registry_test.go b/cmd/api/api/registry_test.go index 4f664ddf..b206ca68 100644 --- a/cmd/api/api/registry_test.go +++ b/cmd/api/api/registry_test.go @@ -52,7 +52,7 @@ func TestRegistryPushAndConvert(t *testing.T) { // Pull a small image from Docker Hub to push to our registry t.Log("Pulling alpine:latest from Docker Hub...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -110,7 +110,7 @@ func TestRegistryPushAndCreateInstance(t *testing.T) { // Pull and push alpine t.Log("Pulling alpine:latest...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -184,7 +184,7 @@ func TestRegistryLayerCaching(t *testing.T) { // Pull alpine image from Docker Hub t.Log("Pulling alpine:latest from Docker Hub...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -268,7 +268,7 @@ func TestRegistrySharedLayerCaching(t *testing.T) { // Pull alpine image (this will be our base) t.Log("Pulling alpine:latest...") - alpineRef, err := name.ParseReference("docker.io/library/alpine:latest") + alpineRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) alpineImg, err := remote.Image(alpineRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) require.NoError(t, err) @@ -300,7 +300,7 @@ func TestRegistrySharedLayerCaching(t *testing.T) { // Now pull a different alpine-based image (e.g., alpine:3.18) // which should share the base layer with alpine:latest t.Log("Pulling alpine:3.18 (shares base layer)...") - alpine318Ref, err := name.ParseReference("docker.io/library/alpine:3.18") + alpine318Ref, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:3.18")) require.NoError(t, err) alpine318Img, err := remote.Image(alpine318Ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) require.NoError(t, err) @@ -350,7 +350,7 @@ func TestRegistryTagPush(t *testing.T) { // Pull alpine image from Docker Hub t.Log("Pulling alpine:latest from Docker Hub...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -404,7 +404,7 @@ func TestRegistryDockerV2ManifestConversion(t *testing.T) { // Pull alpine image from Docker Hub (OCI format) t.Log("Pulling alpine:latest from Docker Hub...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) diff --git a/cmd/api/api/test_prewarm_test.go b/cmd/api/api/test_prewarm_test.go new file mode 100644 index 00000000..e347f7aa --- /dev/null +++ b/cmd/api/api/test_prewarm_test.go @@ -0,0 +1,70 @@ +package api + +import ( + "fmt" + "os" + "strings" + "sync" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +const ( + testPrewarmStrictEnv = "HYPEMAN_TEST_PREWARM_STRICT" + testRegistryEnv = "HYPEMAN_TEST_REGISTRY" +) + +var apiRegistryLogOnce sync.Once +var apiRegistryLogMessage string + +func apiTestImageRef(t *testing.T, source string) string { + t.Helper() + + registry := strings.TrimSpace(os.Getenv(testRegistryEnv)) + if registry == "" { + if isTestPrewarmStrict() { + t.Fatalf("%s is required when %s is enabled", testRegistryEnv, testPrewarmStrictEnv) + } + return source + } + + registry = strings.TrimPrefix(strings.TrimPrefix(registry, "http://"), "https://") + if registry == "" { + t.Fatalf("%s must not be empty", testRegistryEnv) + } + + ref, err := images.ParseNormalizedRef(source) + if err != nil { + t.Fatalf("parse source image ref %q: %v", source, err) + } + + repo := ref.Repository() + if !strings.HasPrefix(repo, "docker.io/") { + return source + } + repo = strings.TrimPrefix(repo, "docker.io/") + + var mapped string + switch { + case ref.Tag() != "": + mapped = registry + "/" + repo + ":" + ref.Tag() + case ref.Digest() != "": + mapped = registry + "/" + repo + "@" + ref.Digest() + default: + mapped = registry + "/" + repo + ":latest" + } + + apiRegistryLogOnce.Do(func() { + apiRegistryLogMessage = fmt.Sprintf("using test registry mirror source=%s mapped=%s", source, mapped) + }) + if apiRegistryLogMessage != "" { + t.Log(apiRegistryLogMessage) + } + return mapped +} + +func isTestPrewarmStrict() bool { + v := strings.TrimSpace(os.Getenv(testPrewarmStrictEnv)) + return v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes") +} diff --git a/cmd/test-prewarm/main.go b/cmd/test-prewarm/main.go index e710f5db..48853006 100644 --- a/cmd/test-prewarm/main.go +++ b/cmd/test-prewarm/main.go @@ -29,8 +29,11 @@ const ( var defaultImages = []string{ "docker.io/library/alpine:latest", + "docker.io/library/alpine:3.18", + "docker.io/library/debian:12-slim", "docker.io/library/nginx:alpine", "docker.io/bitnami/redis:latest", + "docker.io/jrei/systemd-ubuntu:22.04", } type manifestImage struct { diff --git a/integration/systemd_test.go b/integration/systemd_test.go index 986ebb11..ad60510d 100644 --- a/integration/systemd_test.go +++ b/integration/systemd_test.go @@ -75,7 +75,7 @@ func TestSystemdMode(t *testing.T) { instanceManager.DeleteInstance(ctx, "systemd-test") }) - imageName := "docker.io/jrei/systemd-ubuntu:22.04" + imageName := integrationTestImageRef(t, "docker.io/jrei/systemd-ubuntu:22.04") // Pull the systemd image t.Log("Pulling systemd image:", imageName) diff --git a/integration/test_prewarm_test.go b/integration/test_prewarm_test.go new file mode 100644 index 00000000..aa5c82e9 --- /dev/null +++ b/integration/test_prewarm_test.go @@ -0,0 +1,70 @@ +package integration + +import ( + "fmt" + "os" + "strings" + "sync" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +const ( + testPrewarmStrictEnv = "HYPEMAN_TEST_PREWARM_STRICT" + testRegistryEnv = "HYPEMAN_TEST_REGISTRY" +) + +var integrationRegistryLogOnce sync.Once +var integrationRegistryLogMessage string + +func integrationTestImageRef(t *testing.T, source string) string { + t.Helper() + + registry := strings.TrimSpace(os.Getenv(testRegistryEnv)) + if registry == "" { + if isTestPrewarmStrict() { + t.Fatalf("%s is required when %s is enabled", testRegistryEnv, testPrewarmStrictEnv) + } + return source + } + + registry = strings.TrimPrefix(strings.TrimPrefix(registry, "http://"), "https://") + if registry == "" { + t.Fatalf("%s must not be empty", testRegistryEnv) + } + + ref, err := images.ParseNormalizedRef(source) + if err != nil { + t.Fatalf("parse source image ref %q: %v", source, err) + } + + repo := ref.Repository() + if !strings.HasPrefix(repo, "docker.io/") { + return source + } + repo = strings.TrimPrefix(repo, "docker.io/") + + var mapped string + switch { + case ref.Tag() != "": + mapped = registry + "/" + repo + ":" + ref.Tag() + case ref.Digest() != "": + mapped = registry + "/" + repo + "@" + ref.Digest() + default: + mapped = registry + "/" + repo + ":latest" + } + + integrationRegistryLogOnce.Do(func() { + integrationRegistryLogMessage = fmt.Sprintf("using test registry mirror source=%s mapped=%s", source, mapped) + }) + if integrationRegistryLogMessage != "" { + t.Log(integrationRegistryLogMessage) + } + return mapped +} + +func isTestPrewarmStrict() bool { + v := strings.TrimSpace(os.Getenv(testPrewarmStrictEnv)) + return v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes") +} diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 17a92ad1..cbf59d09 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -97,7 +97,7 @@ func TestVGPU(t *testing.T) { t.Log("System files ready") // Step 2: Pull alpine image (lightweight for testing) - imageName := "docker.io/library/alpine:latest" + imageName := integrationTestImageRef(t, "docker.io/library/alpine:latest") t.Log("Step 2: Pulling alpine image...") _, err = imageManager.CreateImage(ctx, images.CreateImageRequest{ Name: imageName, diff --git a/lib/egressproxy/README.md b/lib/egressproxy/README.md new file mode 100644 index 00000000..30a4488f --- /dev/null +++ b/lib/egressproxy/README.md @@ -0,0 +1,41 @@ +# Egress Proxy (Mock Secret Substitution) + +This module provides an optional, default-off networking mode for VM egress. + +When enabled for an instance, hypeman does three things: + +1. It starts (or reuses) a host-side HTTP/HTTPS MITM proxy bound to the VM bridge gateway. +2. It injects proxy environment variables into the guest (`HTTP_PROXY` / `HTTPS_PROXY`) and installs the proxy CA certificate in the guest trust store. +3. It enforces policy on the host to prevent direct outbound TCP egress from the VM unless traffic is going to the bridge gateway (the proxy), depending on `egress_proxy.enforcement_mode`. + +## Secret substitution flow + +- API callers provide real secret values in instance `env`. +- Per instance, `egress_proxy.mock_env_vars` lists which env var names should be mocked. +- Per instance, `egress_proxy.mock_env_var_domains` can optionally restrict each mocked env var to specific destination hosts: + - Exact host: `api.openai.com` + - Single-level wildcard: `*.openai.com` + - If a mocked env var is omitted from this map, substitution is allowed for all HTTPS destinations. +- Per instance, `egress_proxy.enforcement_mode` controls host-side direct egress blocking: + - `all` (default when proxy is enabled): reject direct non-proxy TCP egress from the VM TAP interface. + - `http_https_only`: reject direct TCP egress only on destination ports `80` and `443`. +- Inside the VM, each listed env var is rewritten to `mock-` (for example `mock-OUTBOUND_OPENAI_KEY`). +- Substitution is applied to HTTPS requests only after MITM decryption. +- For HTTPS egress, the proxy validates upstream TLS certificates with the host trust store before forwarding. +- The proxy scans each HTTP header value and replaces configured mock values with real values only when the verified destination host matches the env var's allowlist (if configured). +- The modified request is then forwarded upstream. + +This keeps real secrets out of the VM while still allowing authenticated egress requests. + +## Security behavior + +- Real secret values are persisted in the normal instance `env` metadata, which is already host-side state. +- TLS interception requires guest trust of the proxy CA; hypeman installs this CA in the guest when proxy mode is enabled. +- Egress enforcement is applied per instance TAP device and removed when the instance stops/standbys/deletes. +- Enforcement intentionally targets TCP egress only. DNS/other non-TCP traffic is not rewritten and is not blocked by `all` mode. + +## Limits of enforcement + +- Header replacement is applied to HTTP headers only (not request/response bodies). +- Non-HTTP protocols or custom ports are not rewritten by the MITM layer. +- Plain HTTP requests are not eligible for secret substitution. diff --git a/lib/egressproxy/cert.go b/lib/egressproxy/cert.go new file mode 100644 index 00000000..4bfd36f1 --- /dev/null +++ b/lib/egressproxy/cert.go @@ -0,0 +1,153 @@ +package egressproxy + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +func loadOrCreateCA(dataDir string) (*x509.Certificate, *rsa.PrivateKey, string, error) { + dir := filepath.Join(dataDir, "egressproxy") + certPath := filepath.Join(dir, "ca.crt") + keyPath := filepath.Join(dir, "ca.key") + + certPEM, certErr := os.ReadFile(certPath) + keyPEM, keyErr := os.ReadFile(keyPath) + if certErr == nil && keyErr == nil { + cert, key, err := parseCA(certPEM, keyPEM) + if err == nil { + return cert, key, string(certPEM), nil + } + } + + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, nil, "", fmt.Errorf("create egress proxy cert dir: %w", err) + } + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, "", fmt.Errorf("generate egress proxy CA key: %w", err) + } + + serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, nil, "", fmt.Errorf("generate CA serial: %w", err) + } + + now := time.Now() + tmpl := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: "hypeman-egress-proxy-ca", + Organization: []string{"hypeman"}, + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(10 * 365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLen: 1, + } + + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, "", fmt.Errorf("create egress proxy CA cert: %w", err) + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(caKey)}) + + if err := os.WriteFile(certPath, certPEM, 0644); err != nil { + return nil, nil, "", fmt.Errorf("write egress proxy CA cert: %w", err) + } + if err := os.WriteFile(keyPath, keyPEM, 0600); err != nil { + return nil, nil, "", fmt.Errorf("write egress proxy CA key: %w", err) + } + + cert, key, err := parseCA(certPEM, keyPEM) + if err != nil { + return nil, nil, "", err + } + + return cert, key, string(certPEM), nil +} + +func parseCA(certPEM, keyPEM []byte) (*x509.Certificate, *rsa.PrivateKey, error) { + certBlock, _ := pem.Decode(certPEM) + if certBlock == nil { + return nil, nil, fmt.Errorf("parse egress proxy CA cert PEM") + } + cert, err := x509.ParseCertificate(certBlock.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parse egress proxy CA cert: %w", err) + } + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, nil, fmt.Errorf("parse egress proxy CA key PEM") + } + key, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parse egress proxy CA key: %w", err) + } + return cert, key, nil +} + +func signHostCertificate(caCert *x509.Certificate, caKey *rsa.PrivateKey, host string) (*tls.Certificate, error) { + leafKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, fmt.Errorf("generate leaf key: %w", err) + } + + serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("generate leaf serial: %w", err) + } + + now := time.Now() + tmpl := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: host, + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + if ip := net.ParseIP(host); ip != nil { + tmpl.IPAddresses = []net.IP{ip} + } else { + tmpl.DNSNames = []string{host} + } + + leafDER, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &leafKey.PublicKey, caKey) + if err != nil { + return nil, fmt.Errorf("sign leaf cert: %w", err) + } + + leafPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}) + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCert.Raw}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(leafKey)}) + + tlsCert, err := tls.X509KeyPair(append(leafPEM, caPEM...), keyPEM) + if err != nil { + return nil, fmt.Errorf("build leaf keypair: %w", err) + } + if len(tlsCert.Certificate) > 0 { + if leaf, parseErr := x509.ParseCertificate(tlsCert.Certificate[0]); parseErr == nil { + tlsCert.Leaf = leaf + } + } + return &tlsCert, nil +} diff --git a/lib/egressproxy/domains.go b/lib/egressproxy/domains.go new file mode 100644 index 00000000..4d530203 --- /dev/null +++ b/lib/egressproxy/domains.go @@ -0,0 +1,152 @@ +package egressproxy + +import ( + "fmt" + "net" + "strings" +) + +type domainMatcher struct { + exact string + wildcardSuffix string +} + +func normalizeDestinationHost(raw string) string { + host := strings.TrimSpace(raw) + if host == "" { + return "" + } + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.TrimPrefix(host, "[") + host = strings.TrimSuffix(host, "]") + host = strings.TrimSuffix(host, ".") + host = strings.ToLower(host) + if ip := net.ParseIP(host); ip != nil { + return ip.String() + } + return host +} + +// NormalizeAllowedDomainPattern validates and normalizes one allowlisted destination +// host pattern (exact host/IP or single-level wildcard *.example.com). +func NormalizeAllowedDomainPattern(pattern string) (string, error) { + p := strings.TrimSpace(pattern) + if p == "" { + return "", fmt.Errorf("allowed domain pattern must be non-empty") + } + if strings.Contains(p, "://") || strings.ContainsAny(p, "/?#") { + return "", fmt.Errorf("allowed domain pattern %q must be a host pattern without scheme/path", p) + } + if _, _, err := net.SplitHostPort(p); err == nil { + return "", fmt.Errorf("allowed domain pattern %q must not include a port", p) + } + + if strings.HasPrefix(p, "*.") { + suffixRaw := strings.TrimPrefix(p, "*.") + if strings.Contains(suffixRaw, ":") { + return "", fmt.Errorf("allowed wildcard domain pattern %q must not include a port", p) + } + + base := normalizeDestinationHost(suffixRaw) + if base == "" { + return "", fmt.Errorf("allowed wildcard domain pattern %q has empty suffix", p) + } + if net.ParseIP(base) != nil { + return "", fmt.Errorf("allowed wildcard domain pattern %q cannot target an IP", p) + } + if !isValidDNSName(base) { + return "", fmt.Errorf("allowed wildcard domain pattern %q has invalid hostname suffix", p) + } + if strings.Count(base, ".") < 1 { + return "", fmt.Errorf("allowed wildcard domain pattern %q must target at least a two-label domain", p) + } + return "*." + base, nil + } + + if strings.Contains(p, "*") { + return "", fmt.Errorf("allowed domain pattern %q has unsupported wildcard syntax", p) + } + + host := normalizeDestinationHost(p) + if host == "" { + return "", fmt.Errorf("allowed domain pattern %q has empty host", p) + } + if net.ParseIP(host) != nil { + return host, nil + } + if !isValidDNSName(host) { + return "", fmt.Errorf("allowed domain pattern %q has invalid hostname", p) + } + return host, nil +} + +func compileDomainMatchers(patterns []string) ([]domainMatcher, error) { + if len(patterns) == 0 { + return nil, nil + } + out := make([]domainMatcher, 0, len(patterns)) + for _, raw := range patterns { + normalized, err := NormalizeAllowedDomainPattern(raw) + if err != nil { + return nil, err + } + if strings.HasPrefix(normalized, "*.") { + out = append(out, domainMatcher{wildcardSuffix: "." + strings.TrimPrefix(normalized, "*.")}) + continue + } + out = append(out, domainMatcher{exact: normalized}) + } + return out, nil +} + +func matchesAnyDomain(host string, matchers []domainMatcher) bool { + if len(matchers) == 0 { + return true + } + for _, m := range matchers { + if m.matches(host) { + return true + } + } + return false +} + +func (m domainMatcher) matches(host string) bool { + if host == "" { + return false + } + if m.exact != "" { + return host == m.exact + } + if m.wildcardSuffix == "" || !strings.HasSuffix(host, m.wildcardSuffix) { + return false + } + prefix := strings.TrimSuffix(host, m.wildcardSuffix) + return prefix != "" && !strings.Contains(prefix, ".") +} + +func isValidDNSName(host string) bool { + if host == "" || len(host) > 253 { + return false + } + labels := strings.Split(host, ".") + for _, label := range labels { + if label == "" || len(label) > 63 { + return false + } + for i, ch := range label { + isLetter := ch >= 'a' && ch <= 'z' + isDigit := ch >= '0' && ch <= '9' + isHyphen := ch == '-' + if !(isLetter || isDigit || isHyphen) { + return false + } + if (i == 0 || i == len(label)-1) && isHyphen { + return false + } + } + } + return true +} diff --git a/lib/egressproxy/enforce_linux.go b/lib/egressproxy/enforce_linux.go new file mode 100644 index 00000000..898f8475 --- /dev/null +++ b/lib/egressproxy/enforce_linux.go @@ -0,0 +1,141 @@ +//go:build linux + +package egressproxy + +import ( + "fmt" + "os/exec" + "strings" + "unicode" +) + +const ( + enforcementSuffixPort80 = "80" + enforcementSuffixPort443 = "443" + enforcementSuffixAllTCP = "all-tcp" +) + +func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { + if instanceID == "" || tapDevice == "" || gatewayIP == "" || proxyPort <= 0 { + return fmt.Errorf("invalid egress enforcement inputs") + } + + comment80 := enforcementComment(instanceID, enforcementSuffixPort80) + comment443 := enforcementComment(instanceID, enforcementSuffixPort443) + commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP) + + // Clean old rules first so updates are idempotent across restarts and mode changes. + _ = removeRuleByComment(comment80) + _ = removeRuleByComment(comment443) + _ = removeRuleByComment(commentAllTCP) + + if blockAllTCPEgress { + if err := insertRejectAllTCPRule(tapDevice, gatewayIP, commentAllTCP); err != nil { + return fmt.Errorf("insert all-tcp egress enforcement: %w", err) + } + return nil + } + + if err := insertRejectRule(tapDevice, gatewayIP, 80, comment80); err != nil { + return fmt.Errorf("insert port 80 egress enforcement: %w", err) + } + if err := insertRejectRule(tapDevice, gatewayIP, 443, comment443); err != nil { + _ = removeRuleByComment(comment80) + return fmt.Errorf("insert port 443 egress enforcement: %w", err) + } + + return nil +} + +func removeEgressEnforcement(instanceID string) error { + if instanceID == "" { + return nil + } + comment80 := enforcementComment(instanceID, enforcementSuffixPort80) + comment443 := enforcementComment(instanceID, enforcementSuffixPort443) + commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP) + _ = removeRuleByComment(comment80) + _ = removeRuleByComment(comment443) + _ = removeRuleByComment(commentAllTCP) + return nil +} + +func insertRejectRule(tapDevice, gatewayIP string, port int, comment string) error { + cmd := exec.Command( + "iptables", "-I", "FORWARD", "1", + "-i", tapDevice, + "-p", "tcp", + "--dport", fmt.Sprintf("%d", port), + "!", "-d", gatewayIP, + "-m", "comment", "--comment", comment, + "-j", "REJECT", + ) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +func insertRejectAllTCPRule(tapDevice, gatewayIP, comment string) error { + cmd := exec.Command( + "iptables", "-I", "FORWARD", "1", + "-i", tapDevice, + "-p", "tcp", + "!", "-d", gatewayIP, + "-m", "comment", "--comment", comment, + "-j", "REJECT", + ) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +func removeRuleByComment(comment string) error { + listCmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n") + output, err := listCmd.Output() + if err != nil { + return err + } + + commentMarker := fmt.Sprintf("/* %s */", comment) + var ruleNums []string + for _, line := range strings.Split(string(output), "\n") { + if !strings.Contains(line, commentMarker) { + continue + } + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + ruleNums = append(ruleNums, fields[0]) + } + + for i := len(ruleNums) - 1; i >= 0; i-- { + delCmd := exec.Command("iptables", "-D", "FORWARD", ruleNums[i]) + _ = delCmd.Run() + } + return nil +} + +func enforcementComment(instanceID, suffix string) string { + safeID := sanitizeInstanceIDForComment(instanceID) + return fmt.Sprintf("hypeman-egress-%s-%s", safeID, suffix) +} + +func sanitizeInstanceIDForComment(instanceID string) string { + cleaned := strings.Map(func(r rune) rune { + switch { + case unicode.IsLetter(r), unicode.IsDigit(r): + return r + case r == '-', r == '_', r == '.': + return r + default: + return '_' + } + }, strings.TrimSpace(instanceID)) + if cleaned == "" { + return "unknown" + } + return cleaned +} diff --git a/lib/egressproxy/enforce_other.go b/lib/egressproxy/enforce_other.go new file mode 100644 index 00000000..3eb72c04 --- /dev/null +++ b/lib/egressproxy/enforce_other.go @@ -0,0 +1,12 @@ +//go:build !linux + +package egressproxy + +func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { + _ = blockAllTCPEgress + return nil +} + +func removeEgressEnforcement(instanceID string) error { + return nil +} diff --git a/lib/egressproxy/service.go b/lib/egressproxy/service.go new file mode 100644 index 00000000..07701012 --- /dev/null +++ b/lib/egressproxy/service.go @@ -0,0 +1,481 @@ +package egressproxy + +import ( + "bufio" + "context" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" +) + +type sourcePolicy struct { + secretRewriteRules []secretRewriteRule +} + +type secretRewriteRule struct { + mockValue string + realValue string + domainMatchers []domainMatcher +} + +const defaultLeafCertCacheLimit = 512 + +// Service is a host-side per-process HTTP/HTTPS MITM egress proxy. +type Service struct { + mu sync.RWMutex + + dataDir string + gatewayIP string + listenPort int + + server *http.Server + listener net.Listener + + transport *http.Transport + + caCert *x509.Certificate + caKey *rsa.PrivateKey + caPEM string + + certCache map[string]*tls.Certificate + certCacheOrder []string + certCacheLimit int + + policiesBySourceIP map[string]sourcePolicy + sourceIPByInstance map[string]string +} + +func NewService(dataDir string, listenPort int) (*Service, error) { + return NewServiceWithOptions(dataDir, listenPort, ServiceOptions{}) +} + +func NewServiceWithOptions(dataDir string, listenPort int, opts ServiceOptions) (*Service, error) { + if listenPort <= 0 { + listenPort = DefaultListenPort + } + + caCert, caKey, caPEM, err := loadOrCreateCA(dataDir) + if err != nil { + return nil, err + } + + rootCAs, err := buildRootCAPool(opts) + if err != nil { + return nil, err + } + + transport := &http.Transport{ + Proxy: nil, + DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: false, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 15 * time.Second, + TLSClientConfig: &tls.Config{ + RootCAs: rootCAs, + }, + } + + return &Service{ + dataDir: dataDir, + listenPort: listenPort, + transport: transport, + caCert: caCert, + caKey: caKey, + caPEM: caPEM, + certCache: make(map[string]*tls.Certificate), + certCacheLimit: defaultLeafCertCacheLimit, + policiesBySourceIP: make(map[string]sourcePolicy), + sourceIPByInstance: make(map[string]string), + }, nil +} + +func buildRootCAPool(opts ServiceOptions) (*x509.CertPool, error) { + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + + for _, pemData := range opts.AdditionalRootCAPEM { + trimmed := strings.TrimSpace(pemData) + if trimmed == "" { + continue + } + if ok := pool.AppendCertsFromPEM([]byte(trimmed)); !ok { + return nil, fmt.Errorf("append additional root CA PEM failed") + } + } + return pool, nil +} + +func (s *Service) EnsureStarted(ctx context.Context, gatewayIP string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.listener != nil { + if gatewayIP != "" && gatewayIP != s.gatewayIP { + return fmt.Errorf("%w: current=%s requested=%s", ErrGatewayMismatch, s.gatewayIP, gatewayIP) + } + return nil + } + + s.gatewayIP = gatewayIP + listenAddr := net.JoinHostPort(gatewayIP, strconv.Itoa(s.listenPort)) + ln, err := net.Listen("tcp", listenAddr) + if err != nil { + return fmt.Errorf("listen egress proxy on %s: %w", listenAddr, err) + } + + s.listener = ln + s.server = &http.Server{ + Handler: s, + ReadHeaderTimeout: 15 * time.Second, + } + + go func() { + if serveErr := s.server.Serve(ln); serveErr != nil && serveErr != http.ErrServerClosed { + slog.Error("egress proxy server exited", "error", serveErr) + } + }() + + return nil +} + +func (s *Service) Shutdown(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.server == nil { + return nil + } + err := s.server.Shutdown(ctx) + s.server = nil + s.listener = nil + return err +} + +func (s *Service) RegisterInstance(ctx context.Context, gatewayIP string, cfg InstanceConfig) (GuestConfig, error) { + if err := s.EnsureStarted(ctx, gatewayIP); err != nil { + return GuestConfig{}, err + } + + if err := applyEgressEnforcement(cfg.InstanceID, cfg.TAPDevice, gatewayIP, s.listenPort, cfg.BlockAllTCPEgress); err != nil { + return GuestConfig{}, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if prevIP, ok := s.sourceIPByInstance[cfg.InstanceID]; ok { + delete(s.policiesBySourceIP, prevIP) + } + + rewriteRules, err := compileSecretRewriteRules(cfg.SecretRewriteRules) + if err != nil { + return GuestConfig{}, err + } + + s.sourceIPByInstance[cfg.InstanceID] = cfg.SourceIP + s.policiesBySourceIP[cfg.SourceIP] = sourcePolicy{secretRewriteRules: rewriteRules} + + return GuestConfig{ + Enabled: true, + ProxyURL: s.proxyURLLocked(), + CACertPEM: s.caPEM, + }, nil +} + +func compileSecretRewriteRules(cfgRules []SecretRewriteRuleConfig) ([]secretRewriteRule, error) { + if len(cfgRules) == 0 { + return nil, nil + } + out := make([]secretRewriteRule, 0, len(cfgRules)) + for _, cfg := range cfgRules { + if cfg.MockValue == "" || cfg.RealValue == "" { + continue + } + matchers, err := compileDomainMatchers(cfg.AllowedDomains) + if err != nil { + return nil, err + } + out = append(out, secretRewriteRule{ + mockValue: cfg.MockValue, + realValue: cfg.RealValue, + domainMatchers: matchers, + }) + } + return out, nil +} + +func (s *Service) UnregisterInstance(_ context.Context, instanceID string) { + s.mu.Lock() + sourceIP, ok := s.sourceIPByInstance[instanceID] + if ok { + delete(s.sourceIPByInstance, instanceID) + delete(s.policiesBySourceIP, sourceIP) + } + s.mu.Unlock() + + _ = removeEgressEnforcement(instanceID) +} + +func (s *Service) ProxyURL() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.proxyURLLocked() +} + +func (s *Service) proxyURLLocked() string { + if s.gatewayIP == "" { + return "" + } + return fmt.Sprintf("http://%s:%d", s.gatewayIP, s.listenPort) +} + +func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sourceIP := sourceIPFromRemoteAddr(r.RemoteAddr) + if r.Method == http.MethodConnect { + s.handleConnect(w, r, sourceIP) + return + } + s.handleHTTPProxyRequest(w, r, sourceIP, false) +} + +func (s *Service) handleHTTPProxyRequest(w http.ResponseWriter, r *http.Request, sourceIP string, insideTunnel bool) { + outReq := r.Clone(r.Context()) + outReq.Header = cloneHeader(r.Header) + removeHopByHopHeaders(outReq.Header) + + if !insideTunnel { + if outReq.URL == nil || !outReq.URL.IsAbs() { + outReq.URL = &url.URL{ + Scheme: "http", + Host: r.Host, + Path: r.URL.Path, + RawPath: r.URL.RawPath, + RawQuery: r.URL.RawQuery, + } + } + } + + outReq.RequestURI = "" + destinationHost := normalizeDestinationHost(outReq.URL.Host) + if destinationHost == "" { + destinationHost = normalizeDestinationHost(outReq.Host) + } + s.applyHeaderReplacements(sourceIP, destinationHost, outReq.Header, false) + + resp, err := s.transport.RoundTrip(outReq) + if err != nil { + http.Error(w, fmt.Sprintf("proxy upstream error: %v", err), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + removeHopByHopHeaders(resp.Header) + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) +} + +func (s *Service) handleConnect(w http.ResponseWriter, r *http.Request, sourceIP string) { + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "hijacking not supported", http.StatusInternalServerError) + return + } + + clientConn, _, err := hj.Hijack() + if err != nil { + http.Error(w, fmt.Sprintf("hijack failed: %v", err), http.StatusServiceUnavailable) + return + } + defer clientConn.Close() + + targetAuthority := strings.TrimSpace(r.Host) + targetHost := normalizeDestinationHost(targetAuthority) + if targetHost == "" { + return + } + + _, _ = io.WriteString(clientConn, "HTTP/1.1 200 Connection Established\r\n\r\n") + + cert, err := s.getOrCreateLeafCert(targetHost) + if err != nil { + return + } + + tlsConn := tls.Server(clientConn, &tls.Config{ + Certificates: []tls.Certificate{*cert}, + }) + if err := tlsConn.Handshake(); err != nil { + return + } + defer tlsConn.Close() + + reader := bufio.NewReader(tlsConn) + for { + req, err := http.ReadRequest(reader) + if err != nil { + if err == io.EOF { + return + } + return + } + + if req.URL == nil { + req.URL = &url.URL{} + } + if req.Host == "" { + req.Host = targetAuthority + } + req.URL.Scheme = "https" + req.URL.Host = targetAuthority + req.RequestURI = "" + req.Header = cloneHeader(req.Header) + removeHopByHopHeaders(req.Header) + s.applyHeaderReplacements(sourceIP, targetHost, req.Header, true) + + resp, err := s.transport.RoundTrip(req) + if err != nil { + _, _ = io.WriteString(tlsConn, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + + removeHopByHopHeaders(resp.Header) + if err := resp.Write(tlsConn); err != nil { + resp.Body.Close() + return + } + resp.Body.Close() + + if req.Close || resp.Close { + return + } + } +} + +func (s *Service) getOrCreateLeafCert(host string) (*tls.Certificate, error) { + s.mu.RLock() + cached := s.certCache[host] + s.mu.RUnlock() + if cached != nil { + return cached, nil + } + + cert, err := signHostCertificate(s.caCert, s.caKey, host) + if err != nil { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + if existing := s.certCache[host]; existing != nil { + return existing, nil + } + if s.certCacheLimit > 0 && len(s.certCache) >= s.certCacheLimit && len(s.certCacheOrder) > 0 { + evictHost := s.certCacheOrder[0] + s.certCacheOrder = s.certCacheOrder[1:] + delete(s.certCache, evictHost) + } + s.certCache[host] = cert + s.certCacheOrder = append(s.certCacheOrder, host) + return cert, nil +} + +func (s *Service) applyHeaderReplacements(sourceIP, destinationHost string, headers http.Header, isHTTPS bool) { + rules := s.resolveRewriteRules(sourceIP, destinationHost, isHTTPS) + if len(rules) == 0 { + return + } + + for key, vals := range headers { + for i := range vals { + updated := vals[i] + for _, rule := range rules { + updated = strings.ReplaceAll(updated, rule.mockValue, rule.realValue) + } + vals[i] = updated + } + headers[key] = vals + } +} + +func (s *Service) resolveRewriteRules(sourceIP, destinationHost string, isHTTPS bool) []secretRewriteRule { + if !isHTTPS { + return nil + } + + host := normalizeDestinationHost(destinationHost) + if host == "" { + return nil + } + + s.mu.RLock() + policy, ok := s.policiesBySourceIP[sourceIP] + s.mu.RUnlock() + if !ok { + return nil + } + + resolved := make([]secretRewriteRule, 0, len(policy.secretRewriteRules)) + for _, rule := range policy.secretRewriteRules { + if rule.mockValue == "" || rule.realValue == "" { + continue + } + if !matchesAnyDomain(host, rule.domainMatchers) { + continue + } + resolved = append(resolved, rule) + } + return resolved +} + +var hopByHopHeaders = []string{ + "Connection", + "Proxy-Connection", + "Keep-Alive", + "Proxy-Authenticate", + "Proxy-Authorization", + "Te", + "Trailer", + "Transfer-Encoding", + "Upgrade", +} + +func removeHopByHopHeaders(h http.Header) { + for _, k := range hopByHopHeaders { + h.Del(k) + } +} + +func cloneHeader(src http.Header) http.Header { + dst := make(http.Header, len(src)) + for k, vv := range src { + copied := make([]string, len(vv)) + copy(copied, vv) + dst[k] = copied + } + return dst +} + +func sourceIPFromRemoteAddr(remoteAddr string) string { + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return remoteAddr + } + return host +} diff --git a/lib/egressproxy/service_test.go b/lib/egressproxy/service_test.go new file mode 100644 index 00000000..f1a9667e --- /dev/null +++ b/lib/egressproxy/service_test.go @@ -0,0 +1,91 @@ +package egressproxy + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeAllowedDomainPattern(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want string + shouldErr bool + }{ + {name: "exact host", in: "API.OpenAI.com", want: "api.openai.com"}, + {name: "exact ip", in: "127.0.0.1", want: "127.0.0.1"}, + {name: "single wildcard", in: "*.OpenAI.com", want: "*.openai.com"}, + {name: "reject empty", in: "", shouldErr: true}, + {name: "reject scheme", in: "https://api.openai.com", shouldErr: true}, + {name: "reject port", in: "api.openai.com:443", shouldErr: true}, + {name: "reject global wildcard", in: "*", shouldErr: true}, + {name: "reject multi wildcard", in: "*.*.openai.com", shouldErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NormalizeAllowedDomainPattern(tt.in) + if tt.shouldErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestDomainMatcherSingleLevelWildcard(t *testing.T) { + t.Parallel() + + matchers, err := compileDomainMatchers([]string{"*.openai.com"}) + require.NoError(t, err) + require.Len(t, matchers, 1) + + require.True(t, matchesAnyDomain("api.openai.com", matchers)) + require.False(t, matchesAnyDomain("openai.com", matchers)) + require.False(t, matchesAnyDomain("a.b.openai.com", matchers)) +} + +func TestApplyHeaderReplacementsHTTPSOnlyAndDomainGated(t *testing.T) { + t.Parallel() + + matchers, err := compileDomainMatchers([]string{"api.openai.com"}) + require.NoError(t, err) + + svc := &Service{ + policiesBySourceIP: map[string]sourcePolicy{ + "10.0.0.2": { + secretRewriteRules: []secretRewriteRule{ + { + mockValue: "mock-OUTBOUND_OPENAI_KEY", + realValue: "real-openai-key-123", + domainMatchers: matchers, + }, + }, + }, + }, + } + + httpsAllowed := http.Header{ + "Authorization": []string{"Bearer mock-OUTBOUND_OPENAI_KEY"}, + } + svc.applyHeaderReplacements("10.0.0.2", "api.openai.com", httpsAllowed, true) + require.Equal(t, "Bearer real-openai-key-123", httpsAllowed.Get("Authorization")) + + httpsBlocked := http.Header{ + "Authorization": []string{"Bearer mock-OUTBOUND_OPENAI_KEY"}, + } + svc.applyHeaderReplacements("10.0.0.2", "api.github.com", httpsBlocked, true) + require.Equal(t, "Bearer mock-OUTBOUND_OPENAI_KEY", httpsBlocked.Get("Authorization")) + + httpAllowedDomain := http.Header{ + "Authorization": []string{"Bearer mock-OUTBOUND_OPENAI_KEY"}, + } + svc.applyHeaderReplacements("10.0.0.2", "api.openai.com", httpAllowedDomain, false) + require.Equal(t, "Bearer mock-OUTBOUND_OPENAI_KEY", httpAllowedDomain.Get("Authorization")) +} diff --git a/lib/egressproxy/types.go b/lib/egressproxy/types.go new file mode 100644 index 00000000..172e1231 --- /dev/null +++ b/lib/egressproxy/types.go @@ -0,0 +1,39 @@ +package egressproxy + +import "errors" + +const ( + DefaultListenPort = 18080 +) + +var ( + ErrGatewayMismatch = errors.New("egress proxy already initialized with different gateway") +) + +// InstanceConfig defines per-instance proxy behavior. +type InstanceConfig struct { + InstanceID string + SourceIP string + TAPDevice string + BlockAllTCPEgress bool + SecretRewriteRules []SecretRewriteRuleConfig +} + +// SecretRewriteRuleConfig defines one mocked secret substitution policy. +type SecretRewriteRuleConfig struct { + MockValue string + RealValue string + AllowedDomains []string // optional exact or *.example.com patterns; empty means allow all +} + +// ServiceOptions customizes service construction (primarily for tests). +type ServiceOptions struct { + AdditionalRootCAPEM []string +} + +// GuestConfig is injected into guest config.json when proxy mode is enabled. +type GuestConfig struct { + Enabled bool `json:"enabled"` + ProxyURL string `json:"proxy_url"` + CACertPEM string `json:"ca_cert_pem"` +} diff --git a/lib/instances/configdisk.go b/lib/instances/configdisk.go index 4e17051c..2b267a85 100644 --- a/lib/instances/configdisk.go +++ b/lib/instances/configdisk.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/vmconfig" @@ -16,7 +17,7 @@ import ( // createConfigDisk generates an ext4 disk with instance configuration. // The disk contains /config.json read by the guest init binary. -func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig) error { +func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig, proxyCfg *egressproxy.GuestConfig) error { // Create temporary directory for config files tmpDir, err := os.MkdirTemp("", "hypeman-config-*") if err != nil { @@ -25,7 +26,7 @@ func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInf defer os.RemoveAll(tmpDir) // Generate config.json - cfg := m.buildGuestConfig(ctx, inst, imageInfo, netConfig) + cfg := m.buildGuestConfig(ctx, inst, imageInfo, netConfig, proxyCfg) configData, err := json.MarshalIndent(cfg, "", " ") if err != nil { return fmt.Errorf("marshal config: %w", err) @@ -46,7 +47,7 @@ func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInf } // buildGuestConfig creates the vmconfig.Config struct for the guest init binary. -func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig) *vmconfig.Config { +func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig, proxyCfg *egressproxy.GuestConfig) *vmconfig.Config { // Use instance overrides if set, otherwise fall back to image defaults // (like docker run overriding CMD) entrypoint := imageInfo.Entrypoint @@ -79,6 +80,23 @@ func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInf cfg.GuestDNS = netConfig.DNS } + if proxyCfg != nil && proxyCfg.Enabled { + cfg.EgressProxy = &vmconfig.EgressProxyConfig{ + Enabled: true, + ProxyURL: proxyCfg.ProxyURL, + CACertPEM: proxyCfg.CACertPEM, + } + if inst.EgressProxy != nil { + for _, envVar := range inst.EgressProxy.MockEnvVars { + cfg.Env[envVar] = mockValueForEnvVar(envVar) + } + } + cfg.Env["HTTP_PROXY"] = proxyCfg.ProxyURL + cfg.Env["HTTPS_PROXY"] = proxyCfg.ProxyURL + cfg.Env["http_proxy"] = proxyCfg.ProxyURL + cfg.Env["https_proxy"] = proxyCfg.ProxyURL + } + // Volume mounts // Volumes are attached as /dev/vdd, /dev/vde, etc. (after vda=rootfs, vdb=overlay, vdc=config) deviceIdx := 0 diff --git a/lib/instances/create.go b/lib/instances/create.go index 1cc93876..800bd9cc 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/guestmemory" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" @@ -297,6 +298,7 @@ func (m *manager) createInstance( Env: req.Env, Tags: tags.Clone(req.Tags), NetworkEnabled: req.NetworkEnabled, + EgressProxy: cloneEgressProxyConfig(req.EgressProxy), CreatedAt: time.Now(), StartedAt: nil, StoppedAt: nil, @@ -403,8 +405,21 @@ func (m *manager) createInstance( // 16. Create config disk (needs Instance for buildVMConfig) inst := &Instance{StoredMetadata: *stored} + var proxyGuestConfig *egressproxy.GuestConfig + if networkName != "" { + proxyGuestConfig, err = m.maybeRegisterEgressProxy(ctx, stored, netConfig) + if err != nil { + log.ErrorContext(ctx, "failed to configure egress proxy", "instance_id", id, "error", err) + return nil, fmt.Errorf("configure egress proxy: %w", err) + } + if proxyGuestConfig != nil { + cu.Add(func() { + m.unregisterEgressProxyInstance(ctx, id) + }) + } + } log.DebugContext(ctx, "creating config disk", "instance_id", id) - if err := m.createConfigDisk(ctx, inst, imageInfo, netConfig); err != nil { + if err := m.createConfigDisk(ctx, inst, imageInfo, netConfig, proxyGuestConfig); err != nil { log.ErrorContext(ctx, "failed to create config disk", "instance_id", id, "error", err) return nil, fmt.Errorf("create config disk: %w", err) } @@ -474,6 +489,35 @@ func validateCreateRequest(req CreateInstanceRequest) error { if req.Vcpus < 0 { return fmt.Errorf("vcpus cannot be negative") } + if req.EgressProxy != nil && req.EgressProxy.Enabled { + if !req.NetworkEnabled { + return fmt.Errorf("%w: egress proxy requires network_enabled=true", ErrInvalidRequest) + } + mode, err := normalizeEgressProxyEnforcementMode(req.EgressProxy.EnforcementMode) + if err != nil { + return err + } + req.EgressProxy.EnforcementMode = mode + normalized, err := normalizeMockEnvVars(req.EgressProxy.MockEnvVars) + if err != nil { + return err + } + req.EgressProxy.MockEnvVars = normalized + normalizedDomains, err := normalizeMockEnvVarDomains(normalized, req.EgressProxy.MockEnvVarDomains) + if err != nil { + return err + } + req.EgressProxy.MockEnvVarDomains = normalizedDomains + for _, envVar := range normalized { + real, ok := req.Env[envVar] + if !ok { + return fmt.Errorf("%w: egress proxy mock env var %q must be present in env", ErrInvalidRequest, envVar) + } + if strings.TrimSpace(real) == "" { + return fmt.Errorf("%w: env var %q must be non-empty when listed in egress_proxy.mock_env_vars", ErrInvalidRequest, envVar) + } + } + } if err := tags.Validate(req.Tags); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } diff --git a/lib/instances/create_egress_proxy_test.go b/lib/instances/create_egress_proxy_test.go new file mode 100644 index 00000000..59546632 --- /dev/null +++ b/lib/instances/create_egress_proxy_test.go @@ -0,0 +1,304 @@ +package instances + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateCreateRequest_EgressProxyRequiresNetwork(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: false, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "network_enabled=true") +} + +func TestValidateCreateRequest_EgressProxyMissingEnvVar(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{}, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "must be present in env") +} + +func TestValidateCreateRequest_EgressProxyRejectsEmptyEnvValue(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": " ", + }, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "must be non-empty") +} + +func TestValidateCreateRequest_EgressProxyRejectsEmptyMockEnvVarEntry(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{" "}, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "must be non-empty") +} + +func TestValidateCreateRequest_EgressProxyDedupesMockEnvVars(t *testing.T) { + t.Parallel() + + cfg := &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{ + "OUTBOUND_OPENAI_KEY", + " OUTBOUND_OPENAI_KEY ", + "GITHUB_TOKEN", + }, + } + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + "GITHUB_TOKEN": "real-gh-token", + }, + EgressProxy: cfg, + } + + err := validateCreateRequest(req) + require.NoError(t, err) + assert.Equal(t, []string{"OUTBOUND_OPENAI_KEY", "GITHUB_TOKEN"}, cfg.MockEnvVars) + assert.Equal(t, EgressProxyEnforcementModeAll, cfg.EnforcementMode) +} + +func TestValidateCreateRequest_EgressProxyRejectsInvalidEnforcementMode(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + EnforcementMode: EgressProxyEnforcementMode("bogus"), + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "invalid egress proxy enforcement_mode") +} + +func TestValidateCreateRequest_EgressProxyAllowsHTTPHTTPSOnlyMode(t *testing.T) { + t.Parallel() + + cfg := &EgressProxyConfig{ + Enabled: true, + EnforcementMode: EgressProxyEnforcementModeHTTPHTTPSOnly, + MockEnvVars: []string{ + "OUTBOUND_OPENAI_KEY", + }, + } + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + EgressProxy: cfg, + } + + err := validateCreateRequest(req) + require.NoError(t, err) + assert.Equal(t, EgressProxyEnforcementModeHTTPHTTPSOnly, cfg.EnforcementMode) +} + +func TestValidateCreateRequest_EgressProxyRejectsUnknownDomainMapKey(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + MockEnvVarDomains: map[string][]string{ + "GITHUB_TOKEN": []string{"api.github.com"}, + }, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "must be present in mock_env_vars") +} + +func TestValidateCreateRequest_EgressProxyRejectsEmptyDomainList(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + MockEnvVarDomains: map[string][]string{ + "OUTBOUND_OPENAI_KEY": {}, + }, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "must have at least one domain pattern") +} + +func TestValidateCreateRequest_EgressProxyRejectsInvalidDomainPattern(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + MockEnvVarDomains: map[string][]string{ + "OUTBOUND_OPENAI_KEY": []string{"https://api.openai.com"}, + }, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "invalid egress proxy domain pattern") +} + +func TestValidateCreateRequest_EgressProxyNormalizesDomainPatterns(t *testing.T) { + t.Parallel() + + cfg := &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + MockEnvVarDomains: map[string][]string{ + " OUTBOUND_OPENAI_KEY ": { + " API.OpenAI.com ", + "*.OpenAI.com", + "api.openai.com", + }, + }, + } + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + EgressProxy: cfg, + } + + err := validateCreateRequest(req) + require.NoError(t, err) + assert.Equal(t, map[string][]string{ + "OUTBOUND_OPENAI_KEY": []string{"api.openai.com", "*.openai.com"}, + }, cfg.MockEnvVarDomains) +} + +func TestValidateCreateRequest_EgressProxyAllowsUnmappedMockEnvVarDomains(t *testing.T) { + t.Parallel() + + cfg := &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY", "GITHUB_TOKEN"}, + MockEnvVarDomains: map[string][]string{ + "OUTBOUND_OPENAI_KEY": []string{"api.openai.com"}, + }, + } + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + "GITHUB_TOKEN": "real-gh-token", + }, + EgressProxy: cfg, + } + + err := validateCreateRequest(req) + require.NoError(t, err) + assert.Equal(t, map[string][]string{ + "OUTBOUND_OPENAI_KEY": []string{"api.openai.com"}, + }, cfg.MockEnvVarDomains) +} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index b54d5b3c..21fbfaab 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -76,6 +76,7 @@ func (m *manager) deleteInstance( // 6. Release network allocation if inst.NetworkEnabled { + m.unregisterEgressProxyInstance(ctx, id) log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default") if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil { // Log error but continue with cleanup diff --git a/lib/instances/egress_proxy.go b/lib/instances/egress_proxy.go new file mode 100644 index 00000000..dd56910e --- /dev/null +++ b/lib/instances/egress_proxy.go @@ -0,0 +1,200 @@ +package instances + +import ( + "context" + "fmt" + "strings" + + "github.com/kernel/hypeman/lib/egressproxy" + "github.com/kernel/hypeman/lib/network" +) + +const mockSecretPrefix = "mock-" + +func cloneEgressProxyConfig(cfg *EgressProxyConfig) *EgressProxyConfig { + if cfg == nil { + return nil + } + out := &EgressProxyConfig{ + Enabled: cfg.Enabled, + EnforcementMode: cfg.EnforcementMode, + } + if cfg.MockEnvVars != nil { + out.MockEnvVars = append([]string(nil), cfg.MockEnvVars...) + } + if cfg.MockEnvVarDomains != nil { + out.MockEnvVarDomains = cloneMockEnvVarDomains(cfg.MockEnvVarDomains) + } + return out +} + +func normalizeMockEnvVars(in []string) ([]string, error) { + if len(in) == 0 { + return nil, nil + } + + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, name := range in { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return nil, fmt.Errorf("%w: egress proxy mock_env_vars entries must be non-empty", ErrInvalidRequest) + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out, nil +} + +func mockValueForEnvVar(name string) string { + return mockSecretPrefix + name +} + +func cloneMockEnvVarDomains(in map[string][]string) map[string][]string { + if len(in) == 0 { + return nil + } + out := make(map[string][]string, len(in)) + for envVar, patterns := range in { + out[envVar] = append([]string(nil), patterns...) + } + return out +} + +func normalizeEgressProxyEnforcementMode(mode EgressProxyEnforcementMode) (EgressProxyEnforcementMode, error) { + trimmed := strings.TrimSpace(string(mode)) + switch EgressProxyEnforcementMode(trimmed) { + case "", EgressProxyEnforcementModeAll: + return EgressProxyEnforcementModeAll, nil + case EgressProxyEnforcementModeHTTPHTTPSOnly: + return EgressProxyEnforcementModeHTTPHTTPSOnly, nil + default: + return "", fmt.Errorf("%w: invalid egress proxy enforcement_mode %q", ErrInvalidRequest, trimmed) + } +} + +func normalizeMockEnvVarDomains(mockEnvVars []string, in map[string][]string) (map[string][]string, error) { + if len(in) == 0 { + return nil, nil + } + + allowedVars := make(map[string]struct{}, len(mockEnvVars)) + for _, name := range mockEnvVars { + allowedVars[name] = struct{}{} + } + + out := make(map[string][]string, len(in)) + for rawEnvVar, rawPatterns := range in { + envVar := strings.TrimSpace(rawEnvVar) + if envVar == "" { + return nil, fmt.Errorf("%w: egress proxy mock_env_var_domains key must be non-empty", ErrInvalidRequest) + } + if _, ok := allowedVars[envVar]; !ok { + return nil, fmt.Errorf("%w: egress proxy mock_env_var_domains key %q must be present in mock_env_vars", ErrInvalidRequest, envVar) + } + if len(rawPatterns) == 0 { + return nil, fmt.Errorf("%w: egress proxy mock_env_var_domains[%q] must have at least one domain pattern", ErrInvalidRequest, envVar) + } + + seen := make(map[string]struct{}, len(rawPatterns)) + patterns := make([]string, 0, len(rawPatterns)) + for _, rawPattern := range rawPatterns { + normalized, err := egressproxy.NormalizeAllowedDomainPattern(rawPattern) + if err != nil { + return nil, fmt.Errorf("%w: invalid egress proxy domain pattern %q for %q: %v", ErrInvalidRequest, rawPattern, envVar, err) + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + patterns = append(patterns, normalized) + } + if len(patterns) == 0 { + return nil, fmt.Errorf("%w: egress proxy mock_env_var_domains[%q] must include at least one valid domain pattern", ErrInvalidRequest, envVar) + } + out[envVar] = patterns + } + return out, nil +} + +func buildEgressProxyRewriteRules(cfg *EgressProxyConfig, env map[string]string) []egressproxy.SecretRewriteRuleConfig { + if cfg == nil || !cfg.Enabled || len(cfg.MockEnvVars) == 0 { + return nil + } + out := make([]egressproxy.SecretRewriteRuleConfig, 0, len(cfg.MockEnvVars)) + for _, envVar := range cfg.MockEnvVars { + real := env[envVar] + if real == "" { + continue + } + rule := egressproxy.SecretRewriteRuleConfig{ + MockValue: mockValueForEnvVar(envVar), + RealValue: real, + } + if cfg.MockEnvVarDomains != nil && len(cfg.MockEnvVarDomains[envVar]) > 0 { + rule.AllowedDomains = append([]string(nil), cfg.MockEnvVarDomains[envVar]...) + } + out = append(out, rule) + } + if len(out) == 0 { + return nil + } + return out +} + +func (m *manager) getOrCreateEgressProxyService() (*egressproxy.Service, error) { + m.egressProxyMu.Lock() + defer m.egressProxyMu.Unlock() + + if m.egressProxy != nil { + return m.egressProxy, nil + } + + svc, err := egressproxy.NewServiceWithOptions(m.paths.DataDir(), egressproxy.DefaultListenPort, m.egressProxyServiceOptions) + if err != nil { + return nil, err + } + m.egressProxy = svc + return svc, nil +} + +func (m *manager) maybeRegisterEgressProxy(ctx context.Context, stored *StoredMetadata, netConfig *network.NetworkConfig) (*egressproxy.GuestConfig, error) { + if stored == nil || stored.EgressProxy == nil || !stored.EgressProxy.Enabled { + return nil, nil + } + if !stored.NetworkEnabled || netConfig == nil { + return nil, fmt.Errorf("egress proxy requires network_enabled=true") + } + + svc, err := m.getOrCreateEgressProxyService() + if err != nil { + return nil, fmt.Errorf("create egress proxy service: %w", err) + } + + guestCfg, err := svc.RegisterInstance(ctx, netConfig.Gateway, egressproxy.InstanceConfig{ + InstanceID: stored.Id, + SourceIP: netConfig.IP, + TAPDevice: netConfig.TAPDevice, + BlockAllTCPEgress: stored.EgressProxy.EnforcementMode != EgressProxyEnforcementModeHTTPHTTPSOnly, + SecretRewriteRules: buildEgressProxyRewriteRules(stored.EgressProxy, stored.Env), + }) + if err != nil { + return nil, fmt.Errorf("register instance with egress proxy: %w", err) + } + + return &guestCfg, nil +} + +func (m *manager) unregisterEgressProxyInstance(ctx context.Context, instanceID string) { + _ = ctx + m.egressProxyMu.Lock() + svc := m.egressProxy + m.egressProxyMu.Unlock() + if svc == nil { + return + } + svc.UnregisterInstance(context.Background(), instanceID) +} diff --git a/lib/instances/egress_proxy_integration_test.go b/lib/instances/egress_proxy_integration_test.go new file mode 100644 index 00000000..693f0b41 --- /dev/null +++ b/lib/instances/egress_proxy_integration_test.go @@ -0,0 +1,182 @@ +package instances + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/egressproxy" + "github.com/kernel/hypeman/lib/images" + "github.com/stretchr/testify/require" +) + +func TestEgressProxyRewritesHTTPSHeaders(t *testing.T) { + requireKVMAccess(t) + + manager, _ := setupTestManager(t) + ctx := context.Background() + + caPEM, cert := mustGenerateTLSChain(t, []string{"localhost"}) + manager.egressProxyServiceOptions = egressproxy.ServiceOptions{ + AdditionalRootCAPEM: []string{caPEM}, + } + + target := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, r.Header.Get("Authorization")) + })) + target.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + target.StartTLS() + defer target.Close() + targetHostPort := strings.TrimPrefix(target.URL, "https://") + targetHost, targetPort, err := net.SplitHostPort(targetHostPort) + require.NoError(t, err) + + imageRef := integrationTestImageRef(t, "docker.io/library/nginx:alpine") + t.Logf("Pulling %s image...", imageRef) + created, err := manager.imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageRef}) + require.NoError(t, err) + + for i := 0; i < 120; i++ { + img, err := manager.imageManager.GetImage(ctx, created.Name) + if err == nil && img.Status == images.StatusReady { + break + } + time.Sleep(1 * time.Second) + } + img, err := manager.imageManager.GetImage(ctx, created.Name) + require.NoError(t, err) + require.Equal(t, images.StatusReady, img.Status) + + require.NoError(t, manager.systemManager.EnsureSystemFiles(ctx)) + require.NoError(t, manager.networkManager.Initialize(ctx, nil)) + + inst, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "test-egress-proxy", + Image: imageRef, + Size: 2 * 1024 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 5 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + EgressProxy: &EgressProxyConfig{ + Enabled: true, + MockEnvVars: []string{"OUTBOUND_OPENAI_KEY"}, + MockEnvVarDomains: map[string][]string{ + "OUTBOUND_OPENAI_KEY": []string{"127.0.0.1"}, + }, + }, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-openai-key-123", + }, + Entrypoint: []string{"/bin/sh", "-lc"}, + Cmd: []string{"sleep 3600"}, + }) + require.NoError(t, err) + + deleted := false + t.Cleanup(func() { + if !deleted { + _ = manager.DeleteInstance(context.Background(), inst.Id) + } + }) + + require.NoError(t, waitForVMReady(ctx, inst.SocketPath, 10*time.Second)) + require.NoError(t, waitForLogMessage(ctx, manager, inst.Id, "[guest-agent] listening", 45*time.Second)) + + envOutput, envExitCode, err := execCommand(ctx, inst, "sh", "-lc", "printf '%s' \"$OUTBOUND_OPENAI_KEY\"") + require.NoError(t, err) + require.Equal(t, 0, envExitCode) + require.Equal(t, "mock-OUTBOUND_OPENAI_KEY", envOutput) + + allowedCmd := fmt.Sprintf( + "NO_PROXY= no_proxy= curl -k -sS -H \"Authorization: Bearer $OUTBOUND_OPENAI_KEY\" https://%s:%s", + targetHost, targetPort, + ) + output, exitCode, err := execCommand(ctx, inst, "sh", "-lc", allowedCmd) + require.NoError(t, err) + require.Equal(t, 0, exitCode, "curl output: %s", output) + require.Contains(t, output, "Bearer real-openai-key-123") + + blockedCmd := fmt.Sprintf( + "NO_PROXY= no_proxy= curl -k -sS -H \"Authorization: Bearer $OUTBOUND_OPENAI_KEY\" https://localhost:%s", + targetPort, + ) + blockedOutput, blockedExitCode, err := execCommand(ctx, inst, "sh", "-lc", blockedCmd) + require.NoError(t, err) + require.Equal(t, 0, blockedExitCode, "curl output: %s", blockedOutput) + require.Contains(t, blockedOutput, "Bearer mock-OUTBOUND_OPENAI_KEY") + + require.NoError(t, manager.DeleteInstance(ctx, inst.Id)) + deleted = true +} + +func mustGenerateTLSChain(t *testing.T, dnsNames []string) (string, tls.Certificate) { + t.Helper() + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + serialLimit := new(big.Int).Lsh(big.NewInt(1), 128) + caSerial, err := rand.Int(rand.Reader, serialLimit) + require.NoError(t, err) + + now := time.Now() + caTemplate := &x509.Certificate{ + SerialNumber: caSerial, + Subject: pkix.Name{ + CommonName: "egress-proxy-test-ca", + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLen: 1, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + require.NoError(t, err) + + caCert, err := x509.ParseCertificate(caDER) + require.NoError(t, err) + + serverKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + serverSerial, err := rand.Int(rand.Reader, serialLimit) + require.NoError(t, err) + + serverTemplate := &x509.Certificate{ + SerialNumber: serverSerial, + Subject: pkix.Name{ + CommonName: dnsNames[0], + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: dnsNames, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + SubjectKeyId: []byte{1, 2, 3, 4}, + } + serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caCert, &serverKey.PublicKey, caKey) + require.NoError(t, err) + + serverCertPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}) + serverKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)}) + cert, err := tls.X509KeyPair(serverCertPEM, serverKeyPEM) + require.NoError(t, err) + + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}) + return string(caPEM), cert +} diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 108381f8..5145932f 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -467,6 +467,9 @@ func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { dst.Env[k] = v } } + if src.EgressProxy != nil { + dst.EgressProxy = cloneEgressProxyConfig(src.EgressProxy) + } if src.Tags != nil { dst.Tags = make(map[string]string, len(src.Tags)) for k, v := range src.Tags { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index f2b19571..76ad7e4f 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -7,6 +7,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/guestmemory" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" @@ -71,19 +72,22 @@ type ResourceValidator interface { } type manager struct { - paths *paths.Paths - imageManager images.Manager - systemManager system.Manager - networkManager network.Manager - deviceManager devices.Manager - volumeManager volumes.Manager - limits ResourceLimits - resourceValidator ResourceValidator // Optional validator for aggregate resource limits - instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks - bootMarkerScans sync.Map // map[string]time.Time next allowed boot-marker rescan - hostTopology *HostTopology // Cached host CPU topology - metrics *Metrics - now func() time.Time + paths *paths.Paths + imageManager images.Manager + systemManager system.Manager + networkManager network.Manager + deviceManager devices.Manager + volumeManager volumes.Manager + limits ResourceLimits + resourceValidator ResourceValidator // Optional validator for aggregate resource limits + instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks + bootMarkerScans sync.Map // map[string]time.Time next allowed boot-marker rescan + hostTopology *HostTopology // Cached host CPU topology + metrics *Metrics + now func() time.Time + egressProxy *egressproxy.Service + egressProxyServiceOptions egressproxy.ServiceOptions + egressProxyMu sync.Mutex // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 4b1d4395..f1754c50 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -75,10 +75,15 @@ func (m *manager) restoreInstance( } var allocatedNet *network.Allocation + proxyRegistered := false releaseNetwork := func() { if !stored.NetworkEnabled { return } + if proxyRegistered { + m.unregisterEgressProxyInstance(ctx, id) + proxyRegistered = false + } if allocatedNet != nil { if err := m.networkManager.ReleaseAllocation(ctx, allocatedNet); err != nil { log.WarnContext(ctx, "failed to release allocated network", "instance_id", id, "error", err) @@ -169,6 +174,46 @@ func (m *manager) restoreInstance( } } + // 4b. Register proxy/enforcement once network identity is active. + if stored.NetworkEnabled { + alloc, allocErr := m.networkManager.GetAllocation(ctx, id) + if allocErr != nil && stored.EgressProxy != nil && stored.EgressProxy.Enabled { + log.ErrorContext(ctx, "failed to fetch allocation for proxy setup", "instance_id", id, "error", allocErr) + releaseNetwork() + return nil, fmt.Errorf("get network allocation for proxy setup: %w", allocErr) + } + if allocErr == nil && alloc != nil { + proxyCfg := &network.NetworkConfig{ + IP: alloc.IP, + MAC: alloc.MAC, + Gateway: alloc.Gateway, + Netmask: alloc.Netmask, + TAPDevice: alloc.TAPDevice, + } + proxyGuestConfig, err := m.maybeRegisterEgressProxy(ctx, stored, proxyCfg) + if err != nil { + log.ErrorContext(ctx, "failed to configure egress proxy", "instance_id", id, "error", err) + releaseNetwork() + return nil, fmt.Errorf("configure egress proxy: %w", err) + } + imageInfo, err := m.imageManager.GetImage(ctx, stored.Image) + if err != nil { + log.ErrorContext(ctx, "failed to load image for config disk refresh", "instance_id", id, "image", stored.Image, "error", err) + releaseNetwork() + return nil, fmt.Errorf("get image for restore config disk: %w", err) + } + instForConfig := &Instance{StoredMetadata: *stored} + if err := m.createConfigDisk(ctx, instForConfig, imageInfo, proxyCfg, proxyGuestConfig); err != nil { + log.ErrorContext(ctx, "failed to refresh config disk for restore", "instance_id", id, "error", err) + releaseNetwork() + return nil, fmt.Errorf("refresh restore config disk: %w", err) + } + if stored.EgressProxy != nil && stored.EgressProxy.Enabled { + proxyRegistered = true + } + } + } + // 5. Transition: Standby → Paused (start hypervisor + restore) var restoreSpan trace.Span if m.metrics != nil && m.metrics.tracer != nil { diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 1fca6dfb..ca12b8f3 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -116,6 +116,7 @@ func (m *manager) standbyInstance( // TAP devices with explicit Owner/Group fields do NOT auto-delete when VMM exits // They must be explicitly deleted if inst.NetworkEnabled { + m.unregisterEgressProxyInstance(ctx, id) log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default") if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil { // Log error but continue - snapshot was created successfully diff --git a/lib/instances/start.go b/lib/instances/start.go index 7d5ab265..0345e395 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "go.opentelemetry.io/otel/trace" @@ -105,6 +106,20 @@ func (m *manager) startInstance( }) } + var proxyGuestConfig *egressproxy.GuestConfig + if stored.NetworkEnabled { + proxyGuestConfig, err = m.maybeRegisterEgressProxy(ctx, stored, netConfig) + if err != nil { + log.ErrorContext(ctx, "failed to configure egress proxy", "instance_id", id, "error", err) + return nil, fmt.Errorf("configure egress proxy: %w", err) + } + if proxyGuestConfig != nil { + cu.Add(func() { + m.unregisterEgressProxyInstance(ctx, id) + }) + } + } + // 4b. Recreate vGPU mdev if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { @@ -128,7 +143,7 @@ func (m *manager) startInstance( // 5. Regenerate config disk with new network configuration instForConfig := &Instance{StoredMetadata: *stored} log.DebugContext(ctx, "regenerating config disk", "instance_id", id) - if err := m.createConfigDisk(ctx, instForConfig, imageInfo, netConfig); err != nil { + if err := m.createConfigDisk(ctx, instForConfig, imageInfo, netConfig, proxyGuestConfig); err != nil { log.ErrorContext(ctx, "failed to create config disk", "instance_id", id, "error", err) return nil, fmt.Errorf("create config disk: %w", err) } diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 2b9abfee..f70627af 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -199,6 +199,9 @@ func (m *manager) stopInstance( } // 6. Release network allocation (delete TAP device) + if inst.NetworkEnabled { + m.unregisterEgressProxyInstance(ctx, id) + } if inst.NetworkEnabled && networkAlloc != nil { log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default") if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil { diff --git a/lib/instances/types.go b/lib/instances/types.go index 8bf2043f..9e97274e 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -22,6 +22,13 @@ const ( StateUnknown State = "Unknown" // Failed to determine state (VMM query failed) ) +type EgressProxyEnforcementMode string + +const ( + EgressProxyEnforcementModeAll EgressProxyEnforcementMode = "all" + EgressProxyEnforcementModeHTTPHTTPSOnly EgressProxyEnforcementMode = "http_https_only" +) + // VolumeAttachment represents a volume attached to an instance type VolumeAttachment struct { VolumeID string // Volume ID @@ -31,6 +38,15 @@ type VolumeAttachment struct { OverlaySize int64 // Size of overlay disk in bytes (max diff from base) } +// EgressProxyConfig configures optional per-instance egress MITM behavior. +// Real secret values are provided via Env and persisted there. +type EgressProxyConfig struct { + Enabled bool // Whether egress proxy mode is enabled + MockEnvVars []string // Env var names to mock in guest and rewrite on egress + MockEnvVarDomains map[string][]string // Optional env var -> allowed destination domain patterns for substitution + EnforcementMode EgressProxyEnforcementMode // all (default) blocks direct non-proxy TCP egress, http_https_only blocks only 80/443 +} + // StoredMetadata represents instance metadata that is persisted to disk type StoredMetadata struct { // Identification @@ -51,8 +67,9 @@ type StoredMetadata struct { Env map[string]string Tags tags.Tags // User-defined key-value tags NetworkEnabled bool // Whether instance has networking enabled (uses default network) - IP string // Assigned IP address (empty if NetworkEnabled=false) - MAC string // Assigned MAC address (empty if NetworkEnabled=false) + EgressProxy *EgressProxyConfig + IP string // Assigned IP address (empty if NetworkEnabled=false) + MAC string // Assigned MAC address (empty if NetworkEnabled=false) // Attached volumes Volumes []VolumeAttachment // Volumes attached to this instance @@ -167,6 +184,7 @@ type CreateInstanceRequest struct { Env map[string]string // Optional environment variables Tags tags.Tags // Optional user-defined key-value tags NetworkEnabled bool // Whether to enable networking (uses default network) + EgressProxy *EgressProxyConfig // Optional egress MITM proxy mode Devices []string // Device IDs or names to attach (GPU passthrough) Volumes []VolumeAttachment // Volumes to attach at creation time Hypervisor hypervisor.Type // Optional: hypervisor type (defaults to config) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index e4290fb0..73a2d75a 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -46,6 +46,12 @@ const ( BuildStatusReady BuildStatus = "ready" ) +// Defines values for CreateInstanceRequestEgressProxyEnforcementMode. +const ( + All CreateInstanceRequestEgressProxyEnforcementMode = "all" + HttpHttpsOnly CreateInstanceRequestEgressProxyEnforcementMode = "http_https_only" +) + // Defines values for CreateInstanceRequestHypervisor. const ( CreateInstanceRequestHypervisorCloudHypervisor CreateInstanceRequestHypervisor = "cloud-hypervisor" @@ -307,6 +313,27 @@ type CreateInstanceRequest struct { // DiskIoBps Disk I/O rate limit (e.g., "100MB/s", "500MB/s"). Defaults to proportional share based on CPU allocation if configured. DiskIoBps *string `json:"disk_io_bps,omitempty"` + // EgressProxy Optional host-side HTTP/HTTPS MITM egress proxy mode. + EgressProxy *struct { + // Enabled Whether to enable egress proxy mode. + Enabled *bool `json:"enabled,omitempty"` + + // EnforcementMode Egress proxy host enforcement mode. + // `all` (default when proxy is enabled) rejects direct non-proxy TCP egress from the VM, + // while `http_https_only` rejects direct egress only on TCP ports 80 and 443. + EnforcementMode *CreateInstanceRequestEgressProxyEnforcementMode `json:"enforcement_mode,omitempty"` + + // MockEnvVarDomains Optional per-mocked-env destination domain allowlist for secret substitution. + // Keys are env var names from `mock_env_vars`; values are allowed destination + // host patterns (`api.example.com`, `*.example.com`). If a mock env var is not + // present in this map, substitution is allowed for all HTTPS destinations. + MockEnvVarDomains *map[string][]string `json:"mock_env_var_domains,omitempty"` + + // MockEnvVars Environment variable names (from `env`) that should be mocked inside the VM + // as `mock-` and rewritten back to their real values on egress. + MockEnvVars *[]string `json:"mock_env_vars,omitempty"` + } `json:"egress_proxy,omitempty"` + // Entrypoint Override image entrypoint (like docker run --entrypoint). Omit to use image default. Entrypoint *[]string `json:"entrypoint,omitempty"` @@ -367,6 +394,11 @@ type CreateInstanceRequest struct { Volumes *[]VolumeMount `json:"volumes,omitempty"` } +// CreateInstanceRequestEgressProxyEnforcementMode Egress proxy host enforcement mode. +// `all` (default when proxy is enabled) rejects direct non-proxy TCP egress from the VM, +// while `http_https_only` rejects direct egress only on TCP ports 80 and 443. +type CreateInstanceRequestEgressProxyEnforcementMode string + // CreateInstanceRequestHypervisor Hypervisor to use for this instance. Defaults to server configuration. type CreateInstanceRequestHypervisor string @@ -13298,201 +13330,209 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XLbOJboq6B0d2vkHUmWP+I42ura68RJ2ttx4hvH3rvTylUgEpLQJgE2AMpRUvk7", - "DzCPOE9yCwcAvwRKlGM78SZTUx2ZBPFxcHBwvs/nVsDjhDPClGwNPrdkMCMxhp9HSuFgdsmjNCZvyZ8p", - "kUo/TgRPiFCUQKOYp0yNEqxm+q+QyEDQRFHOWoPWGVYzdD0jgqA59ILkjKdRiMYEwXckbHVa5COOk4i0", - "Bq3tmKntECvc6rTUItGPpBKUTVtfOi1BcMhZtDDDTHAaqdZggiNJOpVhT3XXCEukP+nCN1l/Y84jglnr", - "C/T4Z0oFCVuD34vLeJ815uM/SKD04EdzTCM8jsgxmdOALIMhSIUgTI1CQedELIPimXkfLdCYpyxEph1q", - "szSKEJ0gxhnZKgGDzWlINSR0Ez10a6BESjyQCWFOIxp6duDZCTKv0ckxas/Ix/Igu4/Hh636LhmOyXKn", - "v6YxZl0NXD0t1z+0Lfb9at/XM+VxnI6mgqfJcs8nb05PLxC8RCyNx0QUezzczfqjTJEpEbrDJKAjHIaC", - "SOlfv3tZnFu/3+8P8O6g3+/1fbOcExZyUQtS89oP0p1+SFZ02Qiktv8lkL6+PDk+OULPuEi4wPDt0kgV", - "xC6Cp7iuItqUd8WH/09TGoXLWD/Wj4kYUSYVZjU4eGJfanDxCVIzgux36PIUtSdcoJCM0+mUsulWE3zX", - "BCsiioQjrJaHg6ki24ZyhhSNiVQ4Tlqd1oSLWH/UCrEiXf2m0YCC4DXD6RaNBls+aqnZyVEs63p3TRBl", - "KKZRRCUJOAtlcQzK1MF+/WIKB4YIwT0U6rl+jGIiJZ4S1NZkU9NuhqTCKpWISjTBNCJhoz3yIYJZzB98", - "jGhImKITWj7fBp26eBzs7O55aUeMp2QU0qm9icrdH8NzjWK6H4WgtX8h+qAtmq0DhhRksjzeCyDdMIgg", - "EyKIxvGvHC4RfE6YPi16vH+BcVv/azu/orft/bwNwDzLm3/ptP5MSUpGCZfUzHCJctk3Go0A1Ai+8M8Z", - "Xq3a6wJGSYXF6vMBLW7hJJr5NYLNuWn6pdNSeLr2k3e6TZV2Amm0Q5aoQC2JfD4nzMMkBZwp+6IMnVd8", - "iiLKCLIt7F5omqgH+CXiQBJvCQ4Z+JcPv573DYiXeVDTm37XaRGWxhqYEZ8WoTkjWKgxKQGz5gqzHeWz", - "qwX/Wen4VO4qLMloNQU5o4yREOmW9mCbliiVwKkuLR9O0RVVozkR0nvmYFq/UYVsi9quIh5cTWhERjMs", - "Z2bGOAzhvOLorLQSD7dWYn9xoomg6xC4CIkUR+e/Hu0+OkB2AA8MJU9FYGawvJLC17p70xYpLMY4iry4", - "UY9um9/Ryxjix4Dz7GDU3T0ZBjrENJSuZXdTd99pJamcmV9Au/Ws4O7TZECjV6R/v/cs+hkQCSMl1MpM", - "fh7wTWI2G00jrmG6QCmjf6YlBruHTrSsoJC+KGhIwg7C8EKTbJwq3p0SRoSmU2gieAzcVoEJRm3Sm/Y6", - "aKj5wq7mgrt4t9vvd/vDVpmNjfa70yTVoMBKEaEn+P9+x91PR92/9btP3uc/R73u+7/+iw8BmnLmjiu0", - "62y7s99BbrJFdr060XWs/I2pf3H6PopjtvpE04lNd/rZyTLjYNYa8uCKiB7l2xEdCywW22xK2cdBhBWR", - "qrzy1W1vFRawjhVAYFMNpg3BUBF6AI3bEb8mItAUOCIa8WRHE2GqZAdhLTcD8UL6lvx3FGCmz4JhLrhA", - "hIXomqoZwtCuDK140cUJ7VIz1VanFeOPrwibqllrcLC3hOcaydv2R/f9v7lHW//hRXWRRsSD5G95qiib", - "InhtbvUZlSifA1UkXrsjDrppBGxeTNmJ+WwnmwkWAi++fofdQlbttBHmarc6iD2c/5s5EYKG7lZ9dnqM", - "2hG9IhbdkUgZGqb9/l4ADeAnsU8CHseYhebZVg+9ianSt1maX9JGG9QrbvfvLRLMOPAZUcT1gjJQ1zAx", - "OQwNHfJs57HTpEhkpXO4VzHoyWB7X55dbGvKlmAp1UzwdDorz8qS1c3mQ+XViPLROPHNicordLL9Bmmi", - "jyKqoZMR+Z1+//Tpthy29B+P3B9bPXRsQAbT1/vHhb175AwLAhxQiDhDz84uEI4iHlj5c6IZ1QmdpoKE", - "vYraA3r3HQ7ClFgknPoY4Apm5E2XEaTbzd9ugAfbY8q2pd6GbrAZ3AmbfwUb9pzNqeAs1qzwHAuqaVxJ", - "CfW59frN8fPR89eXrYE+RGEaWI3O2Zu371qD1l6/32/5OB2NQWvO+Muzi2ewU7r9jKskSqcjST95yPBR", - "tj4Uk5gLI37Yb1B7VqbShjtDsDnD1t7Lpwa5dl4CXrlNCamE1q4X03EZY3ZfPvVhy2yREDGn0qej+DV7", - "53a+QFMNYSrjtiRiTkSGtIDFvQLvF0Q8DbuFITutCRUkEFijXavT+pPEmgmaf9Kok8/d851fddDo8l9z", - "q+MooYysuNa/k+v1mouriOOwu3PLtysjSve9vMTX5kV5fy1OkAwlWp0lUZCF1zRUs1HIr5mesoeu2jco", - "a5wR1496JTj659//cXma86g7L8eJpbQ7u4++ktJWaKvu2it/ZgtJE/8yLhL/Ii5P//n3f7iVfNtFEKbx", - "MyzZdYz6p7yU/5oRNSOicOO6DdaPjAABnyOHL4XhS/qkohFoibjyORERXhSIpZ1Ta6cPFKsyK0EVnC/7", - "nSZ9V0h/vIZ06t7cxfyyKtTs9v3E0TMpz5ye6vNtaXmTmWQT2dk9tT93l6dUM6Mrmoymmhcc4Wmm41pl", - "nju/ogmCL7rwhdnGKDKHN0x1z2jMueoN2X/NCEOwd7DB5CMJgE5pIR4dnZ1IdE2jCCRiIATL18GQvSuQ", - "AtNcKv1fkbIOGqcKCRJzRZBlNGGQFOYCjccEpQw7+19vyIpQsQus4pUFyxURjESjGcEhEbIhZMxHyH5U", - "CxxY6gRLRYSh0GlShtfxb6fnqH28YDimAfrN9HrKwzQi6DxN9BneKkOvM2SJIHPCQGbRTAW14/IJ4qnq", - "8klXCULcFGPoLNMpWOPU/OXZhTVvyq3ekL0lGrCEhSSEObtbQiI1wwqFnP1Fn1gSlrstjl8Buv8sbyL7", - "dFrzIEnLO7Jb3Y3XYIDUa59ToVIcafJW4uC89khj6fZw6saQXpQYLNnKkBOrsiGpqYBoegaz9zIf65fz", - "DHNSL+edM5zIGVe1ct4VZeG6eblOftNta/mUTO8lbfO7ZlUSQbppMhUYDLW3yajcWPoGaNbvxhofDJ+x", - "LYNqkErF44LJDbUrikJaVimWgTXnUTfECgNT15DzNNNdNl/HC9OVOSJ199toOvZon/U1Rhma0ikeL1RZ", - "ktrp+w7i16pC3Fx821LnBmIONglHiq82hNMJcm2b2L3AaWSk+Gg+oZ6eM9Yo16JSiYKKz4klN7qLbhJQ", - "S6Q76HpGNTMlkQMC0OnL06IWozdkXbhYBug4GyDrNutSH0zQmEMXbS4Kk6Bg/EDjxRbC6PK0h95ls/2L", - "RAwrOifOL2aGJRoTwlAKTDgJYXy4NIsTSKW+qaiqfm5vJONCswXKGm7f9ZAWImNsb3d9FGKsaAAK9zGt", - "rAeMomaj9EiadLMib9GIF1jlPvCWTKlUouI8gNpvXzzb29t7UuUKdx91+zvdnUfvdvqDvv7/35r7Gdy+", - "l5Cvr6MybbEmjCL1eXZxcrxrWdDyOOrTPn5y+PEjVk8O6LV88ikei+kfe/he/Ij8pOw4t72gdiqJ6Doy", - "qbHKZ3EpGDZqLCo3NpTckd0jN+Ouamsg8U63vAsHKZ/p3Rp+N3dhqhLMtcb7wuKW1qOfai4wPyUFBZK1", - "kQXUaw08pvLqqSD4KuTXzHNvayZMjsx95tfsplqyHi8Q+agZdhIiwbmaSKNBKjOjO/uP9w/3DvYP+32P", - "X9AywvOAjgJ9AzWawJtnJyjCCyIQfIPaIPqHaBzxcRnRH+0dHD7uP9nZbToPIzg3g0PGK7uvUNtC5K/O", - "x9S9KU1qd/fxwd7eXv/gYHe/0awsG99oUo7lL7Ekj/ce7+8c7u43goJPEfHc+WlVfUlCnxI3SSJq1C5d", - "mZCATmiAwNML6Q9QO4YrjGQ6gPKZHONwJCx76b07FKaRXKk7NoPZlsatL04jRZOImHewIY3kGVj5MfTk", - "08tTxogYZW5sG/RkvdvW6krdWrImqOSlWALdKZXAheTMEyVRODAndC2dg93MJ/a+Dg/sGhpiwystOnUj", - "MidREQnM1aUnG3NBUIYnZtNKq6JsjiMajihLUi9K1ILyRSqAFzWdIjzmqTLKG9iw4iBgOwfZY6LJdTM3", - "jxdcXK21QuqbeCRSxnQ3a/UuR1HEr/UWX2nYwC2Okf3aOboUmL5MyWJUUfa9RG/NF0ZVlT9OUoUoU1xL", - "pywcLzowEgmhHUOCSMWBkuLgSnOYtpum3KWfb3mtGRanCDfj5bTznqwA3YlRwt6uhC2mRI2kwmotx6Ix", - "5R20P4fmjZ0a9IdrFSAN4M7I9X0AHbw+uhptu5Lh5G4gvsosl+ka8kZwCwsakh6C0wX2AedlWjlp54on", - "CQkz/U9vyM7NUckeSRSnEnSeVwYOakaoQFzQKS0PbI/NPdj3NkFFh003Rsfih8scKrwEpXj9occTRYSB", - "oHOgL3rB2U1odVoW9q1Oy1KiMmjcQw9EcqPz0hRfnl1saqVLBJ/QyLNc0DDbt1Yyc/arV/v98+7O/zG2", - "aI1vwKJRZrTSMQ9JrxKjAu2b3Twvzy7O6uaUBQih4uyW1pTZETyUI1M3O4hYDXmAGRoTZCUYh/76YskG", - "yXnvJz5ediJwTMbpZELEKPYo117o98g0MAYjytDp0zI/q/nmplLzWWlzQGye4MDGdzSDvkchV1lGpwDN", - "9/7tekvMNVznFaq3Stg21jG0h15nIVno5dmFRLntx6OpK29vrefQ2WwhaYAj06Nx8qasqGAD5GzMIZ/l", - "H1pVpIdPjr28oTsIqD2fJikcw/O33ZM3l9txSOad0pzAXjPjEdHz3ipQi7nzDc3dnEpEYl6n6TCIIZse", - "oAKsshPcGEiF8+qBjuIKRyMZceWZzTv9EsFL1L58YXz39Aw6KCltpX5egEIJvw+8J0ZTpLphz2HAqsq0", - "dMC9smM5ktGoVwrLKw3qOyq/EhyZAM4yPudhBm7j+VV5o/nV2tNrO/GNe+JcZBr4ED47PTYMQ8CZwpQR", - "gWKisA0XLTh8ATvU6rS6+o4KMYnBgDn599XOXzUq+AxdVilxny1Ff92JArcmakETuWhOQhRjRidEKhu1", - "UBpZzvDuo4OBia0KyWT/0UGv19vUW+957p7XaCu2jTNTwXGvJ2dftw934JTXZC2fW2dH735tDVrbqRTb", - "EQ9wtC3HlA0Kf2d/5i/gh/lzTJnXma9ROB6dLIXhlU2a+s4yzwd6JYwEGUJyEODXmphq5BmNmhH9RELk", - "9VpXeKrlE4NxX+uefuMAtjyKWhUC14p2+gZBbPTTak2oY4ygjR0zZYpGeXzfsg70RhGacmUQy1IAS0JY", - "FrYSReZXwNlcnwpfDEuJgLt3X2U/uDYC3SikHkz+LyvthVrwUuBfuv68tbZxkqxHWz+jmNG/prF71sPe", - "cxN9c6p/ExtbefQ30//88//Ks8d/7Pz56vLyv+cv//P4Nf3vy+jszVf5kq4OrvimERK3FhQBhqVSZERT", - "VDrFKvAwVDMuVQ2E7RukOIr1xz30DAS/wZB10SuqiMDRAA1bOKE9C/hewONhC7XJRxwo8xXiDOmurP/Y", - "lv74zKh/9MefnWz5pdpHaB3FhN2QzKdTpuOQx5iyrSEbMtsXcguRYNPXv0IU4ESlgujd0zxstEBjgYPc", - "QSwfvIM+4yT5sjVkIOGSj0roFSRYqCwazI0ASGFnZXwGbHMSojmOUiKthDxk2b0EIr/uxOhoepkSBHTz", - "Fc1qDVC84gsXZQfHw37Hs49It9MbGVGpCEOZtoNKQHTUdp6qh/0SqTjsH/bXMvgZDq1APzgJy7laHFI2", - "OEsGgWFoQ7hHM6WSBrp0TZvMGUG/vnt3psGg/z1HrqMcFtkWGyEPJ0lEiTQ6QhUBr2Odg7dafoc/vbsN", - "F2SUZPBZ1MAX8zkMjN69OkeKiJgyQ+vbgQbnhAZ6fWD+p1KmGhUpRkfPTp9v9RokmwHYZvNfsY/vshVW", - "LMdOaVanC8wwXsO3g06OO5pNsyc0Z+DAreYFFygyBCY/1wN0IUnZPRG2ylj1zU5Gi1zzZm6AYWvL9ZhU", - "KcUAvc34RpxNJYtSzZHBdZmfS+jWGl6Mz89S753yXMGbycpFlrSBhw9WyNo54dquJwWrj78H4nDmOavq", - "NDc720VlqB7Mjxr53t85t7K3qYy6aaha2TO9EImQRas1DzO7i3CtZXntI1WjWiM80q+tyd1JJZenaIYl", - "+4uClxXZZGfvcaOkLXrUpubrouGaT8yUslPl3Nwzs6tx+L+iUWS8GSSdMhyhJ6h9fvLyt5NXr7ZQF715", - "c1rdilVf+PanQdSaQ+2XZxcQCoblyFmA6p0ece44TD5SqeRyFEAjQ+rqKLlfS5Fs3rCKrVsMb3PW56Vl", - "3Efg2rd06/v+guZWhrl9bayaZXbvKFStlrj6wrzKdNY8vt2gszuZTil8zEcfijyB87m+ccRYp0U9/qZH", - "UpNAEqKTszzTRq6Uct1X1vRkt7dzcNjb6fd7O/0mKroYByvGPj161nzw/q5RRAzweBCEAzL5ChWhRWzD", - "vOHoGi8kGjr2etgy/HyBkS8cW8uCNzK/Lgfm3SwOr8pQrIu02ySyrlnI3Ip0WeflRFmNebRHf/uqnFqk", - "6c1sXRfsV6NNlNcEBTyNQs0HjfXJM2IVCa30J4nKc5DBYb1gV4xfs/LSjQ5Tn98/UyIW6PL0tKTxFmRi", - "Uyw1WDi4PNTsA0822obdNazy2tncMHrtPiLWqlSzcFvdenxaUeXmXCgNhjZQveXco9fsTZnZGo0nK9ZU", - "UZqEZD5KUx9TpF+5wImLi5PjEnJgfLBz2D980j0c7xx098P+Thfv7B10dx/h/mQveLxXk+SwudvLzT1Z", - "yqe5PlAJAA8KSBOHFg70ectcUcapQpmbmj7IzzR3iQpsrAnLAZ3ACaOK4oh+omyquwER3XK51zO9JogJ", - "RpRRpd+ZIF3K9JJBF6I7sc5HA/QS2sIrHEO4kJuElm3KagAcLowaVBMGN3QCf62e8vksVZrtgm/kLFVI", - "/wXL1mCw0sbqLgyNGaDXHL4RzkeU8arYYpqD79Vy86qI07ZeQc57FAazBHOAXmREMiOzlqy2JbE/De22", - "js3gtL1Vcp2zO97S2JLvXMErrNMyEG11Wg5Q4D227Edm5+UNkSiios8+QHAEJDT300kVjegnc4r1SqhU", - "NDBCH4bNrTvJNjMBCUfmBq+z9hnnD3vLZx85QnF5itoQjfhXZGVC/ddWZhksnsr93Sf7Tw4e7z45aBRz", - "kE9wPYF/Bq5Jy5NbS+2DJB25/LE1S392dgF3n75XZRobId+uveDimQgeaGaTMpQnpM0Hf9J7Ugy1CHk6", - "jgpKIxuXBf78TbIH15i3/qTRnE4m7M9PwdXuH4LGOx8P5O7YK5tlA/kZ2ZOionNJ6iPjrskm4/eGB4QS", - "sjZg5C2RsAJ0ThQC/OkiHMAlnXkUWZRzYSUW4l7E2t/b2zt8/Gi3EV7Z2RUOzgjEz+VZntoZFI4YtETt", - "t+fnaLuAcKZP52aZCCL14ky8pPecIZvVq1/ywNSiz54PS2r4pRxrbN/zuBbkl5YJsouyQAfHqIxBWjrl", - "Xmjv7fUf7z86fNTsGFuBayQ+rqYwtp11KBAkIHRe2vk2KMffHZ0h3buY4KAsYOzs7u0/Onh8uNGs1Eaz", - "UgIzGVOlNprY4eODR/t7uzvNIp98CnAb01c6sGXa5Tl0HqTw7IYHFMukt1N3W/gYz2VvzJUOoLlHadV9", - "cBN/4Tzmm0rolRZcVVFb82VFHrcQt7zVRM3hJ5F6nLqs9JoDberKu9pz9wyr2Qmb8GULxybypvWHcprv", - "RPNBEnLwhoRREjralQmelrUCD6tIEhSmxELOsEoCW4BjY+VJsJoB/wsfUjYt+5YvDdhECjRzWB3hD+Pa", - "hk0UVtLvl/NOpAAro2KWCOceOo305VSO/ILKcseCTNMIC1R1V18xZbmII8qumvQuF/GYRzRA+oOqNmHC", - "o4hfj/Qr+QusZavR6vQHo9zAXNEOmMlZ9wKzIZVx8yX8ole5VXFugpt/23y/DWVHmuj/vFanF1p2Mh7d", - "F4x+LCB6OQR2f7df5/dW02nJ4205GmBT2m5R1nfinaP+UZY8zGPdNPajilBc5oNL6/WtFgyUq7z8ljkB", - "1HYqRRdiXIZrIdS30UXczEZaVZ672WxLEpRH3z989PigYaz1V7HaKwozfAVjPY9XMNQ1O3XahGs7fHT4", - "5Mne/qMnuxvxR87OUrM/dbaW4v5UcgRWeLZHffjfRpMylhb/lGqsLeUJlfL93XhCX1Yc3TzGpkbqXlUU", - "Kd9JJ+aXGfBmLO4KbumoxHIVUtq2yWRCAkXnZGTg1s0nU/HNajSHACc4oGrhkQDxNbiroKxJJVakQe+V", - "yXpAavu24X6acsl0nLsDtN3g6N+MZFfBhcPGKRtkOq6TIt9URzUypPHvCisaigYKAoMRPpv8dQZMdI1l", - "yaigfweKhJ1CyuKq9cm0aF7YwuF6Vtsit6v74p38dSyK21/ZzoLUUWKSqxBfdYXWH0HNEYDzWBOdvedG", - "9njWBut9Oir0wV6AN/tqNC4mU1mZraaUeSW/dTcft1my5eXvzA22+XgFB4JNPqzmlQB8tHOwIM/77pRQ", - "ogabFBfr0wDeQXS4UWnfKD7casPvJUTcPr6TsPCl7TgveEE19/lzX/nLk5XsmAfd/l63f/BuZ2/w6GCw", - "s3MXAQqZDaNOlfv4087142gXT/ajw8XjP3dmj6e78Z7X6+MO0k9WcuRWslHaNSREVDOCVDPpSBJRRroy", - "M3+sN0SvCD0ySrkEL4DJWyGRbSIGuEI/K07teXmRxcOLVQ6car7S+/BPs7NfKctUp39yvHraN7InVCfi", - "R7DqVACfmk0GAuZ2bjXTKOhV4fR4AVmzKB/KlOzxJSR+v4KC/WYPbh2lsm7hdoZ5dgp3mJx9sYQ1+esl", - "QPlI7OoEGpVLyNhIi/lKMpfU282e8c5uaZ0DcyGpy+6jg3JWl6Pu30wWFzTqDbZ/+ev/7r7/t3/xZ/Iq", - "iY6SiG5IJsAxX5FFFwJqkEauXjn6FZyrW1Jhm/9KERwDtQuuiKGuMf5YnO+jfqbLXrzG8dISQNSIKcv+", - "Xrsgf5GpJUQzziZ1qVljV+e4kjCLmsqRNvQeFRqjNokTtXDhpU6nvrWZ88tR1qGXjb5lx/3+k9sIM7xY", - "GVf4AyYGLvomuQmt9Upa2v/aYB6/Uu646iNsNN822WHZp7WSwk2qFcVUVxXuNhW0Qa1tA+mmaTWjwAbF", - "uusMGfkpc446rlr3Ov38SqthYWWFmdTvjXFM+8rK5lS6kuY3BJlVMq+PTDOOOZoT7VazYZqELYKC1toC", - "yABWgyAzRCxbO1b71p7ij9kIwIFiucQtwzoKladePoUkTG9dVkQ6cV3ANKq1QZ5+Xcl3h1XLm7GqBrxz", - "ffQePEurVlC/urNVQc58jM7qMvOazJEgFVQtzjUZshEABAsijlKDhkCfYBHwOB8cojO/fAFl/MSjk3up", - "pTEaoKOzE8CSGDO4y9HlKYrohASLICI2uG7JoQ3k6TfPTromKjgr4QBVWRUAxGXVPjo7gSS9th5qq9/b", - "7UEVK54QhhPaGrT2ejuQsliDAZa4Dckc4Kc1t+lzCLfeSWhv56emif5K4JgoKKjxu8dspYgwySEkOCzg", - "aYGxSTAVlrNJIjCmGYmM6m/Bn9gR+IG5JToG4LipY65UC6taJMkbu63vNTrIhDNpNnS336/UBMZ58tbt", - "P6SxfeXjNuIyTIH2ZefaJZbPcToW5F86rf3+zkbzWZtv1TfsBcOpmnFBPxGY5qMNgXCjQU+YsXe4MmDE", - "NszPGaBQ8YT9/l7vl0zjGIuFA1cOq4TLOhaNSIQh4+PYlZrtISv6QTignPE0CqGcTGJy2msyipHCojf9", - "hLAIZnROhszeHiZ3LhYQLh0jfWsYvVX5aJihze4bskOkesrDRQW6WXfbujvgtsoA3rhYcqY3SWqqJvso", - "usk3LQPuTbRNGGYqT19sEk1fEXAvm9CP3g4b+UlqggfbQqCuQRZ+v7vlt9BCNJnfueE4e+dqdpcvOS0j", - "UBZEaZhzAuVayd6sS6bmr83HfUU8jNNLaGGBUgy8c1cu4yExQVTJQs04M7/TccpUan6PBb+WROiL2QZT", - "W1jbZLQWdaEwAo0hoNmkatFjbpspbn++IosvvSE7CmOXhsdWYcKR5DZRufFHpRJltcAAd2vK6/uFhWe2", - "cIlJElzMq2qmyVOVpKqHzEKIshHg0BzS7soZCYdMcfRZmCoLiy/bn/MRvwBHTXCo8aTQxCxp+zMNv9TN", - "Wo6wXv0ImnpkEgIAGLb07TJs6d9TgTVHncoZwgF4zeqHxS1tm4PNBXArW1UIB5ihhCdppHk/QCqTf73U", - "B2TTwFGEFBwl963mgWAna9Zjje2+1JDW0m5Mo5VjBEkiC4epv3/oP0+SBIL4xO7/PH/zGsFVBSXFoVke", - "4QkwokzfolnlKz16b8ie42CGDN8Ezv/DFg2Hrbx09BbMNZXWFNDtAuP1C9TUN8N0aPhLr6e7MjzdAP3+", - "2fQy0GcpiUeKXxE2bH3poMKLKVWzdJy9e+8HaJ3B8rxECFDb0P4tlwtJr7BwDZp7A7MQcUtrowXCKKdA", - "Rel+TBkWKxM5eUBvIagFTDyVRWB8HoKKZ9gaDJ2SZ9jqDFuEzeGZ1QQNW1/8ELCJx+o9zU0uK9ssR6KD", - "fn9rvSeRha+HhS411MfvyxL3tXtrjIdlupYZD1vf3obJ6B20ReaB3boHzucpDl2ei58s3hoWz8rTBeYN", - "vi/eAwZ9I2J0xxUOTAvgkePAVkonBi0gTgwkDuf3ZwQO6ji4HHmL4kdVyFwWK/brTlkAU4wc/u3fA/7B", - "uHlmfxj3yX2NiyNTg8rluX5Y6Aib5RCx45eIXxL1PWBc/75IqStA8g3x96Hgz0ti+b4caBVqtg01Povq", - "lmrssyA4lrYX01jLqucwp+45YQo9h6c9+6+TeCBU9EPEpx8GyIAw4lMUUUak9cnIbBj6UrSwhI9Mqsbs", - "O5vtNJhhNiUStc39+c+//wMmRdn0n3//h+amzS847tvG3x8iIT/MCBZqTLD6MEC/EZJ0cUTnxC0GYpXI", - "nIgF2uvbqsbwypM7VQ7ZkL0lKhVMZp7+el0AE9OhLeWh10NZSiSSAEIoTjexLuhG7ekR4d1ZNqC81xPd", - "WZK57AoKC9C3osMB8CmkJhzUyl8tv/bMrLmkP6tqcJd0+uvpiyIflcHerpnghgQGQOw7d/DCLhq1z8+f", - "b/UQyBgGKyDMADjmvBvLPPd+0qT1NMlQlDJBASgb2lRIm1+r/z22bZopgG2PP5IGuK4OQL0K2Kg8iCCh", - "g9dPWaGJOtgPN6ca9ulnj13ZwHoF7c3XWxzC+Wk2EoRvb58d7i3D3NbPzEH2LURg1LblzLIUlqUind8K", - "6e/l1ijUds2uDsRN4sx7E8uecTaJaKBQ180FsmTEJBPVygjyUMjBWztrhN26qgG9xfttuxSfUnvTZaEq", - "+ZV397dHZdBNrpE86DjHtZ83yTrUOaYy4PrbArZ0A5zYBJ6GfcnOaRGL1imkjuF5duWsZJeOs6rP9kDe", - "n2rKDp2y6t1wD0TxuEIQvyEhrCQlLITpPyRsvsh20ZVIXqG5+r5Qs39/XNB9a7F8aP6Q1FhhBWyaCs6y", - "UlV16GWLWd3hRtsRPAs/J8KdajNRk+AuX5b5FAUzElyZBdlK3qs4ghNX7LuJ6Gv6+5EkX1NFbAOOxYL8", - "J4vSQNjNYbVKwD2xmRrvTr6FETYSb2/PzmsRzANkcDYZO421SYKI5YIFWz+UqfdebrNqtfAHdJLO0ihy", - "Fo85ESqvpVa8A7Y/g1vSet7enbaV18HF21ddwgIOfmiZD5WfiXIljm6XwzcbZpbyE02ayIQAKocY9Qz0", - "V+y/cRdEWb78f919YTPm/+vuC5Mz/1/3jkzW/K07Q5b+fZHm++a4HzDyaYabloEGpMkUIlrHoWatGjKp", - "rv0PxafaonabcKoZXH8yq02Y1SK4VvKrWX3BO+RYbSm2b2OSyZDNB2145fwTfzBO9X61fBYjC1X7S2YP", - "m3KSi7z8ma35/fAcKGmGccVro6G6Oj+QK68Ph7onxx1b2c7Uo8sCRO5Jee3mce/MrR33/jXXR/GYTlOe", - "ymLsCRQyJNIGK0WkTIAfGtudX8+1jPd3jKX9+7w67p2v/on3d8TxVzfUEG9jgVrH87tWTXl+2x5KBppq", - "FCZ27a2rcmHTqGzVOBW6OjBN0bhUsmjZ2dE3L58sgi60oJKLCwgkiMGQ/YeWP35XBMfvf3FBMmm/v3sA", - "zwmbv//FxcmwU4cqhClBiURYEHT0+hjMflOIXodkaHlIXnUeJsWZqQ1ty5b+jxOQcstncwnJYeFPCamR", - "hFQA12oJKauicpcikhnkm8lIDt98ALepNX5KSfchJcl0MqEBJUzlGYCXnMRsAvEHGFvGrH2o4NxRumgb", - "S0l5aaPVDGie9u7eHXuywe9fOHIZ9h6mjzw3UTGhE0fyy7BeHvne8KF/v8T5/uWQh4xihuGvgm6ZEG1P", - "bAJiP4PwgourppjnycN56wh4+9xJcYXfIW+ip0cKVQ6/IYsCl7fxrddIU+Zc7uFALiVX/ZYunQ4SVrg1", - "QZGUTfM6l1TNeGqyqozsQ5OVTZ8KW00GWJ7A9vqtyYse/R4Y0NdcIRonEYkJZG3rGmyC4qJpknCR1R+j", - "spCKeDPyp49N0cHWJLexVYA7yCZsBmWd27A26O2Xt8tLNSM+XR9Umw3uIkg9UbVDdiFNkpcPhhX+gDIi", - "ixRHkkQkUOh6RoMZRNjqZ9C/CcDFSfIhS6mx5YqlFjOLwOBtSQTFEVR55JGpV/phHscfBssZ4C5PT+Ej", - "E1xrcr19GCCX9S27IKRuVYyY1auIsFTotY0DbmtMEjyKzI5+0LdQYX1bNpY2T3kyZL64WkaubYd0gj4U", - "Qmw/1MTYOoL6ik/lt+KXOvWJqsxaFEcCAGdwk7CwVafYoZE/unan3/flT2kY6WumcceBvkuTecWnWZKs", - "EirjJGmKvnaagMXzOF6Bw6hdSGYuVchT9VepQiIEfGyxuw65URsH5g+FrzSiMluKzKWDB/Tzqi9N1hov", - "qDRRLeSTNn/N47jVadn5eKrnfn3EdLXDZTWb3plCWPRPTnuTgOcysS9EPFduDlu3op7ltuU4fnh5z5W7", - "/sZo+A30Y/ksKHOsCuxtXkf8YUVOmkotVV7MJM/3nZGs1Ev9KSkrlc/zNP3/A0VUs9ZqfZ57FlIzEPsk", - "s1J5i28unWbVNn5KqJmEygUKUzNcpd7NDyt2ZgQFpawkeVr29KayZ5ZkLgMz1CFkKw0COc3b/ux+ntyA", - "XfhOKGGntupLXTqjfNHfA8mtqYnWiOZ+Iz7JXqsFBuEbkmBXne2+KXAGFS3uZVTuuyDD5sBl1LhIc6Dy", - "PnWFF38S45Ia0GhKb0qMHfO5pAsskGfKukmE6+iy5VNrCbCtAvXDy2u5rPKDS2wBF8K4joEz2kMKXSzY", - "DAuiZzvBqSSd7MB0nN368vR0q+7QCLXyyIjvw6B9M86hUpYzDv11kQUNXZL6Z6fHNqU9lUikrIfexBQy", - "x18RkkBKSspTicAHsFesN1ZXBS0rKEaYEouEU6bWziJvejeT+XKjJN33TKds8PYPr1ayhXYfGpEC2qFv", - "b7uA1UKVMmX2vGY6Z7aizGTW18wHHvNU975UDw1NaETkQioSG5vdJI3gEEF6D5v91X5nfNc6iCoJ1cM7", - "4OuTEBFTKSlncsjGZKK5koQIPTYUnKQRKZgffJatc4UzqnlmSN/3YdqCEmlgzcGqDmrl6mg4SVx1NJ/5", - "JCvoduMpvQBbFZKLeMwjGqCIsiuJ2hG9Mjw4mksU6R9bK41dI/jutnPb3vxkaUifsAn3pv8zOJsh849A", - "4U4qZM0Z8x8cWXtJiofF0R/YaD9Zk2vpmiA4giKgmZstShWN6CdD6nQnVCoamJpJOIMdlHsx4/WG7JQo", - "odtgQVDAo4gEyukathPBg+1h2u/vBQmFeIg9ApMDglf/OoYRn51dQDtTkqYzZPoP6Pjd0RmiGqYTbEXm", - "wkRtYXt0sv1mjfn/HMD0P1geMwtcdSz8G/7Tsru5D2XtGZI1R5QnqwQgnvzwCgPLwf3UFjxMbQE4sWer", - "aU8FDoAplrNUhfya+TUDpkKq3P5sfpysC4VQOJhdulLR3we3a6vFrhvGLfBBHEq7ppCY9KTfRF9vC/o+", - "0HROGnBuCcDEFIM6/LeAKRT+o2H37RvrinD8Di11FqIu9e93c7bu++azc3ARfkV4PJRjbjDNrQRKVha1", - "T1k441rZLEiFIExBKpictQxwggOqFh2EI1dN1ZZHynRIeSH4sSD4St+0vSF7mwVS2vJMWrrqONEKhVRe", - "mR6s9NRDb+ZEyHScTQ4BYTJyHgDfFlQNcBSYSqRkMiGBonNiSoTKGukrm8pdpuXNB/FstHtpQffQRA4/", - "TsDu5WhhpY6Sp1xt+obzrFWz9A1ZrwVvmIKnyEqf55FraKrgb6Ky8wx+RWvd4u2rzbzXftMfNRy77CXl", - "n4R99ZWr/FGy4p0XnFOaJn3IMfyh5V8ozLx0VEsOXusDwRt7dN2lh9W6QPBs8PsOBD/3Ovk8sHRUuOS2", - "VRcB/v0hQv9+vYvvOwL8YeOWZiXkEujqKVGDSPDvAgPvJgT8G3vX3yAE/Lvy94QQ3m/nd/9deXpaj8XM", - "0/NnkPddOniaSG8IaK1z8DRUz2qeVwpKl7ZNMzHJ9vgjcfBWWbkB/+7A/jNlWwORoQAsdwtXyA3QfmkR", - "nsSJWjhtFJ+A302eU1DST+C95wucy5TOdxevdgN97O2hh8PTWm3sz1Rv96bwzfNhnxw//PxuxTNXuli2", - "9a3TxSKY0XkpXmvVCbYgSgTpJjwBPWtoAGbh4e4yhUVv+gnZ7ntD9m5G3F+IumwZJEQhFSRQ0QJRpjhQ", - "BDPGXyQSXEsC8J6LhU99Wzy5LwSPj+xq1tyH9kxZZVju5hcvuiFWuDt31GaFCu0rTFan+CON0xgIHqIM", - "vXyK2uSjEiZ5A5poyQfRSQZS8jEgJJSAk1vFCe/0azSb9BMZTcdNZrkiDccbm+YEBalUPHZ7f3KM2jhV", - "vDslTO+FZvUnwMkmgs9paHLk5kCd88hAdacGoJvqXTVTYf3Bc+HCTO6b8DBNLqTpJ5qUyYJxe2wNWmPK", - "MExubcKL8pkyHrh6PEzBDy4/Ow5zWj+vsGqVbY2JWshxQFSco0hz9Fs/r7mHfM0VPRncnVa67ZplMW3m", - "3NDQ5+AuMphmji/3q7a+/H7s8YWqxA9QdT7PBNI6tfn3hYL9+7sf7ltdfvmA/bdeEid8F1Tl0IHu0Ycw", - "r3iAIxSSOYl4Emu20rRtdVqpiFqD1kypZLC9Hel2My7V4LB/2G99ef/l/wcAAP//du3TWocdAQA=", + "H4sIAAAAAAAC/+x9f1MbO7LoV1H53Vtrdm1jfoQQbp26j0CSwz0h8ELgvrvHeUaekW0tM9IcSWNwUvl3", + "P8B+xP0kr9SS5pc19kCAhE22tnLMjEZqtVqt7lb/+NwKeJxwRpiSrb3PLRlMSYzh575SOJhe8CiNyXvy", + "R0qk0o8TwRMiFCXQKOYpU8MEq6n+KyQyEDRRlLPWXusUqym6nhJB0Ax6QXLK0yhEI4LgOxK2Oi1yg+Mk", + "Iq291nrM1HqIFW51Wmqe6EdSCcomrS+dliA45Cyam2HGOI1Ua2+MI0k6lWGPddcIS6Q/6cI3WX8jziOC", + "WesL9PhHSgUJW3u/F6fxMWvMR38jgdKD788wjfAoIodkRgOyiIYgFYIwNQwFnRGxiIoD8z6aoxFPWYhM", + "O9RmaRQhOkaMM7JWQgab0ZBqTOgmeujWnhIp8WAmBJiGNPSswMERMq/R0SFqT8lNeZDN56PdVn2XDMdk", + "sdNf0xizrkauBsv1D22Lfb/d9vVMeRynw4ngabLY89HJ8fE5gpeIpfGIiGKPu5tZf5QpMiFCd5gEdIjD", + "UBAp/fN3L4uw9fv9/h7e3Ov3e30flDPCQi5qUWpe+1G60Q/Jki4bodT2v4DSdxdHh0f76ICLhAsM3y6M", + "VCHsInqK8yqSTXlVfPT/MqVRuEj1I/2YiCFlUmFWQ4NH9qVGFx8jNSXIfocujlF7zAUKySidTCibrDWh", + "d82wIqJIOMRqcTgAFdk2lDOkaEykwnHS6rTGXMT6o1aIFenqN40GFASvGE63aDTY4lZLzUoOY1nXu2uC", + "KEMxjSIqScBZKItjUKZ2tusnU9gwRAju4VCv9GMUEynxhKC2ZpuadzMkFVapRFSiMaYRCRutkY8QzGT+", + "xkeIhoQpOqbl/W3IqYtHwcbmlpd3xHhChiGd2JOo3P0hPNckpvtRCFr7J6I32rzZPGBIQcaL470G1g2D", + "CDImgmga/8rhEsFnhOndosf7Nxi39b/W8yN63Z7P64DM07z5l07rj5SkZJhwSQ2EC5zLvtFkBKhG8IUf", + "Zni1bK0LFCUVFsv3B7S4h51o4GuEmzPT9EunpfBk5ScfdJsq7wTWaIcscYFaFvlqRphHSAo4U/ZFGTtv", + "+QRFlBFkW9i10DxRD/BLxIEl3hMeMvQvbn4N9x2Yl3lQ05t+12kRlsYamRGfFLE5JVioESkhs+YIsx3l", + "0NWi/7S0fSpnFZZkuJyDnFLGSIh0S7uxTUuUSpBUF6YPu+iKquGMCOndcwDWb1Qh26K2q4gHV2MakeEU", + "y6mBGIch7FccnZZm4pHWSuIvTjQTdB2CFCGR4ujs1/3NZzvIDuDBoeSpCAwEizMpfK27N22RwmKEo8hL", + "G/XkdvszepFC/BRwlm2MurMno0BHmIbTtexq6u47rSSVU/MLeLeGCs4+zQY0eUX690fPpA+ASRgtoVZn", + "8suAJ4lZbDSJuMbpHKWM/pGWBOweOtK6gkL6oKAhCTsIwwvNsnGqeHdCGBGaT6Gx4DFIWwUhGLVJb9Lr", + "oIGWC7taCu7izW6/3+0PWmUxNtruTpJUowIrRYQG8P/9jruf9rt/7XdffMx/Dnvdj3/5Nx8BNJXMnVRo", + "59l2e7+DHLBFcb0K6CpR/s7cvwi+j+OYpT7SfOK2K31wtCg4mLmGPLgiokf5ekRHAov5OptQdrMXYUWk", + "Ks98edt7xQXMYwkS2ESj6ZZoqCg9QMbtiF8TEWgOHBFNeLKjmTBVsoOw1puBeSF9Sv4HCjDTe8EIF1wg", + "wkJ0TdUUYWhXxlY87+KEdqkBtdVpxfjmLWETNW3t7Wwt0Lkm8rb90f34Z/do7T+9pC7SiHiI/D1PFWUT", + "BK/NqT6lEuUwUEXilSvisJtGIObFlB2ZzzYySLAQeP71K+wmsmyljTJXu9RB7JH8T2ZECBq6U/Xg+BC1", + "I3pFLLkjkTI0SPv9rQAawE9inwQ8jjELzbO1HjqJqdKnWZof0sYa1Csu9+8tEkw5yBlRxPWEMlTXCDE5", + "Dg0f8iznobOkSGS1czhXMdjJYHnfnJ6va86WYCnVVPB0Mi1DZdnq7eCh8mpI+XCU+GCi8godrZ8gzfRR", + "RDV2Mia/0e8fv1yXg5b+45n7Y62HDg3KAHy9flzYs0dOsSAgAYWIM3Rweo5wFPHA6p9jLaiO6SQVJOxV", + "zB7Qu29zECDfYSL4zXzJsTflUnWlppJfP3w4Xdf/nKHjow/HyHSAoAMU85DooctkR5jmIeFq4+B/T4ma", + "EqEnbr7x955NrKSIZNZDLT2MuQhITJga6o9KI7eMSFSRiovj6MmiQh9m4AG7xFF0idq2J6OUmS+otACH", + "a0gQvSslCqkggUKMs65p9OHg1M0nO/4vjjsDdj3VouDlVKlkqP+RQ81CL6s92W9BCeEMutO0IdFuH9jv", + "9vZWb8AKwpOZaKVbTd45ZdSIhzEProaEzYYzLIYhjzFlS6Xe2+xfL30lRHT1oCTsEjZDIZGKMkPXZnig", + "9OuISgV7WZJAEIVkOpKKqlQ37A3Yb2Qukd4juo8ZdmwAsH1ZnJO8/A80w1FKTHPom4TFYQcMqMAePBK1", + "L3FCexZxvYDHlx10+efSgzUQADHSA2UQUKlFwgFLBJGalCgzx0yMk04JfBASLRx6hjiKkNlmBaikXWC3", + "fp9bJ+cfXp6cvzscnpy+erd/NPzt1f/A0ie0xxPCMNWgtTqtPxf//OgT0Uv48eiNbEYFZ7AfZljQTCyQ", + "qG0QTNjscg2pKValqwS9qIgyYB6G5AcMS7MeXXOMvHp3MbzYfz98t3/8ypwll0DRglwLqhRhaISDK80W", + "1JRQgQTBkVs/zuzGqKDmdy9qbsHbfTgiTIl5wqnPXFA5R/Omi8dpt5u/vcWpuT6ibF3qQ6sb3O6UImz2", + "FUqrb+llmQzfnRy+Gr56d9Ha07w/TANr/z49ef+htdfa6vf7LR9C9Xm7QiJ6c3p+AOeabj/lKonSyVDS", + "Tx6hdT+bH4pJzIUx1thvUHtalmmNLotgcQatrTcvzVG88QZOYbcoIZXQ2vViOi6fr5tvXvq46HSeEDGj", + "0mfR/TV751a+IIEaMa4sCUgiZkRkRzywg16B2QcRT8NuYchOa6zPDYE12bU6rT9IrFXG2afyCeD5zm9o", + "baQqrdCBcJRQRpYoQd+JMnLNxVXEcdjduGddhBGl+16c4jvzory+liZIRhILotUIs/Cahmo6DPk10yB7", + "pFD7BmWNM1H0Rs8ER//8+z8ujnONfuPNKLFy6cbms6+USyuSqO7aa63LJpIm/mmcJ/5JXBz/8+//cDP5", + "tpPwCbpGRq2Vc61+4hbYnnHuAECOXlYKvT7mymdERHheYJZOBN7oA8eqQCWogv1lv9Os7wrpj1ewTt2b", + "U2PeVE1Am30/c/QA5YHppd7flpc3gSQDZGPz2P7cXASpBqIrmgwnWnMe4kl2I7BMXzm7ogmCL7rwhVnG", + "KLKCa6p7RiPOVW/A/ltrC7B2sMDkhgTAp6TCCu2fHkl0TaMI7IfACBaPgwH7UGAFprlU+l+Rsg4apQoJ", + "EnNFkFXLYRAjXELjEUEpw85boiIv2QkuKlOAlisiGImGU4JD4qTDlZgxHyH7US1yYKpjLBURhkOnSRlf", + "h78dn6H24ZzhmAboN9PrMQ/TiKCzNNF7eK2MvQ4I3DPCwMKjhQpqx+VjxFPV5eOuEoQ4EGPoLLPA2qv8", + "2ZvTc+sMItd6A/aeaMQSFloR3Z0S0gi9IWd/0juWhOVui+NXkF6nwDa3FHVasyBJyyuyWV2Nd+Cuoec+", + "o0KlONLsrSTBeb03jF+QRxcwbkdF+4plWxlxYlW+dm9qTjM9g5OQVyL3WMWMcFJvFTtjOJFTrmqtYleU", + "havgcp38ptvWyimZOitt84cWVRJBumkyERjcWu5TULmzrRKwWb8aKzzWfK4JGVaDVCoeFxwUULtyrULL", + "FzBlZM141A2xwiDUNZQ8DbiLzj7x3HRltkjd+TacjDx3dfoYowxN6ASP5qqsSW30fRvxaw3HDhbfstQ5", + "zZmNTcKh4svdhugYubZNvATAxW6o+HA2pp6eM9Eov3OiEgUVDz3LbnQX3SSglkl30PWUamFKIocE4NMX", + "x0Wbb2/AunCw7KHDbICs26xLY3/AodEh21wUgKBwVYxG8zWE0cVxD33IoP2TRAwrOiPOi3CKJRoRwlAK", + "QjgJYXw4NIsApFKfVFRVP7cnknE4XAPTNrfvekgrkTG2p7veCjFWNIDryRGtzAeslWah9EiadbOibNFI", + "FljmbPWeTKhUouJqhdrvXx9sbW29qEqFm8+6/Y3uxrMPG/29vv7/X5t7Zd2/T6Wvr/0yb7EXvkXuc3B+", + "dLhpRdDyOOrTNn6xe3OD1Ysdei1ffIpHYvK3LfwoXpd+VnaY31SjdiqJ6Do2qanKdz9duAauuX++87Xy", + "A90S504vy9oaTHzQLR/CndTnqGTdZG7v8FllmCtdnQqTW5iPfqqlwHyXFAxI1qMgoF7fiUMqr14Kgq9C", + "fs0857YWwuTQnGf+e7BUa9ajOSI3WmAnIRKcq7E0FqSyMLqx/Xx7d2tne7ff93hRLhI8D+gw0CdQIwBO", + "Do5QhOdEIPgGtUH1D9Eo4qMyoT/b2tl93n+xsdkUDqM4N8NDJiu7r1DbYuQvziPfvSkBtbn5fGdra6u/", + "s7O53QgqK8Y3AsqJ/CWR5PnW8+2N3c3tRljwGSJeOa/Wqudd6DPiJklEjdmlKxMS0DENEPjFIv0Basdw", + "hJHMBlDekyMcDoUVL71nh8I0kkttx2Yw29I4QcdppGgSEfMOFqSRPgMzP4SefHZ5yhgRw8zp9xY9WV/g", + "lbZSN5esCSr5dJdQd0wlSCG58ERJFO6ZHbqSz8Fq5oB9rKMDO4eG1PBWq07diMxIVCQCc3RpYGMuCMro", + "xCxaaVaUzXBEwyFlSeoliVpUvk4FyKKmU4RHPFXGeAMLVhwEPI1A9xhrdt3MKe41F1crfTb0STwUKWO6", + "m5V2l/0o4td6ia80buAUx8h+7dwCC0JfZmQxpij7XqL35gtjqsofJ6lClCmutVMWjuYdGImE0I4hQaTi", + "wEntbZ3tpql06Zdb3mmBxRnCzXg573ykW4Du2Bhh71fDFhOihlJhtVJi0ZTyAdqfQfPGLmD6w5UGkAZ4", + "Z+T6MZAOPnJdTbZdyXDyMBhfdi2XO7zk93PcXuz20L69n8998is77UzxJCFhZv/pDdiZ2SrZI4ni1Dgx", + "XBk8mCttLuiElgcuO3M85P3ebUjRUdOdybH44aKECi/BKF6/6fFYEWEw6MKNij7DdhFanZbFfavTspyo", + "jBr30IOR/NJ5AcQ3p+e3vaVLBB/TyDNdsDDbt1Yzc/dXb7f7Z92N/2PuojW9gYhGmbFKL7hCufbNTp43", + "p+endTBl4ZSoCN3CnLJ7BA/nyMzNDiPWQh5ghkYEWQ3GkT+VhUFy2fuFT5YdCxyTUToeEzGMPca11/o9", + "Mg3MhRFl6PhlWZ7VcnNTrfm0tDigNo9xYKPhmmHfY5CrTKNTwOZH/3K9J+YYrvOh10slbBvrRt9D77IA", + "VvTm9Fyi/O7HY6krL2+tn+XpdC5pgCPTowmJoaxoYAPibCwhn+YfWlOkR06OvbKh2wioPZskKWzDs/fd", + "o5OL9Tgks04JJrivmfKIaLjXCtxi5jzpc6fQEpOY1Vk6DGHIphuogKtsBzdGUmG/erCjuMLRUEZceaD5", + "oF8ieInaF6+Np7OGoIOS0lLq5wUslOh7x7tjNEeqG/YMBqyaTEsb3Ks7luO+jXmlML3SoL6t8ivBkQl3", + "L9NzHpTlFp5flReaX63cvbYT37hHzkWmgcf1wfGhERgCzhSmjAgUE4VtcH3B4QvEoVan1dVnVIhJDBeY", + "4/9Y7vxVY4LPyGWZEfdgIVb2QQy4NTFemslFMxKiGDM6JlLZGK/SyHKKN5/t7JlI1JCMt5/t9Ho9vztG", + "vbfeq9w9r9FSrBtnpoLjXk9Ov24dHsApr8lcPrdO9z/82tprradSrEc8wNG6HFG2V/g7+zN/AT/MnyPK", + "vM58jYKX6XghaLl8panPLPN8T8+EkSAjSA4K/Morphp9RpNmRD+REHljfBSeaP3EUNzXBvPcOdw3zzmh", + "CmG+xXv6BiG/9NNyS6gTjKCNHTNlikZ5NPSiDfRO8exyacjfQrhfQlgW5BdF5lfA2UzvCl/EX4mBu3df", + "dX9wbRS6YUg9lPzfVtszDvngX7p6v7XWcZKsJlu/oJjxv6aRzjYeyXMSfXOuf5c7tvLoJ5P/+uP/ytPn", + "f9v44+3Fxf/M3vzX4Tv6PxfR6clX+ZIuD0X7pvFk9xZCBhdLpTiypqR0jFXgEaimXKoaDNs3SHEU6497", + "6AAUv70B66K3VBGBoz00aFUiKgYt1CY3OFDmK8QZ0l1Z/7E1/fGpMf/ojz873fJLtY/QOooJuyCZT6dM", + "RyaYZG3ABsz2hdxEJNzp618hCnCiUkH06mkZNpqjkcBB7iCWD95Bn3GSfFkbMNBwyY0SegYJFiqLnXUj", + "AFFYqIzPgG1OQhfNYDTkAcvOpdBEjEAw94SoXmYEAdt8xbJagxSv+sJF2cFxt9/xrCNEGOmFjKhUhKHM", + "2kElEHoeDbXbL7GK3f5uf6WAn9HQEvKDnbCY2coRZYO9ZAgYhjaMG6KhGtjSNW8yewQicTQaTESO6yjH", + "RbbERsnDSRJRIo2NUEWyGCDW8jv86dVtOCFjJIPPoga+mK9MKN2Ht2dIERG7uKp2oNE5poGeH1z/UylT", + "TYoUo/2D41drvQapuQC3GfxL1vFDNsPKzbEzmtXZAjOK1/jtoKPDjhbT7A7NBThwq3nNBYoMg8n39R46", + "l6TsnghLZW71zUpG89zyZk6AQWvN9ZhUOcUeep/JjTgDJQvqy4nBdZnvS+jWXrwYn5+F3jtlWMGbyepF", + "lrWBhw9WyN5zwrFdzwqWb38PxmHPc1a1ad5ubxeNoXowP2nka//g0srWbXXU2wb2lj3TC5EIWWxv46Dc", + "BwnXWtTXbqga1l7CI/3aXrk7reTiGE2xZH9S8LKim2xsPW+U4kqP2vT6unhxzccGpGxXOTf37NrVOPxf", + "0Sgy3gySThiO0AvUPjt689vR27drqItOTo6rS7HsC9/6NIhac6T95vQcQsGwHLoboHqnR5w7DpMbKpVc", + "jAJodJG6PEru11IkmzesYu0ew9vc7fPCNB4jcO1buvV9f0FzS8PcvjZWzQq7DxSqVstcfWFeZT5rHt9v", + "0NmDgFMKH/Pxh6JM4Hyu7xwx1mlRj7/pvtQskITo6DTPS5QbpVz3lTm92Oxt7Oz2Nvr93ka/iYkuxsGS", + "sY/3D5oP3t80hog9PNoLwj0y/goToSVsI7zh6BrPJRo48XrQMvJ8QZAvbFsrgje6fl0MzLtbHF5VoFgV", + "aXebyLpmIXNLkgueldMKNpbRnv31qzIQkqYns3VdsF8Nb2O8JijgaRRqOWikd55Rq0hotT9JVJ6xETbr", + "Obti/JqVp25smHr//pESMUcXx8cli7cgY5uQrsHEweWhZh14cqtl2FwhKq+E5o7Ra48RsVblmoXT6t7j", + "04omN+dCaSi0gektlx69196UmaXRdLJkThWjSUhmwzT1CUX6lQucOD8/OiwRB8Y7G7v93Rfd3dHGTnc7", + "7G908cbWTnfzGe6Pt4LnWzUpYZu7vdzdk6W8m+sDlQDxYIA0cWjhnt5vmSvKKFUoc1PTG/lAS5eoIMaa", + "sBywCRwxqiiO6CfKJrobUNGtlGty+UBMMKKMKv3OBOlSSBYEthDdiXU+2kNvoC28wjGECzkgtG5TNgPg", + "cG7MoJoxuKET+Gs5yGfTVGmxC76R01Qh/RdMW6PBahvLuzA8Zg+94/CNcD6ijFfVFtMcfK8Wm1dVnLb1", + "CnLeozCYZZh76HXGJDM2a9lqWxL70/Bu69gMTttrJdc5u+ItTS35yhW8wjotg9FWp+UQBd5ji35kFi5v", + "iESRFH33AwRHwEJzP51U0Yh+MrtYz4RKRQOj9GFY3LqdbDMTkHBoTvC62z7j/GFP+ewjxygujlEbohH/", + "gqxOqP9ay24Gi7tye/PF9oud55svdhrFHOQArmbwB+CatAjcSm4fJOnQZduumfrB6TmcffpclWlslHw7", + "94KLZyJ4oIVNylCevjsf/EXvRTHUIuTpKCoYjWxcFvjzN8m1XnO99QeNZnQ8Zn98Cq42/yZovHGzIzdH", + "Xt0sG8gvyB4VDZ0LWh8ZdU02Gb83PBCUkLUBI++JhBmgM6IQ0E8X4QAO6cyjyJKcCyuxGPcS1vbW1tbu", + "82ebjejKQlfYOENQPxehPLYQFLYYtETt92dnaL1AcKZP52bpEnWZeEnvPkM2B2K/5IGpVZ8tH5XUyEs5", + "1di+Z3Etyi+sEGQnZZEOjlGZgLSwy73Y3trqP99+tvus2Ta2CtdQ3CznMLaddSgQJCB0Vlp5kyLsw/4p", + "0r2LMQ7KCsbG5tb2s53nu7eCSt0KKiUwkzFV6laA7T7feba9tbnRLPLJZwC3MX2lDVvmXZ5N5yEKz2p4", + "ULHIejt1p4VP8Fz0xlzqAJp7lFbdB2/jL5zHfFMJvdKCqypqa7msKOMW4pbXmpg5/CxSj1NXw0NLoE1d", + "eZd77p5iNT1iY754w3EbfdP6QznLd6LlIAkZy0PCKAkd78oUTytagYdVJAkKU2IxZ0QlgS3CsbnlSbCa", + "gvwLH1I2KfuWLwzYRAs0MCyP8IdxbcMmBivp98v5IFLAlTExS4RzD51G9nIqh35FZbFjQSZphAWquqsv", + "AVnO44iyqya9y3k84hENkP6gak0Y8yji10P9Sv4Cc1lrNDv9wTC/YK5YBwxw1r3ALEhl3HwKv+hZrlWc", + "m+DkXzffr0ORpib2P++t02utOxmP7nNGbwqEXg6B3d7s1/m91XRa8nhbjAa4LW+3JOvb8c5Rfz9LHua5", + "3TT3RxWluCwHl+brmy1cUC7z8luUBFDbmRRdiHEZr4VQ30YHcbM70qrx3EGzLklQHn1799nznYax1l8l", + "ai8pY/MVgvUsXiJQ16zUcROpbffZ7osXW9vPXmzeSj5y9yw161N311Jcn0qOwIrM9qwP/7sVUOamxQ9S", + "zW1LGaBSvr87A/RlydbNY2xqtO5lJeTylXRqflkAbybiLpGW9ksiVyEBeJuMxyRQdEaGBm/dHJiKb1Yj", + "GAKc4IAqT1rw9/jaJMnOmlRiRRr0XgHWg1Lbtw3305xLpqPcHaDtBkd/NppdhRZ2G6dskOmoTos8qY5q", + "dEjj3xVWLBQNDASGInx38tcZMtE1lqVLBf07UCTsFBK8V2+fTIvmZYAcrWeVgPJ7dV+8k7/qT3H5K8tZ", + "0DpKQnIV48uO0PotqCWCUt7xZTZ7z4ns8awNVvt0VPiDPQDv9tVwVEymsjRbTSnzSn7q3n7cZsmWF78z", + "J9jtxys4ENzmw2peCaBHC4NFed53p0QSNdSkuFidBvABosONSftO8eHWGv4oIeL28YOEhS8sx1nBC6q5", + "z5/7yl/MsXSPudPtb3X7Ox82tvae7extbDxEgEJ2h1Fnyn3+aeP6ebSJx9vR7vz5HxvT55PNeMvr9fEA", + "6ScrOXIr2SjtHBIiqhlBqpl0JIkoI12ZXX+svoheEnpkjHIJnoOQt0Qju40a4MqiLdm1Z+VJFjcvVjly", + "qvlKH8M/zUK/VJepgn90uBzsO90nVAHxE1gVFKCnZsBAwNzGvWYaBbsq7B4vImsm5SOZ0n18iYg/LuFg", + "v9mNW8eprFu4hTDPTuE2k7tfLFFN/noBUT4WuzyBRuUQMnekxXwlmUvq/WbP+GCXtM6BuZDUZfPZTjmr", + "y373ryaLCxr29tZ/+cv/7n7887/5M3mVVEdJRDckY5CYr8i8CwE1SBNXrxz9Cs7VLamwzX+lCI6B2wVX", + "xHDXGN8U4X3Wz2zZ83c4XpgCqBoxZdnfKyfkL8m3QGjG2aQuNWvsqsJXEmZRU2fXht6jQmPUJnGi5i68", + "1NnU127n/LKfdegVo+/Zcb//4j7CDM+XxhX+gImBi75JDqCVXkkL618bzOM3yh1WfYSN5dsmOyz7tFZS", + "uEm1pPR0zFOmhmB6XrSv6XfGrG0D6SZpNaPAeszUug3bXYz2JDiEullLLzLyXeYcdbrw0Wr7/NJbw8LM", + "CpDUr41xTFuMlluCoFONmuspEaSwEPBBHnt4S5RZI/PqyDTjmAPFv6rZME3CFkHBam0RZBCrUZBdRCze", + "diz3rT3GN9kIIIFiuSAtwzwKdfrevIQkTO9dVkQ6dl0AGNXaIC9XU1GTGniLi1GkqsV5m/bejWd51RLu", + "V7e3KsSZj1EizUV61GyOBKmgan6m2ZCNACBYELGfGjIE/gSTgMf54BCd+eULGOPHHpvcG62N0QDtnx4B", + "lcSYwVmOLo5RRMckmAcRscF1Cw5toE+fHBx1TVRwVsIBalgrQIjLqr1/egRJem316Fa/t9mDKlZQSS2h", + "rb3WVm8DUhZrNMAU1yGZA/y01216H8KpdxTa0/mlaaK/EjgmCgpq/O65tlJEmOQQEhwW8KQg2CSYCivZ", + "JBFcphmNjOpvwZ/YMfg9c0p0DMJxU8dcqebWtEiSE7usHzU5yIQzaRZ0s9+vVFDHefLW9b9Jc/eVj9tI", + "ygD0eJxrF0Q+J+lYlH/ptLb7G7eCZ2W+Vd+w5wynasoF/UQAzGe3RMKdBj1i5r7DlQEjtmG+z4CEijvs", + "9496vWQax1jMHbpyXCVc1oloRCIMGR9HrjB3D1nVD8IB8+p+5jKHhJqNYqSw6E0+ISyCKZ2RAbOnh8md", + "iwWES8dInxrGblXeGmZos/qG7RCpXvJwXsFu1t267g6krTKCb11aPq9CWVNj3sfRTb5pGXBvom3CMFN5", + "+mKTaPqKgHvZmN54O2zkJ6kZHiwLgboGWfj95pr/hhaiyfzODYfZO2TRWz7ktI5AWRClYS4JlCvLe7Mu", + "mQrpNh/3FfEITm+ghUVKMfDOHbmMh8QEUSVzNeXM/E5HKVOp+T0S/FoSoQ9mG0xtcW2T0VrShcIINIaA", + "ZpOqRY+5bkBc/3xF5l96A7Yfxi4Nj63ChCPJbaJy449KJcpqgQHt+sP9auwmB7ZwiUkSXMyrasDkqUpS", + "1UNmIkTZCHBoDml35ZSEA6Y4+ixMlYX5l/XP+YhfQKImONR0UmhiprT+mYZf6qCWQ6xnP4SmHp2EAAIG", + "LX26DFr690RgLVGncopwAF6z+mFxSdtmY3MB0spaFcMBZijhSRpp2Q+IyuRfL/UB2TRwFCEFW8l9q2Ug", + "WMma+djLdl9qSHvTbq5GK9sIkkQWNlN/e9e/n0zdWc8+/a+zk3cIjiq9BrY8bRbhCTiiTJ+iWeUrPXpv", + "wF7hYIqM3ATO/4MWDQetvND+GsCaSnsV0O2C4PWLBu0XM0yHhr/0erorI9Ptod8/m1729F5K4qHiV4QN", + "Wl86qPBiQtU0HWXvPvoRWndheVZiBKhteP+ay4WkZ1g4Bs25gVmIuOW10RxhlHOgonY/ogyLpYmcPKi3", + "GNQKJp7IIjI+D8DEM2jtDZyRZ9DqDFqEzeCZtQQNWl/8GLCJx+o9zU0uK9ssJ6Kdfn9ttSeRxa9HhC41", + "1Nvvy4L0tXlvgocVuhYFDzM5FyajV9BkJTPi1iNIPi9x6PJc/BTxVoh4Vp8uCG/wffEcMOQbEWM7rkhg", + "WgGPnAS2VDsxZAFxYqBxOL8/o3BQJ8HlxFtUP6pK5qJasV23ywIAMXL0t/0I9Afj5pn9YdwXjzUujkwN", + "Kpfn+mmRIyyWI8SOXyN+Q9T3QHH9x2KlrgDJN6Tfp0I/b4iV+3KkVbjZOtT4LJpbqrHPguBY2l5MY62r", + "ngFM3TPCFHoFT3v2v07jgVDRy4hPLveQQWHEJyiijEjrk5HdYehD0eISPjKpGrPvbLbTYIrZhEjUNufn", + "P//+DwCKssk///4PLU2bX7Dd142/P0RCXk4JFmpEsLrcQ78RknRxRGfETQZilciMiDna6tuqxvDKkztV", + "DtiAvScqFUxmnv56XoAT06Et5aHnQ1lKJJKAQihON7Yu6Mbs6VHh3V42qHzUHd1Z0LnsDAoT0KeiowHw", + "KaQmHNTqXy2/9czMuWQ/q1pwF2z6q/mLIjfKUG/XAHhLBgMo9u07eGEnjdpnZ6/Wegh0DEMVEGYAEnPe", + "jRWeez950mqeZDhKmaEAlg1vKqTNr7X/Hto2zQzAtscfyQJcVweg3gRsTB5EkNDh66eu0MQc7MebMw37", + "7LOHrmxgvYH27vMtDuH8NBspwve3zo72FnFu62fmKPsWKjBq23JmWQrLUpHOb0X0j3JqFGq7ZkcH4iZx", + "5qOpZQecjSMaKNR1sECWjJhkqlqZQJ4KO3hvoUbYzasa0Fs839ZL8Sm1J10WqpIfeQ9/elQGvc0xkgcd", + "57T28yRZRTqHVAZcf1uglm6AE5vA04gv2T4tUtEqg9QhPM+OnKXi0mFW9dluyMczTdmhU1Y9Gx6BKR5W", + "GOI3ZISVpISFMP2nRM3n2Sq6EslLLFffF2n2H08Kemwrlo/Mn5IZK6ygTXPBaVaqqo68bDGrB1xoO4Jn", + "4mdEuF1tADUJ7vJpmU9RMCXBlZmQreS9TCI4csW+m6i+pr8fSfM1VcRuIbFYlP8UURoouzmulim4RzZT", + "48PptzDCrdTb+7vntQTmQTI4m4ycxdokQcRyzoK1H+qq91FOs2q18Ce0k07TKHI3HjMiVF5LrXgGrH8G", + "t6TVsr3bbUuPg/P3b7uEBRz80DIfKr8Q5Uoc3a+EbxbMTOUnmTTRCQFVjjDqBeivWH/jLoiyfPn/vvna", + "Zsz/983XJmf+v2/tm6z5aw9GLP3HYs2PLXE/YeLTAjctIw1YkylEtEpCzVo1FFJd+x9KTrVF7W4jqWZ4", + "/SmsNhFWi+haKq9m9QUfUGK1pdi+zZVMRmw+bMMr55/4g0mqj2vlsxRZqNpfuvawKSe5yMuf2ZrfT8+B", + "kmYUVzw2Gpqr8w259PhwpHt02LGV7Uw9uixA5JGM1w6ORxdu7biPb7nej0d0kvJUFmNPoJAhkTZYKSJl", + "BvzUxO78eK4VvL9jKu0/5tHx6HL1T7p/IIm/uqCGeZsbqFUyv2vVVOa37aFkoKlGYWLX3rsqFzaNylqN", + "U6GrA9OUjEslixadHX1w+XQRdK4VlVxdQKBB7A3Yf2r943dFcPzxFxckk/b7mzvwnLDZx19cnAw7dqRC", + "mBKUSIQFQfvvDuHabwLR65AMLQ/Jq8JhUpyZ2tC2bOm/nIKU33w215AcFf7UkBppSAV0LdeQsioqD6ki", + "mUG+mY7k6M2HcJta46eW9BhakkzHYxpQwlSeAXjBScwmEH+CsWXM3g8VnDtKB21jLSkvbbRcAM3T3j26", + "Y082+OMrRy7D3tP0kecmKiZ06kh+GNbrI98bPfQflzk/vh7ylEnMCPxV1C0yovWxTUDsFxBec3HVlPI8", + "eTjvnQDvXzopzvA7lE00eKRQ5fAbiihweBvfek00ZcnlETbkQnLVb+nS6TBhlVsTFEnZJK9zSdWUpyar", + "ytA+NFnZ9K6w1WRA5Alsr9+avejRH0EAfccVonESkZhA1rauoSYoLpomCRdZ/TEqC6mIb8f+9LYpOtia", + "5Da2CnAH2YTNYKxzC9YGu/3icnm5ZsQnq4Nqs8FdBKknqnbAzqVJ8nJpROFLlDFZpDiSJCKBQtdTGkwh", + "wlY/g/5NAC5OkssspcaaK5ZazCwCg7clERRHUOWRR6Ze6eUsji/3FjPAXRwfw0cmuNbkervcQy7rW3ZA", + "SN2qGDGrZxFhqdA7Gwfc1pQkeBSZFb3Up1Bhfms2ljZPeTJgvrhaRq5th3SMLgshtpc1MbaOob7lE/mt", + "5KVOfaIqMxfFkQDEGdokLGzVGXZo5I+u3ej3fflTGkb6GjAeONB3AZi3fJIlySqRMk6SpuRrwQQqnsXx", + "EhpG7UIyc6lCnqq/SBUSIeBjS911xI3aODB/KHylCZXZUmQuHTyQn9d8abLWeFGlmWohn7T5axbHrU7L", + "wuOpnvv1EdPVDhfNbHplCmHRPyXt2wQ8l5l9IeK5cnLYuhX1Irctx/HD63uu3PU3JsNvYB/LoaDMiSqw", + "tnkd8acVOWkqtVRlMZM837dHslIv9bukbFQ+y9P0/wuqqGau1fo8j6ykZij2aWal8hbfXDvNqm381FAz", + "DZULFKZmuEq9mx9W7cwYCkpZSfO04ulddc8syVyGZqhDyJZeCOQ8b/2z+3l0B3HhO+GEndqqL3XpjPJJ", + "fw8st6YmWiOe+43kJHusFgSEb8iCXXW2x+bAGVa0updxue+CDZsNl3HjIs+ByvvUFV78yYxLZkBjKb0r", + "M3bC54ItsMCeKesmEa7jy1ZOrWXAtgrUD6+v5brKD66xBVwI4zoGzmhPKXSxcGdYUD3bCU4l6WQbpuPu", + "rS+Oj9fqNo1QS7eM+D4utO8mOVTKcsahvy6yoKFLUn9wfGhT2lOJRMp66CSmkDn+ipAEUlJSnkoEPoC9", + "Yr2xuipoWUExwpSYJ5wytRKKvOnDAPPlTkm6H5lP2eDtH96sZAvtPjUmBbxDn952AsuVKmXK7Hmv6dy1", + "FWUms74WPvCIp7r3hXpoaEwjIudSkdjc2Y3TCDYRpPew2V/td8Z3rYOoklA9vAO+PgkRMZWSciYHbETG", + "WipJiNBjQ8FJGpHC9YPvZutM4YxrnhrW931cbUGJNLjNwaoOa+XqaDhJXHU03/VJVtDtziC9hrsqJOfx", + "iEc0QBFlVxK1I3plZHA0kyjSP9aWXnYN4bv7zm17952lMX3Extyb/s/QbEbMPwKHO6qwNXeZ/+TY2htS", + "3CyO/8BC+9maXMnXBMERFAHN3GxRqmhEPxlWpzuhUtHA1EzCGe6g3IsZrzdgx0QJ3QYLggIeRSRQztaw", + "nggerA/Sfn8rSCjEQ2wRAA4YXv3rGEY8OD2HdqYkTWfA9B/Q8Yf9U0Q1TsfYqswFQG1he3S0frLi+v8M", + "0PQvrI+ZCS7bFv4F/3mze3sfyto9JGu2KE+WKUA8+eENBlaC+2kteJrWAnBiz2bTnggcgFAsp6kK+TXz", + "WwZMhVS5/tn8OFoVCqFwML1wpaK/D2nXVotdNYyb4JPYlHZOITHpSb+Jvd4W9H2i6Zw04twUQIgpBnX4", + "TwFTKPxHo+77v6wr4vE7vKmzGHWpf7+bvfXYJ5+FwUX4FfHxVLa5oTQ3EyhZWbQ+ZeGMK3WzIBWCMAWp", + "YHLRMsAJDqiadxCOXDVVWx4psyHlheBHguArfdL2Bux9FkhpyzNp7arjVCsUUnllerDaUw+dzIiQ6SgD", + "DgFjMnoeIN8WVA1wFJhKpGQ8JoGiM2JKhMoa7SsD5SHT8uaDeBbavbSoe2oqh58mYPVysrBaR8lTrjZ9", + "w1nWqln6hqzXgjdMwVNkqc/z0DU0VfBvY7LzDH5Fa93i7avbea/9pj9qOHbZS8oPhH31lbP8UbLinRWc", + "U5omfcgp/KnlXyhAXtqqJQev1YHgjT26HtLDalUgeDb4YweCn3mdfJ5YOipcctuqiwD//gih/7jexY8d", + "Af60aUuLEnIBdfWcqEEk+HdBgQ8TAv6NvevvEAL+Xfl7Qgjvt/O7/648Pa3HYubp+TPI+yEdPE2kNwS0", + "1jl4Gq5nLc9LFaUL26aZmmR7/JEkeGusvIX87tD+M2VbA5WhgCx3ClfYDfB+aQmexImaO2sUH4PfTZ5T", + "UNJP4L3nC5zLjM4PF692B3vs/ZGHo9Naa+zPVG+PZvDN82EfHT79/G7FPVc6WNb1qdPFIpjSWSlea9kO", + "tihKBOkmPAE7a2gQZvHhzjKFRW/yCdnuewP2YUrcX4i6bBkkRCEVJFDRHFGmOHAEM8afJBJcawLwnou5", + "z3xb3LmvBY/37WxWnId2T1ljWO7mF8+7IVa4O3PcZokJ7SuurI7xDY3TGBgeogy9eYna5EYJk7wBjbXm", + "g+g4Qym5CQgJJdDkWhHgjX6NZZN+IsPJqAmUS9JwnNg0JyhIpeKxW/ujQ9TGqeLdCWF6LbSoPwZJNhF8", + "RkOTIzdH6oxHBqsbNQi9rd1VCxXWHzxXLgxw30SGaXIgTT7RpMwWjNtja681ogwDcCsTXpT3lPHA1eNh", + "Cn5w+d5xlNP6eYRVq2xrStRKjkOi4hxFWqJf+3nMPeVjrujJ4M600mnXLItpM+eGhj4HD5HBNHN8eVyz", + "9cX3cx9fqEr8BE3ns0whrTObf18k2H+88+GxzeUXT9h/6w1xynfBVA4d6B59BPOWBzhCIZmRiCexFitN", + "21anlYqotdeaKpXsra9Hut2US7W329/tt758/PL/AwAA//8YmkERtSIBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/system/init/egress_proxy.go b/lib/system/init/egress_proxy.go new file mode 100644 index 00000000..bd0c9515 --- /dev/null +++ b/lib/system/init/egress_proxy.go @@ -0,0 +1,32 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + + "github.com/kernel/hypeman/lib/vmconfig" +) + +func installEgressProxyCA(log *Logger, cfg *vmconfig.Config) { + if cfg == nil || cfg.EgressProxy == nil || !cfg.EgressProxy.Enabled || cfg.EgressProxy.CACertPEM == "" { + return + } + + caPath := "/usr/local/share/ca-certificates/hypeman-egress-proxy.crt" + if err := os.MkdirAll(filepath.Dir(caPath), 0755); err != nil { + log.Error("hypeman-init:egress-proxy", "failed to create CA directory", err) + return + } + if err := os.WriteFile(caPath, []byte(cfg.EgressProxy.CACertPEM), 0644); err != nil { + log.Error("hypeman-init:egress-proxy", "failed to write proxy CA", err) + return + } + + cmd := exec.Command("/bin/sh", "-c", "if command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates >/dev/null 2>&1; fi") + if err := cmd.Run(); err != nil { + log.Error("hypeman-init:egress-proxy", "failed to run update-ca-certificates", err) + return + } + log.Info("hypeman-init:egress-proxy", "installed egress proxy CA certificate") +} diff --git a/lib/system/init/mode_exec.go b/lib/system/init/mode_exec.go index 1e1792e8..013512f6 100644 --- a/lib/system/init/mode_exec.go +++ b/lib/system/init/mode_exec.go @@ -41,6 +41,8 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { dropToShell() } + installEgressProxyCA(log, cfg) + // Set up environment os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") os.Setenv("HOME", "/root") diff --git a/lib/system/init/mode_systemd.go b/lib/system/init/mode_systemd.go index eb1ffd61..249a9cd2 100644 --- a/lib/system/init/mode_systemd.go +++ b/lib/system/init/mode_systemd.go @@ -54,6 +54,8 @@ func runSystemdMode(log *Logger, cfg *vmconfig.Config) { dropToShell() } + installEgressProxyCA(log, cfg) + // Build effective command from entrypoint + cmd argv := append(cfg.Entrypoint, cfg.Cmd...) if len(argv) == 0 { diff --git a/lib/vmconfig/config.go b/lib/vmconfig/config.go index df1bf6b7..9d073077 100644 --- a/lib/vmconfig/config.go +++ b/lib/vmconfig/config.go @@ -29,6 +29,9 @@ type Config struct { // Boot optimizations SkipKernelHeaders bool `json:"skip_kernel_headers,omitempty"` SkipGuestAgent bool `json:"skip_guest_agent,omitempty"` + + // Optional egress MITM proxy configuration. + EgressProxy *EgressProxyConfig `json:"egress_proxy,omitempty"` } // VolumeMount represents a volume mount configuration. @@ -38,3 +41,10 @@ type VolumeMount struct { Mode string `json:"mode"` // "ro", "rw", or "overlay" OverlayDevice string `json:"overlay_device,omitempty"` } + +// EgressProxyConfig configures guest-side trust and proxy endpoint wiring. +type EgressProxyConfig struct { + Enabled bool `json:"enabled"` + ProxyURL string `json:"proxy_url"` + CACertPEM string `json:"ca_cert_pem,omitempty"` +} diff --git a/openapi.yaml b/openapi.yaml index cc61fc8f..32200832 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -162,6 +162,45 @@ components: example: PORT: "3000" NODE_ENV: production + egress_proxy: + type: object + description: Optional host-side HTTP/HTTPS MITM egress proxy mode. + properties: + enabled: + type: boolean + description: Whether to enable egress proxy mode. + default: false + example: true + mock_env_vars: + type: array + items: + type: string + description: | + Environment variable names (from `env`) that should be mocked inside the VM + as `mock-` and rewritten back to their real values on egress. + example: [OUTBOUND_OPENAI_KEY] + mock_env_var_domains: + type: object + additionalProperties: + type: array + items: + type: string + description: | + Optional per-mocked-env destination domain allowlist for secret substitution. + Keys are env var names from `mock_env_vars`; values are allowed destination + host patterns (`api.example.com`, `*.example.com`). If a mock env var is not + present in this map, substitution is allowed for all HTTPS destinations. + example: + OUTBOUND_OPENAI_KEY: [api.openai.com, "*.openai.com"] + enforcement_mode: + type: string + enum: [all, http_https_only] + default: all + description: | + Egress proxy host enforcement mode. + `all` (default when proxy is enabled) rejects direct non-proxy TCP egress from the VM, + while `http_https_only` rejects direct egress only on TCP ports 80 and 443. + example: all tags: $ref: "#/components/schemas/Tags" network: diff --git a/scripts/e2e-install-test.sh b/scripts/e2e-install-test.sh index 764d0353..1ebd78db 100755 --- a/scripts/e2e-install-test.sh +++ b/scripts/e2e-install-test.sh @@ -139,14 +139,34 @@ pass "hypeman ps works" # VM lifecycle test E2E_VM_NAME="e2e-test-vm" +E2E_IMAGE="${HYPEMAN_E2E_IMAGE:-nginx:alpine}" -$HYPEMAN_CMD pull nginx:alpine || fail "hypeman pull failed" +# Pull is async and can fail transiently on first attempt (registry/network startup race). +PULL_OK=false +for i in $(seq 1 5); do + if $HYPEMAN_CMD pull "$E2E_IMAGE"; then + PULL_OK=true + break + fi + warn "hypeman pull attempt ${i}/5 failed for ${E2E_IMAGE}" + sleep 2 +done +if [ "$PULL_OK" != true ]; then + if [ "$OS" = "darwin" ]; then + LOG_FILE="$HOME/Library/Application Support/hypeman/logs/hypeman.log" + if [ -f "$LOG_FILE" ]; then + warn "Service logs (last 100 lines):" + tail -100 "$LOG_FILE" || true + fi + fi + fail "hypeman pull failed" +fi pass "hypeman pull works" # Wait for image to be available (pull is async) IMAGE_READY=false for i in $(seq 1 30); do - if $HYPEMAN_CMD run --name "$E2E_VM_NAME" nginx:alpine 2>&1; then + if $HYPEMAN_CMD run --name "$E2E_VM_NAME" "$E2E_IMAGE" 2>&1; then IMAGE_READY=true break fi