From 07c2c4d87077e05da852be63b3560dd9f73f8568 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 11 Aug 2026 09:16:24 +0300 Subject: [PATCH 1/3] feat(aichat): resolve hierarchical runtime profiles Claude-Session-Id: 019fef59-46c9-7b83-bc02-6bd4246d90de --- .../aimock_lifecycle_integration_test.go | 4 +- pkg/aichat/approval_execution.go | 6 +- pkg/aichat/execution.go | 1 + pkg/aichat/execution_authority_ginkgo_test.go | 10 + pkg/aichat/execution_database_authority.go | 18 +- pkg/aichat/messages.go | 35 +- pkg/aichat/provider_config.go | 83 +++++ pkg/aichat/runtime_profile_ginkgo_test.go | 80 +++++ pkg/aichat/runtime_settings.go | 35 +- pkg/aichat/service.go | 75 +++-- pkg/aichat/service_ginkgo_test.go | 42 +-- pkg/api/spec_layers.go | 299 ++++++++++++++++++ pkg/api/spec_layers_ginkgo_test.go | 100 ++++++ pkg/cli/serve_chat.go | 15 +- 14 files changed, 712 insertions(+), 91 deletions(-) create mode 100644 pkg/aichat/runtime_profile_ginkgo_test.go create mode 100644 pkg/api/spec_layers.go create mode 100644 pkg/api/spec_layers_ginkgo_test.go diff --git a/pkg/aichat/aimock_lifecycle_integration_test.go b/pkg/aichat/aimock_lifecycle_integration_test.go index a95ad0d1..b0c7db5d 100644 --- a/pkg/aichat/aimock_lifecycle_integration_test.go +++ b/pkg/aichat/aimock_lifecycle_integration_test.go @@ -132,8 +132,8 @@ var _ = Describe("Mocked Captain chat lifecycle", func() { var approvedInput map[string]any service := aichat.NewService(aichat.ServiceOptions{ Resolver: realChatResolver{}, Threads: aichat.FixedThreadStore(store), Authority: authority, - Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) { - return aichat.RuntimeSettings{ProviderConfig: api.Config{ + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return aichat.RuntimeProfile{ProviderConfig: api.Config{ APIURL: mock.apiURL, APIKey: aimock.DummyKey, }}, nil }), diff --git a/pkg/aichat/approval_execution.go b/pkg/aichat/approval_execution.go index 51a15a1c..d54e01f8 100644 --- a/pkg/aichat/approval_execution.go +++ b/pkg/aichat/approval_execution.go @@ -51,9 +51,9 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti } execution := continuation.Execution defer closeExecution(execution) - settings, err := s.runtimeSettings(ctx) + profile, err := s.runtimeProfile(ctx) if err != nil { - return fmt.Errorf("load chat runtime settings: %w", err) + return fmt.Errorf("load chat runtime profile: %w", err) } set, err := s.loadTools(ctx) if err != nil { @@ -63,7 +63,7 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti if err != nil { return err } - config := settings.ProviderConfig + config := profile.ProviderConfig config.Model = continuation.Spec.Model config.Budget = continuation.Spec.Budget config.SessionID = continuation.Spec.SessionID diff --git a/pkg/aichat/execution.go b/pkg/aichat/execution.go index 9a5a0304..a36c338c 100644 --- a/pkg/aichat/execution.go +++ b/pkg/aichat/execution.go @@ -15,6 +15,7 @@ type ExecutionRequest struct { RequestID string Title string Spec api.Spec + Profile api.ResolvedSpec Definitions []api.ToolDefinition } diff --git a/pkg/aichat/execution_authority_ginkgo_test.go b/pkg/aichat/execution_authority_ginkgo_test.go index b7ea15c9..62b36401 100644 --- a/pkg/aichat/execution_authority_ginkgo_test.go +++ b/pkg/aichat/execution_authority_ginkgo_test.go @@ -171,6 +171,12 @@ var _ = Describe("Authoritative aichat execution", func() { } service := aichat.NewService(aichat.ServiceOptions{ Resolver: &fakeResolver{provider: provider}, Threads: aichat.FixedThreadStore(store), Authority: authority, + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return mustRuntimeProfile(api.SpecLayer{ + Name: "accounts", Scope: api.SpecLayerContext, + Spec: api.Spec{Prompt: api.Prompt{System: "Use account policy."}}, + }), nil + }), Tools: aichat.StaticToolProvider([]api.ToolDefinition{{ Name: "account_edit", DefaultPermission: api.ToolModeAsk, Handler: func(context.Context, map[string]any) (any, error) { return nil, nil }, @@ -191,6 +197,10 @@ var _ = Describe("Authoritative aichat execution", func() { Expect(authority.begins).To(HaveLen(1)) Expect(authority.begins[0].ThreadID).To(Equal(thread.ID)) Expect(authority.begins[0].RequestID).To(Equal("user-message-1")) + Expect(authority.begins[0].Profile.Trace).To(HaveLen(2)) + Expect(authority.begins[0].Profile.Trace[0].Name).To(Equal("accounts")) + Expect(authority.begins[0].Profile.Spec).To(Equal(authority.begins[0].Spec)) + Expect(authority.begins[0].Profile.Spec.Prompt.System).To(Equal("Use account policy.")) Expect(provider.specs).To(HaveLen(1)) Expect(execution.observed).To(ContainElement( MatchFields(IgnoreExtras, Fields{"Kind": Equal(api.EventResult)}), diff --git a/pkg/aichat/execution_database_authority.go b/pkg/aichat/execution_database_authority.go index 1a38f9e7..b3789e6d 100644 --- a/pkg/aichat/execution_database_authority.go +++ b/pkg/aichat/execution_database_authority.go @@ -46,7 +46,7 @@ func (a *DatabaseExecutionAuthority) Begin( if session.Source != "aichat" { return nil, fmt.Errorf("chat thread %s has incompatible source %q", request.ThreadID, session.Source) } - renderedSpec, err := renderedSpecMap(request.Spec) + renderedSpec, err := renderedSpecMap(request.Spec, request.Profile) if err != nil { return nil, err } @@ -278,7 +278,7 @@ func runtimeSelection(model api.Model) database.PromptRunRuntimeSelection { } } -func renderedSpecMap(spec api.Spec) (map[string]any, error) { +func renderedSpecMap(spec api.Spec, profile api.ResolvedSpec) (map[string]any, error) { raw, err := json.Marshal(spec) if err != nil { return nil, fmt.Errorf("encode authoritative chat spec: %w", err) @@ -287,6 +287,20 @@ func renderedSpecMap(spec api.Spec) (map[string]any, error) { if err := json.Unmarshal(raw, &rendered); err != nil { return nil, fmt.Errorf("decode authoritative chat spec: %w", err) } + if len(profile.Trace) > 0 { + resolution, err := json.Marshal(struct { + Constraints api.RuntimeConstraints `json:"constraints"` + Trace []api.SpecLayer `json:"trace"` + }{Constraints: profile.Constraints, Trace: profile.Trace}) + if err != nil { + return nil, fmt.Errorf("encode authoritative chat profile: %w", err) + } + var value map[string]any + if err := json.Unmarshal(resolution, &value); err != nil { + return nil, fmt.Errorf("decode authoritative chat profile: %w", err) + } + rendered["resolution"] = value + } return rendered, nil } diff --git a/pkg/aichat/messages.go b/pkg/aichat/messages.go index 0665549f..3529ecb3 100644 --- a/pkg/aichat/messages.go +++ b/pkg/aichat/messages.go @@ -68,19 +68,31 @@ func (s *Service) resolveAttachments(ctx context.Context, messages []UIMessage) return resolved, nil } -func requestSpec(request ChatRequest, settings RuntimeSettings, attachments map[partLocation]api.AttachmentRef) (api.Spec, error) { - override, err := chatModel(request, settings.Spec.Model) +func requestSpec(request ChatRequest, profile RuntimeProfile, attachments map[partLocation]api.AttachmentRef) (api.ResolvedSpec, error) { + if len(profile.Resolved.Trace) == 0 && (!api.IsEmpty(profile.Resolved.Spec) || !api.IsEmpty(profile.Resolved.Constraints)) { + return api.ResolvedSpec{}, fmt.Errorf("chat runtime profile must include its resolution trace") + } + override, err := chatModel(request, profile.Resolved.Spec.Model) if err != nil { - return api.Spec{}, err + return api.ResolvedSpec{}, err } - spec := settings.Spec.Merge(api.Spec{ + user := api.SpecLayer{Name: "chat request", Scope: api.SpecLayerUser, Spec: api.Spec{ Model: override, Budget: request.Budget, ToolPreferences: request.ToolPreferences, ToolApproval: request.ToolApproval, Permissions: api.Permissions{Mode: request.PermissionMode}, SessionID: request.ProviderSessionID, - }) + }} + layers := append([]api.SpecLayer(nil), profile.Resolved.Trace...) + resolved, err := api.ResolveSpecLayers(append(layers, user)...) + if err != nil { + return api.ResolvedSpec{}, fmt.Errorf("resolve chat runtime profile: %w", err) + } + spec := resolved.Spec + baseSystem := strings.TrimSpace(strings.Join([]string{ + profile.System, spec.Prompt.System, spec.Prompt.AppendSystem, + }, "\n\n")) spec.Prompt.User = "" spec.Prompt.System = "" spec.Prompt.AppendSystem = "" @@ -88,16 +100,16 @@ func requestSpec(request ChatRequest, settings RuntimeSettings, attachments map[ if request.ToolApproval == nil { messages, err := canonicalMessages(request.Messages, attachments) if err != nil { - return api.Spec{}, err + return api.ResolvedSpec{}, err } - system, err := requestSystem(settings.System, request) + system, err := requestSystem(baseSystem, request) if err != nil { - return api.Spec{}, err + return api.ResolvedSpec{}, err } if isAgentBackend(spec.Backend) { user, promptAttachments, err := agentPrompt(messages, request.ProviderSessionID != "") if err != nil { - return api.Spec{}, err + return api.ResolvedSpec{}, err } spec.Messages = nil spec.Prompt.System = system @@ -113,9 +125,10 @@ func requestSpec(request ChatRequest, settings RuntimeSettings, attachments map[ spec.Messages = nil } if err := spec.Validate(); err != nil { - return api.Spec{}, err + return api.ResolvedSpec{}, err } - return spec, nil + resolved.Spec = spec + return resolved, nil } func chatModel(request ChatRequest, fallback api.Model) (api.Model, error) { diff --git a/pkg/aichat/provider_config.go b/pkg/aichat/provider_config.go index 2d37b888..3547e2fb 100644 --- a/pkg/aichat/provider_config.go +++ b/pkg/aichat/provider_config.go @@ -4,11 +4,94 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/api" ) +func annotateProfileModels(resolved api.ResolvedSpec, models ModelCatalogResponse) { + if len(resolved.Constraints.Models) == 0 { + return + } + for index := range models { + if resolved.AllowsModel(models[index].Runtime) { + continue + } + layer := modelRestrictionLayer(resolved, models[index].Runtime) + models[index].Configured = false + models[index].Default = false + models[index].Availability = restrictedAvailability(layer, resolved.Constraints.Models) + } +} + +func annotateProfileRuntimes(resolved api.ResolvedSpec, runtimes []api.RuntimeFamily) { + if len(resolved.Constraints.Models) == 0 { + return + } + for familyIndex := range runtimes { + for modeIndex := range runtimes[familyIndex].Modes { + mode := &runtimes[familyIndex].Modes[modeIndex] + backend := api.Backend(mode.Backend) + if runtimeAllowed(resolved.Constraints.Models, backend) { + continue + } + layer := runtimeRestrictionLayer(resolved, backend) + mode.Disabled = true + if layer != nil { + mode.DisabledReason = fmt.Sprintf("%s layer %s", layer.Scope, layer.Name) + } + mode.Availability = restrictedAvailability(layer, resolved.Constraints.Models) + } + } +} + +func restrictedAvailability(layer *api.SpecLayer, allowed []string) api.Availability { + reason := "Unavailable because the resolved runtime profile restricts the model catalog." + if layer != nil { + reason = fmt.Sprintf("Unavailable because %s layer %q restricts the model catalog.", layer.Scope, layer.Name) + } + return api.Availability{ + State: api.AvailabilityDisabled, + Reason: reason, + Remediation: "Select one of the allowed models: " + strings.Join(allowed, ", ") + ".", + } +} + +func modelRestrictionLayer(resolved api.ResolvedSpec, model api.Model) *api.SpecLayer { + for index := len(resolved.Trace) - 1; index >= 0; index-- { + layer := &resolved.Trace[index] + if len(layer.Constraints.Models) > 0 && !(api.ResolvedSpec{Constraints: layer.Constraints}).AllowsModel(model) { + return layer + } + } + return nil +} + +func runtimeRestrictionLayer(resolved api.ResolvedSpec, backend api.Backend) *api.SpecLayer { + for index := len(resolved.Trace) - 1; index >= 0; index-- { + layer := &resolved.Trace[index] + if len(layer.Constraints.Models) > 0 && !runtimeAllowed(layer.Constraints.Models, backend) { + return layer + } + } + return nil +} + +func runtimeAllowed(models []string, backend api.Backend) bool { + providerPrefix := ai.BackendToProvider(backend) + "/" + for _, selector := range models { + if strings.HasPrefix(selector, providerPrefix) { + return true + } + model, err := (api.Model{Name: selector}).Expand() + if err == nil && model.Backend == backend { + return true + } + } + return false +} + // ProviderConfigRequest carries the canonically resolved model and the runtime // config assembled by the chat service. type ProviderConfigRequest struct { diff --git a/pkg/aichat/runtime_profile_ginkgo_test.go b/pkg/aichat/runtime_profile_ginkgo_test.go new file mode 100644 index 00000000..fecc247d --- /dev/null +++ b/pkg/aichat/runtime_profile_ginkgo_test.go @@ -0,0 +1,80 @@ +package aichat_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/flanksource/captain/pkg/aichat" + "github.com/flanksource/captain/pkg/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Resolved runtime profiles", func() { + It("annotates models outside the effective catalog with actionable provenance", func() { + resolver := &fakeResolver{models: aichat.ModelCatalogResponse{ + {ID: "openai/gpt-5.6-sol", Provider: "openai", Label: "GPT", Runtime: api.Model{Name: "gpt-5.6-sol", Backend: api.BackendOpenAI}, Configured: true, Availability: api.Available()}, + {ID: "anthropic/claude-sonnet-5", Provider: "anthropic", Label: "Claude", Runtime: api.Model{Name: "claude-sonnet-5", Backend: api.BackendAnthropic}, Configured: true, Availability: api.Available()}, + }} + profile := mustRuntimeProfile(api.SpecLayer{ + Name: "claims", Scope: api.SpecLayerContext, + Constraints: api.RuntimeConstraints{Models: []string{"gpt-5.6-sol"}}, + }) + service := aichat.NewService(aichat.ServiceOptions{ + Resolver: resolver, + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return profile, nil + }), + }) + + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/chat/models", nil)) + + Expect(response.Code).To(Equal(http.StatusOK)) + var models aichat.ModelCatalogResponse + Expect(json.Unmarshal(response.Body.Bytes(), &models)).To(Succeed()) + Expect(models[0].Availability).To(Equal(api.Available())) + Expect(models[1].Configured).To(BeFalse()) + Expect(models[1].Availability.State).To(Equal(api.AvailabilityDisabled)) + Expect(models[1].Availability.Reason).To(ContainSubstring(`context layer "claims"`)) + Expect(models[1].Availability.Remediation).To(ContainSubstring("allowed models")) + }) + + It("checks every named quota and reports the independently exhausted allowance", func() { + resolver := &fakeResolver{provider: &fakeStreamingProvider{}} + profile := mustRuntimeProfile( + api.SpecLayer{ + Name: "platform", Scope: api.SpecLayerGlobal, + Spec: api.Spec{Model: api.Model{Name: "openai/test-model"}}, + Constraints: api.RuntimeConstraints{Quotas: []api.RuntimeQuota{{Name: "platform-monthly", TokenLimit: 100, TokensUsed: 10}}}, + }, + api.SpecLayer{ + Name: "claims", Scope: api.SpecLayerContext, + Constraints: api.RuntimeConstraints{Quotas: []api.RuntimeQuota{{Name: "claims-monthly", CostLimitUSD: 20, CostUsedUSD: 20}}}, + }, + ) + service := aichat.NewService(aichat.ServiceOptions{ + Resolver: resolver, + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return profile, nil + }), + }) + + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, requestJSON(http.MethodPost, "/api/chat", aichat.ChatRequest{ + Messages: []aichat.UIMessage{{Role: "user", Parts: []aichat.UIPart{{Type: "text", Text: "hello"}}}}, + })) + + Expect(response.Code).To(Equal(http.StatusPaymentRequired)) + Expect(response.Body.String()).To(ContainSubstring(`context quota "claims-monthly" from layer "claims"`)) + Expect(resolver.configs).To(BeEmpty()) + }) +}) + +func mustRuntimeProfile(layers ...api.SpecLayer) aichat.RuntimeProfile { + resolved, err := api.ResolveSpecLayers(layers...) + Expect(err).NotTo(HaveOccurred()) + return aichat.RuntimeProfile{Resolved: resolved} +} diff --git a/pkg/aichat/runtime_settings.go b/pkg/aichat/runtime_settings.go index 0e9fff1f..5229f2b5 100644 --- a/pkg/aichat/runtime_settings.go +++ b/pkg/aichat/runtime_settings.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "net/http" + + "github.com/flanksource/captain/pkg/api" ) type requestError struct { @@ -20,20 +22,23 @@ func requestErrorStatus(err error) int { return http.StatusBadRequest } -func enforceRuntimeSettings(request ChatRequest, settings RuntimeSettings) error { - if settings.MonthlyBudgetUSD > 0 && settings.CurrentMonthCostUSD >= settings.MonthlyBudgetUSD { - return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( - "chat monthly cost budget exhausted: $%.4f used of $%.4f", - settings.CurrentMonthCostUSD, settings.MonthlyBudgetUSD, - )} - } - if settings.MonthlyTokenBudget > 0 && settings.CurrentMonthTokens >= settings.MonthlyTokenBudget { - return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( - "chat monthly token budget exhausted: %d used of %d", - settings.CurrentMonthTokens, settings.MonthlyTokenBudget, - )} +func enforceRuntimeProfile(request ChatRequest, resolved api.ResolvedSpec) error { + for _, quota := range resolved.Constraints.Quotas { + if quota.CostLimitUSD > 0 && quota.CostUsedUSD >= quota.CostLimitUSD { + return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( + "chat %s quota %q from layer %q exhausted: $%.4f used of $%.4f", + quota.Scope, quota.Name, quota.Layer, quota.CostUsedUSD, quota.CostLimitUSD, + )} + } + if quota.TokenLimit > 0 && quota.TokensUsed >= quota.TokenLimit { + return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( + "chat %s quota %q from layer %q exhausted: %d tokens used of %d", + quota.Scope, quota.Name, quota.Layer, quota.TokensUsed, quota.TokenLimit, + )} + } } - if settings.MaxInputTokens <= 0 { + maxInputTokens := resolved.Constraints.Limits.MaxInputTokens + if maxInputTokens <= 0 { return nil } raw, err := json.Marshal(struct { @@ -45,10 +50,10 @@ func enforceRuntimeSettings(request ChatRequest, settings RuntimeSettings) error return fmt.Errorf("estimate chat input tokens: %w", err) } estimated := (len(raw) + 3) / 4 - if estimated > settings.MaxInputTokens { + if estimated > maxInputTokens { return requestError{status: http.StatusRequestEntityTooLarge, text: fmt.Sprintf( "chat input is about %d tokens, exceeding the configured per-turn limit of %d", - estimated, settings.MaxInputTokens, + estimated, maxInputTokens, )} } return nil diff --git a/pkg/aichat/service.go b/pkg/aichat/service.go index d8ec5944..118342e8 100644 --- a/pkg/aichat/service.go +++ b/pkg/aichat/service.go @@ -17,34 +17,23 @@ import ( var serviceLog = logger.GetLogger("aichat") -// RuntimeSettings are application-owned defaults and provider construction -// settings evaluated for each request. -// RuntimeSettings is the request-scoped application configuration for a chat. -// -// The default model lives in Spec.Model — there is deliberately no DefaultModel -// string beside it. A bare name next to a structured Spec is the lossy pattern: -// it cannot carry a backend/mode/effort, so whatever it named got re-inferred, and -// when both were set they could silently disagree. Spec.Model can say -// {Name: "sol", Mode: ModeAgent} and mean it. -type RuntimeSettings struct { - System string - Spec api.Spec - ProviderConfig api.Config - MaxInputTokens int - MonthlyTokenBudget int - CurrentMonthTokens int - MonthlyBudgetUSD float64 - CurrentMonthCostUSD float64 +// RuntimeProfile is the request-scoped, hierarchically resolved application +// configuration for a chat. Resolved carries the effective Spec, constraints, +// and ordered provenance; provider credentials remain runtime-only. +type RuntimeProfile struct { + System string + Resolved api.ResolvedSpec + ProviderConfig api.Config } -// RuntimeSettingsProvider supplies request-scoped application settings. -type RuntimeSettingsProvider interface { - RuntimeSettings(context.Context) (RuntimeSettings, error) +// RuntimeProfileProvider supplies request-scoped application profiles. +type RuntimeProfileProvider interface { + RuntimeProfile(context.Context) (RuntimeProfile, error) } -type RuntimeSettingsProviderFunc func(context.Context) (RuntimeSettings, error) +type RuntimeProfileProviderFunc func(context.Context) (RuntimeProfile, error) -func (f RuntimeSettingsProviderFunc) RuntimeSettings(ctx context.Context) (RuntimeSettings, error) { +func (f RuntimeProfileProviderFunc) RuntimeProfile(ctx context.Context) (RuntimeProfile, error) { return f(ctx) } @@ -71,7 +60,7 @@ func FixedThreadStore(store ThreadStore) ThreadStoreProvider { type ServiceOptions struct { Resolver Resolver ProviderConfig ProviderConfigSource - Settings RuntimeSettingsProvider + Profile RuntimeProfileProvider Tools ToolProvider MCP ToolProvider Attachments AttachmentResolver @@ -106,6 +95,11 @@ func (s *Service) Handler() http.Handler { } func (s *Service) handleRuntimes(w http.ResponseWriter, request *http.Request) { + profile, err := s.runtimeProfile(request.Context()) + if err != nil { + http.Error(w, fmt.Sprintf("load chat runtime profile: %v", err), http.StatusInternalServerError) + return + } runtimes, err := s.resolver.Runtimes(request.Context()) if err != nil { http.Error(w, err.Error(), http.StatusServiceUnavailable) @@ -115,12 +109,18 @@ func (s *Service) handleRuntimes(w http.ResponseWriter, request *http.Request) { http.Error(w, err.Error(), http.StatusServiceUnavailable) return } + annotateProfileRuntimes(profile.Resolved, runtimes) if err := writeJSON(w, http.StatusOK, runtimes); err != nil { serviceLog.Errorf("write chat runtimes response: %v", err) } } func (s *Service) handleModels(w http.ResponseWriter, request *http.Request) { + profile, err := s.runtimeProfile(request.Context()) + if err != nil { + http.Error(w, fmt.Sprintf("load chat runtime profile: %v", err), http.StatusInternalServerError) + return + } models, err := s.resolver.Models(request.Context()) if err != nil { http.Error(w, err.Error(), http.StatusServiceUnavailable) @@ -130,12 +130,17 @@ func (s *Service) handleModels(w http.ResponseWriter, request *http.Request) { http.Error(w, err.Error(), http.StatusServiceUnavailable) return } + annotateProfileModels(profile.Resolved, models) if err := writeJSON(w, http.StatusOK, models); err != nil { serviceLog.Errorf("write chat models response: %v", err) } } func (s *Service) handleTools(w http.ResponseWriter, request *http.Request) { + if _, err := s.runtimeProfile(request.Context()); err != nil { + http.Error(w, fmt.Sprintf("load chat runtime profile: %v", err), http.StatusInternalServerError) + return + } set, err := s.loadTools(request.Context()) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -157,12 +162,12 @@ func (s *Service) handleChat(w http.ResponseWriter, request *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - settings, err := s.runtimeSettings(request.Context()) + profile, err := s.runtimeProfile(request.Context()) if err != nil { - http.Error(w, fmt.Sprintf("load chat runtime settings: %v", err), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("load chat runtime profile: %v", err), http.StatusInternalServerError) return } - if err := enforceRuntimeSettings(chat, settings); err != nil { + if err := enforceRuntimeProfile(chat, profile.Resolved); err != nil { http.Error(w, err.Error(), requestErrorStatus(err)) return } @@ -183,11 +188,12 @@ func (s *Service) handleChat(w http.ResponseWriter, request *http.Request) { return } } - spec, err := requestSpec(chat, settings, attachments) + resolved, err := requestSpec(chat, profile, attachments) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } + spec := resolved.Spec set, err := s.loadTools(request.Context()) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -205,7 +211,7 @@ func (s *Service) handleChat(w http.ResponseWriter, request *http.Request) { definitions = append(definitions, s.sessionTitleTool(chat.ThreadID)) appendSessionTitleInstruction(&spec) } - config := settings.ProviderConfig + config := profile.ProviderConfig config.Model = spec.Model config.Budget = spec.Budget config.SessionID = spec.SessionID @@ -217,6 +223,7 @@ func (s *Service) handleChat(w http.ResponseWriter, request *http.Request) { return } spec.Model = config.Model + resolved.Spec = spec var execution Execution var callerToolEvents <-chan api.Event if s.options.Authority != nil && chat.ThreadID != "" { @@ -226,7 +233,7 @@ func (s *Service) handleChat(w http.ResponseWriter, request *http.Request) { } execution, err = s.options.Authority.Begin(request.Context(), ExecutionRequest{ ThreadID: chat.ThreadID, RequestID: turnID, Title: title, - Spec: spec, Definitions: definitions, + Spec: spec, Profile: resolved, Definitions: definitions, }) if err != nil { http.Error(w, fmt.Sprintf("admit chat execution: %v", err), http.StatusInternalServerError) @@ -365,11 +372,11 @@ func validateThreadTurn(request ChatRequest, thread *Thread) error { return nil } -func (s *Service) runtimeSettings(ctx context.Context) (RuntimeSettings, error) { - if s.options.Settings == nil { - return RuntimeSettings{}, nil +func (s *Service) runtimeProfile(ctx context.Context) (RuntimeProfile, error) { + if s.options.Profile == nil { + return RuntimeProfile{}, nil } - return s.options.Settings.RuntimeSettings(ctx) + return s.options.Profile.RuntimeProfile(ctx) } func (s *Service) resolveThreadSession(ctx context.Context, request *ChatRequest) (*Thread, error) { diff --git a/pkg/aichat/service_ginkgo_test.go b/pkg/aichat/service_ginkgo_test.go index 2af07692..a7c00ae2 100644 --- a/pkg/aichat/service_ginkgo_test.go +++ b/pkg/aichat/service_ginkgo_test.go @@ -153,8 +153,8 @@ var _ = Describe("Captain aichat service", func() { } service := aichat.NewService(aichat.ServiceOptions{ Resolver: resolver, ProviderConfig: source, - Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) { - return aichat.RuntimeSettings{Spec: api.Spec{Model: api.Model{Name: "api:gpt-5.4"}}}, nil + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return mustRuntimeProfile(api.SpecLayer{Name: "application", Scope: api.SpecLayerGlobal, Spec: api.Spec{Model: api.Model{Name: "api:gpt-5.4"}}}), nil }), }) @@ -180,8 +180,8 @@ var _ = Describe("Captain aichat service", func() { }} service := aichat.NewService(aichat.ServiceOptions{ Resolver: resolver, ProviderConfig: source, - Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) { - return aichat.RuntimeSettings{Spec: api.Spec{Model: api.Model{Name: "api:gpt-5.4"}}}, nil + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return mustRuntimeProfile(api.SpecLayer{Name: "application", Scope: api.SpecLayerGlobal, Spec: api.Spec{Model: api.Model{Name: "api:gpt-5.4"}}}), nil }), }) @@ -202,11 +202,11 @@ var _ = Describe("Captain aichat service", func() { resolver := &fakeResolver{provider: provider} service := aichat.NewService(aichat.ServiceOptions{ Resolver: resolver, - Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) { - return aichat.RuntimeSettings{Spec: api.Spec{ + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return mustRuntimeProfile(api.SpecLayer{Name: "application", Scope: api.SpecLayerGlobal, Spec: api.Spec{ Model: api.Model{Name: "openai/test-model"}, Budget: api.Budget{Cost: 5, MaxTokens: 2_000, MaxTurns: 3}, - }}, nil + }}), nil }), }) @@ -224,11 +224,14 @@ var _ = Describe("Captain aichat service", func() { resolver := &fakeResolver{provider: &fakeStreamingProvider{}} service := aichat.NewService(aichat.ServiceOptions{ Resolver: resolver, - Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) { - return aichat.RuntimeSettings{ - Spec: api.Spec{Model: api.Model{Name: "openai/test-model"}}, - MonthlyBudgetUSD: 10, CurrentMonthCostUSD: 10, - }, nil + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return mustRuntimeProfile(api.SpecLayer{ + Name: "application", Scope: api.SpecLayerGlobal, + Spec: api.Spec{Model: api.Model{Name: "openai/test-model"}}, + Constraints: api.RuntimeConstraints{Quotas: []api.RuntimeQuota{{ + Name: "application-monthly", CostLimitUSD: 10, CostUsedUSD: 10, + }}}, + }), nil }), }) @@ -288,12 +291,11 @@ var _ = Describe("Captain aichat service", func() { resolver := &fakeResolver{provider: provider} service := aichat.NewService(aichat.ServiceOptions{ Resolver: resolver, - Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) { - return aichat.RuntimeSettings{ - Spec: api.Spec{Model: api.Model{Name: "openai/test-model"}}, - System: "Use application tools.", - ProviderConfig: api.Config{APIURL: "https://example.com/ai", ProjectName: "tenant-x"}, - }, nil + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + profile := mustRuntimeProfile(api.SpecLayer{Name: "application", Scope: api.SpecLayerGlobal, Spec: api.Spec{Model: api.Model{Name: "openai/test-model"}}}) + profile.System = "Use application tools." + profile.ProviderConfig = api.Config{APIURL: "https://example.com/ai", ProjectName: "tenant-x"} + return profile, nil }), Attachments: fakeAttachmentResolver{}, Tools: aichat.ToolProviderFunc(func(context.Context) (aichat.ToolSet, error) { @@ -346,8 +348,8 @@ var _ = Describe("Captain aichat service", func() { resolver := &fakeResolver{provider: provider} service := aichat.NewService(aichat.ServiceOptions{ Resolver: resolver, Threads: aichat.FixedThreadStore(store), - Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) { - return aichat.RuntimeSettings{System: "Use accounting tools."}, nil + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return aichat.RuntimeProfile{System: "Use accounting tools."}, nil }), }) diff --git a/pkg/api/spec_layers.go b/pkg/api/spec_layers.go new file mode 100644 index 00000000..3320ffce --- /dev/null +++ b/pkg/api/spec_layers.go @@ -0,0 +1,299 @@ +package api + +import ( + "fmt" + "slices" + "strings" + "time" +) + +// SpecLayerScope identifies one deterministic level in a resolved runtime profile. +type SpecLayerScope string + +const ( + SpecLayerGlobal SpecLayerScope = "global" + SpecLayerContext SpecLayerScope = "context" + SpecLayerSurface SpecLayerScope = "surface" + SpecLayerUser SpecLayerScope = "user" +) + +// RuntimeLimits are ceilings applied after structural Spec defaults are layered. +type RuntimeLimits struct { + MaxInputTokens int `json:"maxInputTokens,omitempty" yaml:"maxInputTokens,omitempty"` + Budget Budget `json:"budget,omitempty" yaml:"budget,omitempty"` +} + +// RuntimeQuota is one independently enforced usage allowance. +type RuntimeQuota struct { + Name string `json:"name" yaml:"name"` + Scope SpecLayerScope `json:"scope" yaml:"scope"` + Layer string `json:"layer" yaml:"layer"` + TokenLimit int `json:"tokenLimit,omitempty" yaml:"tokenLimit,omitempty"` + TokensUsed int `json:"tokensUsed,omitempty" yaml:"tokensUsed,omitempty"` + CostLimitUSD float64 `json:"costLimitUsd,omitempty" yaml:"costLimitUsd,omitempty"` + CostUsedUSD float64 `json:"costUsedUsd,omitempty" yaml:"costUsedUsd,omitempty"` +} + +// RuntimeConstraints restrict values a later Spec layer may select. +type RuntimeConstraints struct { + Models []string `json:"models,omitempty" yaml:"models,omitempty"` + Limits RuntimeLimits `json:"limits,omitempty" yaml:"limits,omitempty"` + Quotas []RuntimeQuota `json:"quotas,omitempty" yaml:"quotas,omitempty"` +} + +// SpecLayer is one named source of runtime defaults and constraints. +type SpecLayer struct { + Name string `json:"name" yaml:"name"` + Scope SpecLayerScope `json:"scope" yaml:"scope"` + Spec Spec `json:"spec,omitempty" yaml:"spec,omitempty"` + Constraints RuntimeConstraints `json:"constraints,omitempty" yaml:"constraints,omitempty"` +} + +// ResolvedSpec is Captain's effective runtime profile plus ordered provenance. +type ResolvedSpec struct { + Spec Spec `json:"spec" yaml:"spec"` + Constraints RuntimeConstraints `json:"constraints" yaml:"constraints"` + Trace []SpecLayer `json:"trace" yaml:"trace"` +} + +// PromptSpecLayer adapts parsed .prompt frontmatter into the normal surface layer. +func PromptSpecLayer(name string, spec Spec) SpecLayer { + return SpecLayer{Name: name, Scope: SpecLayerSurface, Spec: spec} +} + +// ResolveSpecLayers deterministically overlays defaults and intersects constraints. +func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { + layers := append([]SpecLayer(nil), input...) + slices.SortStableFunc(layers, func(left, right SpecLayer) int { + return scopeRank(left.Scope) - scopeRank(right.Scope) + }) + + resolved := ResolvedSpec{Trace: make([]SpecLayer, 0, len(layers))} + for _, layer := range layers { + if err := validateSpecLayer(layer); err != nil { + return ResolvedSpec{}, err + } + resolved.Spec = resolved.Spec.Merge(layer.Spec) + if len(layer.Constraints.Models) > 0 { + resolved.Constraints.Models = intersectModels(resolved.Constraints.Models, layer.Constraints.Models) + if len(resolved.Constraints.Models) == 0 { + return ResolvedSpec{}, fmt.Errorf("spec layer %q leaves the effective model catalog empty", layer.Name) + } + } + limits, err := strictRuntimeLimits(resolved.Constraints.Limits, layer.Constraints.Limits) + if err != nil { + return ResolvedSpec{}, fmt.Errorf("spec layer %q limits: %w", layer.Name, err) + } + resolved.Constraints.Limits = limits + for _, quota := range layer.Constraints.Quotas { + quota.Name = strings.TrimSpace(quota.Name) + quota.Scope = layer.Scope + quota.Layer = layer.Name + resolved.Constraints.Quotas = append(resolved.Constraints.Quotas, quota) + } + resolved.Trace = append(resolved.Trace, cloneSpecLayer(layer)) + } + + budget, err := strictBudget(resolved.Spec.Budget, resolved.Constraints.Limits.Budget) + if err != nil { + return ResolvedSpec{}, fmt.Errorf("effective run budget: %w", err) + } + resolved.Spec.Budget = budget + if err := validateResolvedModels(resolved); err != nil { + return ResolvedSpec{}, err + } + return resolved, nil +} + +// AllowsModel reports whether a model belongs to the effective restrictive catalog. +func (resolved ResolvedSpec) AllowsModel(model Model) bool { + if len(resolved.Constraints.Models) == 0 { + return true + } + for _, allowed := range resolved.Constraints.Models { + if modelSelectorMatches(allowed, model) { + return true + } + } + return false +} + +func validateSpecLayer(layer SpecLayer) error { + if strings.TrimSpace(layer.Name) == "" { + return fmt.Errorf("spec layer name is required") + } + if scopeRank(layer.Scope) < 0 { + return fmt.Errorf("spec layer %q has invalid scope %q", layer.Name, layer.Scope) + } + if _, err := strictRuntimeLimits(RuntimeLimits{}, layer.Constraints.Limits); err != nil { + return fmt.Errorf("spec layer %q limits: %w", layer.Name, err) + } + seenModels := map[string]bool{} + for _, model := range layer.Constraints.Models { + model = strings.TrimSpace(model) + if model == "" { + return fmt.Errorf("spec layer %q model catalog contains an empty selector", layer.Name) + } + if seenModels[model] { + return fmt.Errorf("spec layer %q model catalog repeats %q", layer.Name, model) + } + seenModels[model] = true + } + seenQuotas := map[string]bool{} + for _, quota := range layer.Constraints.Quotas { + name := strings.TrimSpace(quota.Name) + if name == "" { + return fmt.Errorf("spec layer %q quota name is required", layer.Name) + } + if layer.Scope != SpecLayerGlobal && layer.Scope != SpecLayerContext { + return fmt.Errorf("spec layer %q quota %q requires global or context scope", layer.Name, name) + } + if seenQuotas[name] { + return fmt.Errorf("spec layer %q repeats quota %q", layer.Name, name) + } + seenQuotas[name] = true + if quota.TokenLimit < 0 || quota.TokensUsed < 0 || quota.CostLimitUSD < 0 || quota.CostUsedUSD < 0 { + return fmt.Errorf("spec layer %q quota %q cannot contain negative usage or limits", layer.Name, name) + } + } + return nil +} + +func scopeRank(scope SpecLayerScope) int { + switch scope { + case SpecLayerGlobal: + return 0 + case SpecLayerContext: + return 1 + case SpecLayerSurface: + return 2 + case SpecLayerUser: + return 3 + default: + return -1 + } +} + +func intersectModels(current, restrictive []string) []string { + if len(current) == 0 { + return append([]string(nil), restrictive...) + } + allowed := make(map[string]bool, len(restrictive)) + for _, model := range restrictive { + allowed[strings.TrimSpace(model)] = true + } + out := make([]string, 0, len(current)) + for _, model := range current { + if allowed[model] { + out = append(out, model) + } + } + return out +} + +func strictRuntimeLimits(current, next RuntimeLimits) (RuntimeLimits, error) { + if current.MaxInputTokens < 0 || next.MaxInputTokens < 0 { + return RuntimeLimits{}, fmt.Errorf("maxInputTokens must be non-negative") + } + budget, err := strictBudget(current.Budget, next.Budget) + if err != nil { + return RuntimeLimits{}, err + } + return RuntimeLimits{ + MaxInputTokens: strictPositiveInt(current.MaxInputTokens, next.MaxInputTokens), + Budget: budget, + }, nil +} + +func strictBudget(current, next Budget) (Budget, error) { + if err := current.Validate(); err != nil { + return Budget{}, err + } + if err := next.Validate(); err != nil { + return Budget{}, err + } + timeout, err := strictTimeout(current.Timeout, next.Timeout) + if err != nil { + return Budget{}, err + } + return Budget{ + Cost: strictPositiveFloat(current.Cost, next.Cost), + MaxTokens: strictPositiveInt(current.MaxTokens, next.MaxTokens), + MaxTurns: strictPositiveInt(current.MaxTurns, next.MaxTurns), + Timeout: timeout, + }, nil +} + +func strictPositiveInt(left, right int) int { + if left == 0 || right > 0 && right < left { + return right + } + return left +} + +func strictPositiveFloat(left, right float64) float64 { + if left == 0 || right > 0 && right < left { + return right + } + return left +} + +func strictTimeout(left, right string) (string, error) { + leftDuration, err := parseOptionalDuration(left) + if err != nil { + return "", err + } + rightDuration, err := parseOptionalDuration(right) + if err != nil { + return "", err + } + if leftDuration == 0 || rightDuration > 0 && rightDuration < leftDuration { + return strings.TrimSpace(right), nil + } + return strings.TrimSpace(left), nil +} + +func parseOptionalDuration(value string) (time.Duration, error) { + value = strings.TrimSpace(value) + if value == "" { + return 0, nil + } + duration, err := time.ParseDuration(value) + if err != nil || duration <= 0 { + return 0, fmt.Errorf("invalid positive timeout %q", value) + } + return duration, nil +} + +func validateResolvedModels(resolved ResolvedSpec) error { + model := resolved.Spec.Model + if strings.TrimSpace(model.Name) == "" { + return nil + } + if !resolved.AllowsModel(model) { + return fmt.Errorf("selected model %q is outside the effective model catalog", model.Name) + } + for _, fallback := range model.Fallbacks { + if !resolved.AllowsModel(fallback) { + return fmt.Errorf("fallback model %q is outside the effective model catalog", fallback.Name) + } + } + return nil +} + +func modelSelectorMatches(selector string, model Model) bool { + selector = strings.TrimSpace(selector) + if selector == model.Name || selector == model.ID { + return true + } + allowed, allowedErr := (Model{Name: selector}).Expand() + actual, actualErr := model.Expand() + return allowedErr == nil && actualErr == nil && allowed.Name == actual.Name && allowed.Backend == actual.Backend +} + +func cloneSpecLayer(layer SpecLayer) SpecLayer { + layer.Spec = Spec{}.Merge(layer.Spec) + layer.Constraints.Models = append([]string(nil), layer.Constraints.Models...) + layer.Constraints.Quotas = append([]RuntimeQuota(nil), layer.Constraints.Quotas...) + return layer +} diff --git a/pkg/api/spec_layers_ginkgo_test.go b/pkg/api/spec_layers_ginkgo_test.go new file mode 100644 index 00000000..93f537ad --- /dev/null +++ b/pkg/api/spec_layers_ginkgo_test.go @@ -0,0 +1,100 @@ +package api + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Hierarchical spec profiles", func() { + It("orders layers by scope and structurally overlays defaults without mutating inputs", func() { + global := SpecLayer{ + Name: "platform", Scope: SpecLayerGlobal, + Spec: Spec{Model: Model{Name: "claude-sonnet-5"}, Budget: Budget{MaxTokens: 8000}}, + } + context := SpecLayer{ + Name: "claims", Scope: SpecLayerContext, + Spec: Spec{Model: Model{Effort: EffortHigh}, Budget: Budget{MaxTurns: 6}}, + } + surface := PromptSpecLayer("triage.prompt", Spec{Prompt: Prompt{System: "Triage claims."}}) + user := SpecLayer{ + Name: "request", Scope: SpecLayerUser, + Spec: Spec{Model: Model{Effort: EffortLow}}, + } + + resolved, err := ResolveSpecLayers(user, surface, context, global) + + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Model.Name).To(Equal("claude-sonnet-5")) + Expect(resolved.Spec.Model.Effort).To(Equal(EffortLow)) + Expect(resolved.Spec.Budget).To(Equal(Budget{MaxTokens: 8000, MaxTurns: 6})) + Expect(resolved.Spec.Prompt.System).To(Equal("Triage claims.")) + Expect(resolved.Trace).To(HaveLen(4)) + Expect([]SpecLayerScope{ + resolved.Trace[0].Scope, resolved.Trace[1].Scope, resolved.Trace[2].Scope, resolved.Trace[3].Scope, + }).To(Equal([]SpecLayerScope{SpecLayerGlobal, SpecLayerContext, SpecLayerSurface, SpecLayerUser})) + Expect(context.Spec.Model.Name).To(BeEmpty()) + Expect(global.Spec.Model.Effort).To(BeEmpty()) + }) + + It("intersects restrictive model catalogs and validates every fallback", func() { + layers := []SpecLayer{ + { + Name: "platform", Scope: SpecLayerGlobal, + Constraints: RuntimeConstraints{Models: []string{"claude-sonnet-5", "gpt-5.6-sol", "gpt-5.4"}}, + }, + { + Name: "claims", Scope: SpecLayerContext, + Constraints: RuntimeConstraints{Models: []string{"gpt-5.6-sol", "claude-sonnet-5"}}, + }, + { + Name: "request", Scope: SpecLayerUser, + Spec: Spec{Model: Model{Name: "gpt-5.6-sol", Fallbacks: []Model{{Name: "claude-sonnet-5"}}}}, + }, + } + + resolved, err := ResolveSpecLayers(layers...) + + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Constraints.Models).To(Equal([]string{"claude-sonnet-5", "gpt-5.6-sol"})) + Expect(resolved.AllowsModel(Model{Name: "gpt-5.4"})).To(BeFalse()) + Expect(resolved.AllowsModel(Model{Name: "gpt-5.6-sol"})).To(BeTrue()) + + layers[2].Spec.Model.Fallbacks = []Model{{Name: "gpt-5.4"}} + _, err = ResolveSpecLayers(layers...) + Expect(err).To(MatchError(ContainSubstring(`fallback model "gpt-5.4" is outside the effective model catalog`))) + }) + + It("uses strict non-zero run ceilings and retains each named quota independently", func() { + resolved, err := ResolveSpecLayers( + SpecLayer{ + Name: "platform", Scope: SpecLayerGlobal, + Spec: Spec{Budget: Budget{Cost: 12, MaxTokens: 9000, MaxTurns: 10, Timeout: "10m"}}, + Constraints: RuntimeConstraints{ + Limits: RuntimeLimits{MaxInputTokens: 12000, Budget: Budget{Cost: 8, MaxTokens: 7000, MaxTurns: 8, Timeout: "8m"}}, + Quotas: []RuntimeQuota{{Name: "platform-monthly", TokenLimit: 1_000_000, TokensUsed: 10}}, + }, + }, + SpecLayer{ + Name: "claims", Scope: SpecLayerContext, + Spec: Spec{Budget: Budget{Cost: 10, MaxTokens: 8000, MaxTurns: 6, Timeout: "9m"}}, + Constraints: RuntimeConstraints{ + Limits: RuntimeLimits{MaxInputTokens: 4000, Budget: Budget{Cost: 5, MaxTokens: 6000, MaxTurns: 7, Timeout: "5m"}}, + Quotas: []RuntimeQuota{{Name: "claims-monthly", CostLimitUSD: 50, CostUsedUSD: 2}}, + }, + }, + ) + + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Spec.Budget).To(Equal(Budget{Cost: 5, MaxTokens: 6000, MaxTurns: 6, Timeout: "5m"})) + Expect(resolved.Constraints.Limits.MaxInputTokens).To(Equal(4000)) + Expect(resolved.Constraints.Quotas).To(Equal([]RuntimeQuota{ + {Name: "platform-monthly", Scope: SpecLayerGlobal, Layer: "platform", TokenLimit: 1_000_000, TokensUsed: 10}, + {Name: "claims-monthly", Scope: SpecLayerContext, Layer: "claims", CostLimitUSD: 50, CostUsedUSD: 2}, + })) + duration, err := time.ParseDuration(resolved.Spec.Budget.Timeout) + Expect(err).NotTo(HaveOccurred()) + Expect(duration).To(Equal(5 * time.Minute)) + }) +}) diff --git a/pkg/cli/serve_chat.go b/pkg/cli/serve_chat.go index 84512c9b..72b6252a 100644 --- a/pkg/cli/serve_chat.go +++ b/pkg/cli/serve_chat.go @@ -37,14 +37,21 @@ func newCaptainChatService( return nil, nil, err } chat := aichat.NewService(aichat.ServiceOptions{ - Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) { - return aichat.RuntimeSettings{ - System: "You are Captain's coding-agent launcher assistant. Use Captain and Clicky tools when useful, " + - "prefer read-only inspection unless the user explicitly asks for edits, and keep follow-up guidance concise.", + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + resolved, err := api.ResolveSpecLayers(api.SpecLayer{ + Name: "captain serve", Scope: api.SpecLayerGlobal, Spec: api.Spec{ Model: api.Model{Name: "sol", Mode: registry.ModeAgent}, Setup: &shell.Setup{Cwd: cwd}, }, + }) + if err != nil { + return aichat.RuntimeProfile{}, err + } + return aichat.RuntimeProfile{ + System: "You are Captain's coding-agent launcher assistant. Use Captain and Clicky tools when useful, " + + "prefer read-only inspection unless the user explicitly asks for edits, and keep follow-up guidance concise.", + Resolved: resolved, }, nil }), // Thread reads follow the request's database context; writes never reach From 4abe4a14a7293a47f2f51db2808aec3c3776ebcc Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 12 Aug 2026 06:59:23 +0000 Subject: [PATCH 2/3] fix(aichat): revalidate profile on approval resume Amp-Thread-ID: https://ampcode.com/threads/T-019ff459-21d4-70a1-b90e-00557d59a994 --- pkg/aichat/approval_execution.go | 21 ++++++++ pkg/aichat/approval_execution_test.go | 49 ++++++++++++++++++ pkg/aichat/execution_authority_ginkgo_test.go | 50 +++++++++++++++++-- pkg/aichat/runtime_settings.go | 33 +++++++----- 4 files changed, 135 insertions(+), 18 deletions(-) diff --git a/pkg/aichat/approval_execution.go b/pkg/aichat/approval_execution.go index d54e01f8..bfcde8bd 100644 --- a/pkg/aichat/approval_execution.go +++ b/pkg/aichat/approval_execution.go @@ -55,6 +55,12 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti if err != nil { return fmt.Errorf("load chat runtime profile: %w", err) } + if err := enforceApprovalRuntimeProfile(continuation.Spec, profile.Resolved); err != nil { + if interruptErr := execution.Interrupt(ctx, err.Error()); interruptErr != nil { + return fmt.Errorf("%w (interrupt rejected approval continuation: %v)", err, interruptErr) + } + return err + } set, err := s.loadTools(ctx) if err != nil { return err @@ -123,6 +129,21 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti return nil } +func enforceApprovalRuntimeProfile(spec api.Spec, resolved api.ResolvedSpec) error { + if err := enforceRuntimeQuotas(resolved); err != nil { + return err + } + if !resolved.AllowsModel(spec.Model) { + return fmt.Errorf("approval continuation model %q is outside the current effective model catalog", spec.Model.Name) + } + for _, fallback := range spec.Model.Fallbacks { + if !resolved.AllowsModel(fallback) { + return fmt.Errorf("approval continuation fallback model %q is outside the current effective model catalog", fallback.Name) + } + } + return nil +} + // suspendedSeedWait bounds how long an approval resolution waits for the // suspending turn's assistant message to land in the thread store. const suspendedSeedWait = 5 * time.Second diff --git a/pkg/aichat/approval_execution_test.go b/pkg/aichat/approval_execution_test.go index 590c8279..6afc046c 100644 --- a/pkg/aichat/approval_execution_test.go +++ b/pkg/aichat/approval_execution_test.go @@ -2,10 +2,59 @@ package aichat import ( "context" + "strings" "testing" "time" + + "github.com/flanksource/captain/pkg/api" ) +func TestEnforceApprovalRuntimeProfile(t *testing.T) { + tests := []struct { + name string + spec api.Spec + resolved api.ResolvedSpec + wantErr string + }{ + { + name: "persisted model is no longer allowed", + spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}}, + resolved: api.ResolvedSpec{Constraints: api.RuntimeConstraints{Models: []string{"claude-sonnet-5"}}}, + wantErr: `model "gpt-5.6-sol" is outside the current effective model catalog`, + }, + { + name: "current quota is exhausted", + spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}}, + resolved: api.ResolvedSpec{Constraints: api.RuntimeConstraints{Quotas: []api.RuntimeQuota{{ + Name: "monthly", Scope: api.SpecLayerUser, Layer: "claims", TokenLimit: 100, TokensUsed: 100, + }}}}, + wantErr: `quota "monthly" from layer "claims" exhausted`, + }, + { + name: "changed default does not replace an allowed persisted model", + spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}}, + resolved: api.ResolvedSpec{ + Spec: api.Spec{Model: api.Model{Name: "claude-sonnet-5"}}, + Constraints: api.RuntimeConstraints{Models: []string{"gpt-5.6-sol", "claude-sonnet-5"}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := enforceApprovalRuntimeProfile(tt.spec, tt.resolved) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("enforceApprovalRuntimeProfile() error = %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("enforceApprovalRuntimeProfile() error = %v, want containing %q", err, tt.wantErr) + } + }) + } +} + func TestAwaitSuspendedSeedWaitsForTheInFlightAssistantMessage(t *testing.T) { store := NewMemoryThreadStore() thread, err := store.Create(context.Background(), "Approve") diff --git a/pkg/aichat/execution_authority_ginkgo_test.go b/pkg/aichat/execution_authority_ginkgo_test.go index 62b36401..cfb2271f 100644 --- a/pkg/aichat/execution_authority_ginkgo_test.go +++ b/pkg/aichat/execution_authority_ginkgo_test.go @@ -16,10 +16,11 @@ import ( ) type fakeExecutionAuthority struct { - execution *fakeExecution - beginErr error - begins []aichat.ExecutionRequest - resolutions []aichat.ToolApprovalResolution + execution *fakeExecution + beginErr error + begins []aichat.ExecutionRequest + resolutions []aichat.ToolApprovalResolution + continuation *aichat.ApprovalContinuation } func (f *fakeExecutionAuthority) Begin(_ context.Context, request aichat.ExecutionRequest) (aichat.Execution, error) { @@ -36,7 +37,7 @@ func (f *fakeExecutionAuthority) ResolveToolApproval( resolution aichat.ToolApprovalResolution, ) (*aichat.ApprovalContinuation, error) { f.resolutions = append(f.resolutions, resolution) - return nil, nil + return f.continuation, nil } type fakeExecution struct { @@ -412,4 +413,43 @@ var _ = Describe("Authoritative aichat execution", func() { Expect(missing.Code).To(Equal(http.StatusNotFound)) Expect(authority.resolutions).To(HaveLen(1)) }) + + It("interrupts an approval continuation whose persisted model is no longer allowed", func() { + store := aichat.NewMemoryThreadStore() + thread, err := store.Create(context.Background(), "Restricted") + Expect(err).NotTo(HaveOccurred()) + execution := &fakeExecution{} + authority := &fakeExecutionAuthority{continuation: &aichat.ApprovalContinuation{ + Execution: execution, + Spec: api.Spec{ + Model: api.Model{Name: "gpt-5.6-sol"}, + ToolApproval: &api.ToolApprovalResume{}, + }, + }} + resolver := &fakeResolver{provider: &fakeStreamingProvider{}} + profile := mustRuntimeProfile(api.SpecLayer{ + Name: "claims", Scope: api.SpecLayerContext, + Spec: api.Spec{Model: api.Model{Name: "claude-sonnet-5"}}, + Constraints: api.RuntimeConstraints{Models: []string{"claude-sonnet-5"}}, + }) + service := aichat.NewService(aichat.ServiceOptions{ + Threads: aichat.FixedThreadStore(store), Authority: authority, Resolver: resolver, + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return profile, nil + }), + }) + + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, requestJSON( + http.MethodPost, + "/api/chat/sessions/"+thread.ID+"/approvals/0e5dc2fe-8b77-44e9-a3de-6a00298c8bde", + map[string]any{"approved": true}, + )) + + Expect(response.Code).To(Equal(http.StatusBadGateway)) + Expect(response.Body.String()).To(ContainSubstring(`model "gpt-5.6-sol" is outside the current effective model catalog`)) + Expect(execution.interrupts).To(ConsistOf(ContainSubstring("outside the current effective model catalog"))) + Expect(execution.closed).To(BeTrue()) + Expect(resolver.configs).To(BeEmpty()) + }) }) diff --git a/pkg/aichat/runtime_settings.go b/pkg/aichat/runtime_settings.go index 5229f2b5..4888fe62 100644 --- a/pkg/aichat/runtime_settings.go +++ b/pkg/aichat/runtime_settings.go @@ -23,19 +23,8 @@ func requestErrorStatus(err error) int { } func enforceRuntimeProfile(request ChatRequest, resolved api.ResolvedSpec) error { - for _, quota := range resolved.Constraints.Quotas { - if quota.CostLimitUSD > 0 && quota.CostUsedUSD >= quota.CostLimitUSD { - return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( - "chat %s quota %q from layer %q exhausted: $%.4f used of $%.4f", - quota.Scope, quota.Name, quota.Layer, quota.CostUsedUSD, quota.CostLimitUSD, - )} - } - if quota.TokenLimit > 0 && quota.TokensUsed >= quota.TokenLimit { - return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( - "chat %s quota %q from layer %q exhausted: %d tokens used of %d", - quota.Scope, quota.Name, quota.Layer, quota.TokensUsed, quota.TokenLimit, - )} - } + if err := enforceRuntimeQuotas(resolved); err != nil { + return err } maxInputTokens := resolved.Constraints.Limits.MaxInputTokens if maxInputTokens <= 0 { @@ -58,3 +47,21 @@ func enforceRuntimeProfile(request ChatRequest, resolved api.ResolvedSpec) error } return nil } + +func enforceRuntimeQuotas(resolved api.ResolvedSpec) error { + for _, quota := range resolved.Constraints.Quotas { + if quota.CostLimitUSD > 0 && quota.CostUsedUSD >= quota.CostLimitUSD { + return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( + "chat %s quota %q from layer %q exhausted: $%.4f used of $%.4f", + quota.Scope, quota.Name, quota.Layer, quota.CostUsedUSD, quota.CostLimitUSD, + )} + } + if quota.TokenLimit > 0 && quota.TokensUsed >= quota.TokenLimit { + return requestError{status: http.StatusPaymentRequired, text: fmt.Sprintf( + "chat %s quota %q from layer %q exhausted: %d tokens used of %d", + quota.Scope, quota.Name, quota.Layer, quota.TokensUsed, quota.TokenLimit, + )} + } + } + return nil +} From 404080cd9cb681573018221d7e22b4691ca7ab72 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 12 Aug 2026 12:48:40 +0545 Subject: [PATCH 3/3] fix(aichat): harden runtime profile constraints Bare model selectors could disable every runtime, and whitespace-only selector differences could empty an otherwise valid catalog. Server-owned profile resolution failures were also reported as client errors. Match runtime constraints against concrete registry models, normalize catalog intersections, validate profiles before layering request fields, and clarify per-run limit and accumulated quota names. --- pkg/aichat/approval_execution.go | 4 +- pkg/aichat/approval_execution_test.go | 2 +- pkg/aichat/messages.go | 3 - pkg/aichat/provider_config.go | 26 +++++- pkg/aichat/runtime_profile_ginkgo_test.go | 98 ++++++++++++++++++++++- pkg/aichat/service.go | 19 ++++- pkg/aichat/service_ginkgo_test.go | 2 +- pkg/api/spec_layers.go | 35 ++++---- pkg/api/spec_layers_ginkgo_test.go | 26 ++++-- 9 files changed, 182 insertions(+), 33 deletions(-) diff --git a/pkg/aichat/approval_execution.go b/pkg/aichat/approval_execution.go index bfcde8bd..a2fde9a5 100644 --- a/pkg/aichat/approval_execution.go +++ b/pkg/aichat/approval_execution.go @@ -134,9 +134,9 @@ func enforceApprovalRuntimeProfile(spec api.Spec, resolved api.ResolvedSpec) err return err } if !resolved.AllowsModel(spec.Model) { - return fmt.Errorf("approval continuation model %q is outside the current effective model catalog", spec.Model.Name) + return fmt.Errorf("approval continuation model %q is outside the current effective model catalog", spec.Name) } - for _, fallback := range spec.Model.Fallbacks { + for _, fallback := range spec.Fallbacks { if !resolved.AllowsModel(fallback) { return fmt.Errorf("approval continuation fallback model %q is outside the current effective model catalog", fallback.Name) } diff --git a/pkg/aichat/approval_execution_test.go b/pkg/aichat/approval_execution_test.go index 6afc046c..35346577 100644 --- a/pkg/aichat/approval_execution_test.go +++ b/pkg/aichat/approval_execution_test.go @@ -25,7 +25,7 @@ func TestEnforceApprovalRuntimeProfile(t *testing.T) { { name: "current quota is exhausted", spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}}, - resolved: api.ResolvedSpec{Constraints: api.RuntimeConstraints{Quotas: []api.RuntimeQuota{{ + resolved: api.ResolvedSpec{Constraints: api.RuntimeConstraints{Quotas: []api.UsageQuota{{ Name: "monthly", Scope: api.SpecLayerUser, Layer: "claims", TokenLimit: 100, TokensUsed: 100, }}}}, wantErr: `quota "monthly" from layer "claims" exhausted`, diff --git a/pkg/aichat/messages.go b/pkg/aichat/messages.go index 3529ecb3..7a9640ad 100644 --- a/pkg/aichat/messages.go +++ b/pkg/aichat/messages.go @@ -69,9 +69,6 @@ func (s *Service) resolveAttachments(ctx context.Context, messages []UIMessage) } func requestSpec(request ChatRequest, profile RuntimeProfile, attachments map[partLocation]api.AttachmentRef) (api.ResolvedSpec, error) { - if len(profile.Resolved.Trace) == 0 && (!api.IsEmpty(profile.Resolved.Spec) || !api.IsEmpty(profile.Resolved.Constraints)) { - return api.ResolvedSpec{}, fmt.Errorf("chat runtime profile must include its resolution trace") - } override, err := chatModel(request, profile.Resolved.Spec.Model) if err != nil { return api.ResolvedSpec{}, err diff --git a/pkg/aichat/provider_config.go b/pkg/aichat/provider_config.go index 3547e2fb..219daed5 100644 --- a/pkg/aichat/provider_config.go +++ b/pkg/aichat/provider_config.go @@ -8,6 +8,7 @@ import ( "github.com/flanksource/captain/pkg/ai" "github.com/flanksource/captain/pkg/api" + "github.com/flanksource/captain/pkg/api/registry" ) func annotateProfileModels(resolved api.ResolvedSpec, models ModelCatalogResponse) { @@ -78,14 +79,33 @@ func runtimeRestrictionLayer(resolved api.ResolvedSpec, backend api.Backend) *ap return nil } +// runtimeAllowed checks concrete registry rows so bare names follow the same +// matching rules as the model catalog instead of requiring a provider prefix. func runtimeAllowed(models []string, backend api.Backend) bool { providerPrefix := ai.BackendToProvider(backend) + "/" for _, selector := range models { - if strings.HasPrefix(selector, providerPrefix) { + if strings.HasPrefix(strings.TrimSpace(selector), providerPrefix) { return true } - model, err := (api.Model{Name: selector}).Expand() - if err == nil && model.Backend == backend { + } + + provider, mode, ok := registry.ProviderFor(backend) + if !ok { + return false + } + resolved := api.ResolvedSpec{Constraints: api.RuntimeConstraints{Models: models}} + for _, model := range provider.Models() { + if !model.Preferred { + continue + } + if known, available := provider.Availability(mode, model.ID); !known || !available { + continue + } + candidate := api.Model{Name: model.ID, Backend: backend} + if mode == registry.ModeAPI { + candidate.ID = provider.CatalogPrefix + "/" + model.ID + } + if resolved.AllowsModel(candidate) { return true } } diff --git a/pkg/aichat/runtime_profile_ginkgo_test.go b/pkg/aichat/runtime_profile_ginkgo_test.go index fecc247d..cf683952 100644 --- a/pkg/aichat/runtime_profile_ginkgo_test.go +++ b/pkg/aichat/runtime_profile_ginkgo_test.go @@ -42,17 +42,111 @@ var _ = Describe("Resolved runtime profiles", func() { Expect(models[1].Availability.Remediation).To(ContainSubstring("allowed models")) }) + It("keeps runtimes enabled when a bare selector admits one of their models", func() { + resolver := &fakeResolver{runtimes: []api.RuntimeFamily{ + { + Family: "codex", Provider: "openai", CatalogPrefix: "openai", + Modes: []api.RuntimeModeEntry{{Mode: "api", Backend: string(api.BackendOpenAI), Availability: api.Available()}}, + }, + { + Family: "claude", Provider: "anthropic", CatalogPrefix: "anthropic", + Modes: []api.RuntimeModeEntry{{Mode: "api", Backend: string(api.BackendAnthropic), Availability: api.Available()}}, + }, + }} + profile := mustRuntimeProfile(api.SpecLayer{ + Name: "claims", Scope: api.SpecLayerContext, + Constraints: api.RuntimeConstraints{Models: []string{"gpt-5.6-sol"}}, + }) + service := aichat.NewService(aichat.ServiceOptions{ + Resolver: resolver, + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return profile, nil + }), + }) + + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/chat/runtimes", nil)) + + Expect(response.Code).To(Equal(http.StatusOK)) + var runtimes []api.RuntimeFamily + Expect(json.Unmarshal(response.Body.Bytes(), &runtimes)).To(Succeed()) + Expect(runtimes[0].Modes[0].Disabled).To(BeFalse()) + Expect(runtimes[0].Modes[0].Availability).To(Equal(api.Available())) + Expect(runtimes[1].Modes[0].Disabled).To(BeTrue()) + Expect(runtimes[1].Modes[0].Availability.State).To(Equal(api.AvailabilityDisabled)) + }) + + It("reports malformed server profiles as internal errors", func() { + cases := []struct { + name string + resolved api.ResolvedSpec + message string + }{ + { + name: "missing trace", + resolved: api.ResolvedSpec{Spec: api.Spec{ + Model: api.Model{Name: "gpt-5.4"}, + }}, + message: "must include its resolution trace", + }, + { + name: "invalid trace", + resolved: api.ResolvedSpec{Trace: []api.SpecLayer{{ + Name: "broken", Scope: api.SpecLayerScope("invalid"), + }}}, + message: "invalid scope", + }, + } + + for _, test := range cases { + service := aichat.NewService(aichat.ServiceOptions{ + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return aichat.RuntimeProfile{Resolved: test.resolved}, nil + }), + }) + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, requestJSON(http.MethodPost, "/api/chat", aichat.ChatRequest{ + Messages: []aichat.UIMessage{{Role: "user", Parts: []aichat.UIPart{{Type: "text", Text: "hello"}}}}, + })) + + Expect(response.Code).To(Equal(http.StatusInternalServerError), test.name) + Expect(response.Body.String()).To(ContainSubstring(test.message), test.name) + } + }) + + It("keeps request model violations as bad requests", func() { + profile := mustRuntimeProfile(api.SpecLayer{ + Name: "claims", Scope: api.SpecLayerContext, + Spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}}, + Constraints: api.RuntimeConstraints{Models: []string{"gpt-5.6-sol"}}, + }) + service := aichat.NewService(aichat.ServiceOptions{ + Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) { + return profile, nil + }), + }) + + response := httptest.NewRecorder() + service.Handler().ServeHTTP(response, requestJSON(http.MethodPost, "/api/chat", aichat.ChatRequest{ + Model: "claude-sonnet-5", + Messages: []aichat.UIMessage{{Role: "user", Parts: []aichat.UIPart{{Type: "text", Text: "hello"}}}}, + })) + + Expect(response.Code).To(Equal(http.StatusBadRequest)) + Expect(response.Body.String()).To(ContainSubstring("outside the effective model catalog")) + }) + It("checks every named quota and reports the independently exhausted allowance", func() { resolver := &fakeResolver{provider: &fakeStreamingProvider{}} profile := mustRuntimeProfile( api.SpecLayer{ Name: "platform", Scope: api.SpecLayerGlobal, Spec: api.Spec{Model: api.Model{Name: "openai/test-model"}}, - Constraints: api.RuntimeConstraints{Quotas: []api.RuntimeQuota{{Name: "platform-monthly", TokenLimit: 100, TokensUsed: 10}}}, + Constraints: api.RuntimeConstraints{Quotas: []api.UsageQuota{{Name: "platform-monthly", TokenLimit: 100, TokensUsed: 10}}}, }, api.SpecLayer{ Name: "claims", Scope: api.SpecLayerContext, - Constraints: api.RuntimeConstraints{Quotas: []api.RuntimeQuota{{Name: "claims-monthly", CostLimitUSD: 20, CostUsedUSD: 20}}}, + Constraints: api.RuntimeConstraints{Quotas: []api.UsageQuota{{Name: "claims-monthly", CostLimitUSD: 20, CostUsedUSD: 20}}}, }, ) service := aichat.NewService(aichat.ServiceOptions{ diff --git a/pkg/aichat/service.go b/pkg/aichat/service.go index 118342e8..8f803492 100644 --- a/pkg/aichat/service.go +++ b/pkg/aichat/service.go @@ -372,11 +372,28 @@ func validateThreadTurn(request ChatRequest, thread *Thread) error { return nil } +// runtimeProfile validates the server-owned profile before request fields are +// layered onto it, so profile defects remain server errors at the HTTP boundary. func (s *Service) runtimeProfile(ctx context.Context) (RuntimeProfile, error) { if s.options.Profile == nil { return RuntimeProfile{}, nil } - return s.options.Profile.RuntimeProfile(ctx) + profile, err := s.options.Profile.RuntimeProfile(ctx) + if err != nil { + return RuntimeProfile{}, err + } + if len(profile.Resolved.Trace) == 0 { + if !api.IsEmpty(profile.Resolved.Spec) || !api.IsEmpty(profile.Resolved.Constraints) { + return RuntimeProfile{}, fmt.Errorf("chat runtime profile must include its resolution trace") + } + return profile, nil + } + resolved, err := api.ResolveSpecLayers(profile.Resolved.Trace...) + if err != nil { + return RuntimeProfile{}, fmt.Errorf("resolve chat runtime profile: %w", err) + } + profile.Resolved = resolved + return profile, nil } func (s *Service) resolveThreadSession(ctx context.Context, request *ChatRequest) (*Thread, error) { diff --git a/pkg/aichat/service_ginkgo_test.go b/pkg/aichat/service_ginkgo_test.go index a7c00ae2..170ba909 100644 --- a/pkg/aichat/service_ginkgo_test.go +++ b/pkg/aichat/service_ginkgo_test.go @@ -228,7 +228,7 @@ var _ = Describe("Captain aichat service", func() { return mustRuntimeProfile(api.SpecLayer{ Name: "application", Scope: api.SpecLayerGlobal, Spec: api.Spec{Model: api.Model{Name: "openai/test-model"}}, - Constraints: api.RuntimeConstraints{Quotas: []api.RuntimeQuota{{ + Constraints: api.RuntimeConstraints{Quotas: []api.UsageQuota{{ Name: "application-monthly", CostLimitUSD: 10, CostUsedUSD: 10, }}}, }), nil diff --git a/pkg/api/spec_layers.go b/pkg/api/spec_layers.go index 3320ffce..b3ee18cf 100644 --- a/pkg/api/spec_layers.go +++ b/pkg/api/spec_layers.go @@ -17,14 +17,14 @@ const ( SpecLayerUser SpecLayerScope = "user" ) -// RuntimeLimits are ceilings applied after structural Spec defaults are layered. -type RuntimeLimits struct { +// RunLimits are per-run ceilings applied after structural Spec defaults are layered. +type RunLimits struct { MaxInputTokens int `json:"maxInputTokens,omitempty" yaml:"maxInputTokens,omitempty"` Budget Budget `json:"budget,omitempty" yaml:"budget,omitempty"` } -// RuntimeQuota is one independently enforced usage allowance. -type RuntimeQuota struct { +// UsageQuota is one independently enforced accumulated usage allowance. +type UsageQuota struct { Name string `json:"name" yaml:"name"` Scope SpecLayerScope `json:"scope" yaml:"scope"` Layer string `json:"layer" yaml:"layer"` @@ -36,9 +36,9 @@ type RuntimeQuota struct { // RuntimeConstraints restrict values a later Spec layer may select. type RuntimeConstraints struct { - Models []string `json:"models,omitempty" yaml:"models,omitempty"` - Limits RuntimeLimits `json:"limits,omitempty" yaml:"limits,omitempty"` - Quotas []RuntimeQuota `json:"quotas,omitempty" yaml:"quotas,omitempty"` + Models []string `json:"models,omitempty" yaml:"models,omitempty"` + Limits RunLimits `json:"limits,omitempty" yaml:"limits,omitempty"` + Quotas []UsageQuota `json:"quotas,omitempty" yaml:"quotas,omitempty"` } // SpecLayer is one named source of runtime defaults and constraints. @@ -80,7 +80,7 @@ func ResolveSpecLayers(input ...SpecLayer) (ResolvedSpec, error) { return ResolvedSpec{}, fmt.Errorf("spec layer %q leaves the effective model catalog empty", layer.Name) } } - limits, err := strictRuntimeLimits(resolved.Constraints.Limits, layer.Constraints.Limits) + limits, err := strictRunLimits(resolved.Constraints.Limits, layer.Constraints.Limits) if err != nil { return ResolvedSpec{}, fmt.Errorf("spec layer %q limits: %w", layer.Name, err) } @@ -125,7 +125,7 @@ func validateSpecLayer(layer SpecLayer) error { if scopeRank(layer.Scope) < 0 { return fmt.Errorf("spec layer %q has invalid scope %q", layer.Name, layer.Scope) } - if _, err := strictRuntimeLimits(RuntimeLimits{}, layer.Constraints.Limits); err != nil { + if _, err := strictRunLimits(RunLimits{}, layer.Constraints.Limits); err != nil { return fmt.Errorf("spec layer %q limits: %w", layer.Name, err) } seenModels := map[string]bool{} @@ -176,7 +176,11 @@ func scopeRank(scope SpecLayerScope) int { func intersectModels(current, restrictive []string) []string { if len(current) == 0 { - return append([]string(nil), restrictive...) + out := make([]string, 0, len(restrictive)) + for _, model := range restrictive { + out = append(out, strings.TrimSpace(model)) + } + return out } allowed := make(map[string]bool, len(restrictive)) for _, model := range restrictive { @@ -184,6 +188,7 @@ func intersectModels(current, restrictive []string) []string { } out := make([]string, 0, len(current)) for _, model := range current { + model = strings.TrimSpace(model) if allowed[model] { out = append(out, model) } @@ -191,15 +196,15 @@ func intersectModels(current, restrictive []string) []string { return out } -func strictRuntimeLimits(current, next RuntimeLimits) (RuntimeLimits, error) { +func strictRunLimits(current, next RunLimits) (RunLimits, error) { if current.MaxInputTokens < 0 || next.MaxInputTokens < 0 { - return RuntimeLimits{}, fmt.Errorf("maxInputTokens must be non-negative") + return RunLimits{}, fmt.Errorf("maxInputTokens must be non-negative") } budget, err := strictBudget(current.Budget, next.Budget) if err != nil { - return RuntimeLimits{}, err + return RunLimits{}, err } - return RuntimeLimits{ + return RunLimits{ MaxInputTokens: strictPositiveInt(current.MaxInputTokens, next.MaxInputTokens), Budget: budget, }, nil @@ -294,6 +299,6 @@ func modelSelectorMatches(selector string, model Model) bool { func cloneSpecLayer(layer SpecLayer) SpecLayer { layer.Spec = Spec{}.Merge(layer.Spec) layer.Constraints.Models = append([]string(nil), layer.Constraints.Models...) - layer.Constraints.Quotas = append([]RuntimeQuota(nil), layer.Constraints.Quotas...) + layer.Constraints.Quotas = append([]UsageQuota(nil), layer.Constraints.Quotas...) return layer } diff --git a/pkg/api/spec_layers_ginkgo_test.go b/pkg/api/spec_layers_ginkgo_test.go index 93f537ad..abf35f78 100644 --- a/pkg/api/spec_layers_ginkgo_test.go +++ b/pkg/api/spec_layers_ginkgo_test.go @@ -66,22 +66,38 @@ var _ = Describe("Hierarchical spec profiles", func() { Expect(err).To(MatchError(ContainSubstring(`fallback model "gpt-5.4" is outside the effective model catalog`))) }) + It("normalizes model selectors before intersecting catalogs", func() { + resolved, err := ResolveSpecLayers( + SpecLayer{ + Name: "platform", Scope: SpecLayerGlobal, + Constraints: RuntimeConstraints{Models: []string{" gpt-5.4 "}}, + }, + SpecLayer{ + Name: "claims", Scope: SpecLayerContext, + Constraints: RuntimeConstraints{Models: []string{"gpt-5.4"}}, + }, + ) + + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Constraints.Models).To(Equal([]string{"gpt-5.4"})) + }) + It("uses strict non-zero run ceilings and retains each named quota independently", func() { resolved, err := ResolveSpecLayers( SpecLayer{ Name: "platform", Scope: SpecLayerGlobal, Spec: Spec{Budget: Budget{Cost: 12, MaxTokens: 9000, MaxTurns: 10, Timeout: "10m"}}, Constraints: RuntimeConstraints{ - Limits: RuntimeLimits{MaxInputTokens: 12000, Budget: Budget{Cost: 8, MaxTokens: 7000, MaxTurns: 8, Timeout: "8m"}}, - Quotas: []RuntimeQuota{{Name: "platform-monthly", TokenLimit: 1_000_000, TokensUsed: 10}}, + Limits: RunLimits{MaxInputTokens: 12000, Budget: Budget{Cost: 8, MaxTokens: 7000, MaxTurns: 8, Timeout: "8m"}}, + Quotas: []UsageQuota{{Name: "platform-monthly", TokenLimit: 1_000_000, TokensUsed: 10}}, }, }, SpecLayer{ Name: "claims", Scope: SpecLayerContext, Spec: Spec{Budget: Budget{Cost: 10, MaxTokens: 8000, MaxTurns: 6, Timeout: "9m"}}, Constraints: RuntimeConstraints{ - Limits: RuntimeLimits{MaxInputTokens: 4000, Budget: Budget{Cost: 5, MaxTokens: 6000, MaxTurns: 7, Timeout: "5m"}}, - Quotas: []RuntimeQuota{{Name: "claims-monthly", CostLimitUSD: 50, CostUsedUSD: 2}}, + Limits: RunLimits{MaxInputTokens: 4000, Budget: Budget{Cost: 5, MaxTokens: 6000, MaxTurns: 7, Timeout: "5m"}}, + Quotas: []UsageQuota{{Name: "claims-monthly", CostLimitUSD: 50, CostUsedUSD: 2}}, }, }, ) @@ -89,7 +105,7 @@ var _ = Describe("Hierarchical spec profiles", func() { Expect(err).NotTo(HaveOccurred()) Expect(resolved.Spec.Budget).To(Equal(Budget{Cost: 5, MaxTokens: 6000, MaxTurns: 6, Timeout: "5m"})) Expect(resolved.Constraints.Limits.MaxInputTokens).To(Equal(4000)) - Expect(resolved.Constraints.Quotas).To(Equal([]RuntimeQuota{ + Expect(resolved.Constraints.Quotas).To(Equal([]UsageQuota{ {Name: "platform-monthly", Scope: SpecLayerGlobal, Layer: "platform", TokenLimit: 1_000_000, TokensUsed: 10}, {Name: "claims-monthly", Scope: SpecLayerContext, Layer: "claims", CostLimitUSD: 50, CostUsedUSD: 2}, }))