Resolve hierarchical runtime profiles

pkg/aichat/aimock_lifecycle_integration_test.go

@@ -132,8 +132,8 @@ var _ = Describe("Mocked Captain chat lifecycle", func() {
 			var approvedInput map[string]any
 			service := aichat.NewService(aichat.ServiceOptions{
 				Resolver: realChatResolver{}, Threads: aichat.FixedThreadStore(store), Authority: authority,
-				Settings: aichat.RuntimeSettingsProviderFunc(func(context.Context) (aichat.RuntimeSettings, error) {
-					return aichat.RuntimeSettings{ProviderConfig: api.Config{
+				Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) {
+					return aichat.RuntimeProfile{ProviderConfig: api.Config{
 						APIURL: mock.apiURL, APIKey: aimock.DummyKey,
 					}}, nil
 				}),

pkg/aichat/approval_execution.go

@@ -51,9 +51,15 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti
 	}
 	execution := continuation.Execution
 	defer closeExecution(execution)
-	settings, err := s.runtimeSettings(ctx)
+	profile, err := s.runtimeProfile(ctx)
 	if err != nil {
-		return fmt.Errorf("load chat runtime settings: %w", err)
+		return fmt.Errorf("load chat runtime profile: %w", err)
+	}
+	if err := enforceApprovalRuntimeProfile(continuation.Spec, profile.Resolved); err != nil {
+		if interruptErr := execution.Interrupt(ctx, err.Error()); interruptErr != nil {
+			return fmt.Errorf("%w (interrupt rejected approval continuation: %v)", err, interruptErr)
+		}
+		return err
 	}
 	set, err := s.loadTools(ctx)
 	if err != nil {
@@ -63,7 +69,7 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti
 	if err != nil {
 		return err
 	}
-	config := settings.ProviderConfig
+	config := profile.ProviderConfig
 	config.Model = continuation.Spec.Model
 	config.Budget = continuation.Spec.Budget
 	config.SessionID = continuation.Spec.SessionID
@@ -123,6 +129,21 @@ func (s *Service) resumeToolApproval(ctx context.Context, threadID string, conti
 	return nil
 }
 
+func enforceApprovalRuntimeProfile(spec api.Spec, resolved api.ResolvedSpec) error {
+	if err := enforceRuntimeQuotas(resolved); err != nil {
+		return err
+	}
+	if !resolved.AllowsModel(spec.Model) {
+		return fmt.Errorf("approval continuation model %q is outside the current effective model catalog", spec.Name)
+	}
+	for _, fallback := range spec.Fallbacks {
+		if !resolved.AllowsModel(fallback) {
+			return fmt.Errorf("approval continuation fallback model %q is outside the current effective model catalog", fallback.Name)
+		}
+	}
+	return nil
+}
+
 // suspendedSeedWait bounds how long an approval resolution waits for the
 // suspending turn's assistant message to land in the thread store.
 const suspendedSeedWait = 5 * time.Second

pkg/aichat/approval_execution_test.go

@@ -2,10 +2,59 @@ package aichat
 
 import (
 	"context"
+	"strings"
 	"testing"
 	"time"
+
+	"github.com/flanksource/captain/pkg/api"
 )
 
+func TestEnforceApprovalRuntimeProfile(t *testing.T) {
+	tests := []struct {
+		name     string
+		spec     api.Spec
+		resolved api.ResolvedSpec
+		wantErr  string
+	}{
+		{
+			name:     "persisted model is no longer allowed",
+			spec:     api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}},
+			resolved: api.ResolvedSpec{Constraints: api.RuntimeConstraints{Models: []string{"claude-sonnet-5"}}},
+			wantErr:  `model "gpt-5.6-sol" is outside the current effective model catalog`,
+		},
+		{
+			name: "current quota is exhausted",
+			spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}},
+			resolved: api.ResolvedSpec{Constraints: api.RuntimeConstraints{Quotas: []api.UsageQuota{{
+				Name: "monthly", Scope: api.SpecLayerUser, Layer: "claims", TokenLimit: 100, TokensUsed: 100,
+			}}}},
+			wantErr: `quota "monthly" from layer "claims" exhausted`,
+		},
+		{
+			name: "changed default does not replace an allowed persisted model",
+			spec: api.Spec{Model: api.Model{Name: "gpt-5.6-sol"}},
+			resolved: api.ResolvedSpec{
+				Spec:        api.Spec{Model: api.Model{Name: "claude-sonnet-5"}},
+				Constraints: api.RuntimeConstraints{Models: []string{"gpt-5.6-sol", "claude-sonnet-5"}},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := enforceApprovalRuntimeProfile(tt.spec, tt.resolved)
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("enforceApprovalRuntimeProfile() error = %v", err)
+				}
+				return
+			}
+			if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+				t.Fatalf("enforceApprovalRuntimeProfile() error = %v, want containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func TestAwaitSuspendedSeedWaitsForTheInFlightAssistantMessage(t *testing.T) {
 	store := NewMemoryThreadStore()
 	thread, err := store.Create(context.Background(), "Approve")

pkg/aichat/execution.go

@@ -15,6 +15,7 @@ type ExecutionRequest struct {
 	RequestID   string
 	Title       string
 	Spec        api.Spec
+	Profile     api.ResolvedSpec
 	Definitions []api.ToolDefinition
 }
 

pkg/aichat/execution_authority_ginkgo_test.go

@@ -16,10 +16,11 @@ import (
 )
 
 type fakeExecutionAuthority struct {
-	execution   *fakeExecution
-	beginErr    error
-	begins      []aichat.ExecutionRequest
-	resolutions []aichat.ToolApprovalResolution
+	execution    *fakeExecution
+	beginErr     error
+	begins       []aichat.ExecutionRequest
+	resolutions  []aichat.ToolApprovalResolution
+	continuation *aichat.ApprovalContinuation
 }
 
 func (f *fakeExecutionAuthority) Begin(_ context.Context, request aichat.ExecutionRequest) (aichat.Execution, error) {
@@ -36,7 +37,7 @@ func (f *fakeExecutionAuthority) ResolveToolApproval(
 	resolution aichat.ToolApprovalResolution,
 ) (*aichat.ApprovalContinuation, error) {
 	f.resolutions = append(f.resolutions, resolution)
-	return nil, nil
+	return f.continuation, nil
 }
 
 type fakeExecution struct {
@@ -171,6 +172,12 @@ var _ = Describe("Authoritative aichat execution", func() {
 		}
 		service := aichat.NewService(aichat.ServiceOptions{
 			Resolver: &fakeResolver{provider: provider}, Threads: aichat.FixedThreadStore(store), Authority: authority,
+			Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) {
+				return mustRuntimeProfile(api.SpecLayer{
+					Name: "accounts", Scope: api.SpecLayerContext,
+					Spec: api.Spec{Prompt: api.Prompt{System: "Use account policy."}},
+				}), nil
+			}),
 			Tools: aichat.StaticToolProvider([]api.ToolDefinition{{
 				Name: "account_edit", DefaultPermission: api.ToolModeAsk,
 				Handler: func(context.Context, map[string]any) (any, error) { return nil, nil },
@@ -191,6 +198,10 @@ var _ = Describe("Authoritative aichat execution", func() {
 		Expect(authority.begins).To(HaveLen(1))
 		Expect(authority.begins[0].ThreadID).To(Equal(thread.ID))
 		Expect(authority.begins[0].RequestID).To(Equal("user-message-1"))
+		Expect(authority.begins[0].Profile.Trace).To(HaveLen(2))
+		Expect(authority.begins[0].Profile.Trace[0].Name).To(Equal("accounts"))
+		Expect(authority.begins[0].Profile.Spec).To(Equal(authority.begins[0].Spec))
+		Expect(authority.begins[0].Profile.Spec.Prompt.System).To(Equal("Use account policy."))
 		Expect(provider.specs).To(HaveLen(1))
 		Expect(execution.observed).To(ContainElement(
 			MatchFields(IgnoreExtras, Fields{"Kind": Equal(api.EventResult)}),
@@ -402,4 +413,43 @@ var _ = Describe("Authoritative aichat execution", func() {
 		Expect(missing.Code).To(Equal(http.StatusNotFound))
 		Expect(authority.resolutions).To(HaveLen(1))
 	})
+
+	It("interrupts an approval continuation whose persisted model is no longer allowed", func() {
+		store := aichat.NewMemoryThreadStore()
+		thread, err := store.Create(context.Background(), "Restricted")
+		Expect(err).NotTo(HaveOccurred())
+		execution := &fakeExecution{}
+		authority := &fakeExecutionAuthority{continuation: &aichat.ApprovalContinuation{
+			Execution: execution,
+			Spec: api.Spec{
+				Model:        api.Model{Name: "gpt-5.6-sol"},
+				ToolApproval: &api.ToolApprovalResume{},
+			},
+		}}
+		resolver := &fakeResolver{provider: &fakeStreamingProvider{}}
+		profile := mustRuntimeProfile(api.SpecLayer{
+			Name: "claims", Scope: api.SpecLayerContext,
+			Spec:        api.Spec{Model: api.Model{Name: "claude-sonnet-5"}},
+			Constraints: api.RuntimeConstraints{Models: []string{"claude-sonnet-5"}},
+		})
+		service := aichat.NewService(aichat.ServiceOptions{
+			Threads: aichat.FixedThreadStore(store), Authority: authority, Resolver: resolver,
+			Profile: aichat.RuntimeProfileProviderFunc(func(context.Context) (aichat.RuntimeProfile, error) {
+				return profile, nil
+			}),
+		})
+
+		response := httptest.NewRecorder()
+		service.Handler().ServeHTTP(response, requestJSON(
+			http.MethodPost,
+			"/api/chat/sessions/"+thread.ID+"/approvals/0e5dc2fe-8b77-44e9-a3de-6a00298c8bde",
+			map[string]any{"approved": true},
+		))
+
+		Expect(response.Code).To(Equal(http.StatusBadGateway))
+		Expect(response.Body.String()).To(ContainSubstring(`model "gpt-5.6-sol" is outside the current effective model catalog`))
+		Expect(execution.interrupts).To(ConsistOf(ContainSubstring("outside the current effective model catalog")))
+		Expect(execution.closed).To(BeTrue())
+		Expect(resolver.configs).To(BeEmpty())
+	})
 })

pkg/aichat/execution_database_authority.go

@@ -46,7 +46,7 @@ func (a *DatabaseExecutionAuthority) Begin(
 	if session.Source != "aichat" {
 		return nil, fmt.Errorf("chat thread %s has incompatible source %q", request.ThreadID, session.Source)
 	}
-	renderedSpec, err := renderedSpecMap(request.Spec)
+	renderedSpec, err := renderedSpecMap(request.Spec, request.Profile)
 	if err != nil {
 		return nil, err
 	}
@@ -278,7 +278,7 @@ func runtimeSelection(model api.Model) database.PromptRunRuntimeSelection {
 	}
 }
 
-func renderedSpecMap(spec api.Spec) (map[string]any, error) {
+func renderedSpecMap(spec api.Spec, profile api.ResolvedSpec) (map[string]any, error) {
 	raw, err := json.Marshal(spec)
 	if err != nil {
 		return nil, fmt.Errorf("encode authoritative chat spec: %w", err)
@@ -287,6 +287,20 @@ func renderedSpecMap(spec api.Spec) (map[string]any, error) {
 	if err := json.Unmarshal(raw, &rendered); err != nil {
 		return nil, fmt.Errorf("decode authoritative chat spec: %w", err)
 	}
+	if len(profile.Trace) > 0 {
+		resolution, err := json.Marshal(struct {
+			Constraints api.RuntimeConstraints `json:"constraints"`
+			Trace       []api.SpecLayer        `json:"trace"`
+		}{Constraints: profile.Constraints, Trace: profile.Trace})
+		if err != nil {
+			return nil, fmt.Errorf("encode authoritative chat profile: %w", err)
+		}
+		var value map[string]any
+		if err := json.Unmarshal(resolution, &value); err != nil {
+			return nil, fmt.Errorf("decode authoritative chat profile: %w", err)
+		}
+		rendered["resolution"] = value
+	}
 	return rendered, nil
 }
 

pkg/aichat/messages.go

@@ -68,36 +68,45 @@ func (s *Service) resolveAttachments(ctx context.Context, messages []UIMessage)
 	return resolved, nil
 }
 
-func requestSpec(request ChatRequest, settings RuntimeSettings, attachments map[partLocation]api.AttachmentRef) (api.Spec, error) {
-	override, err := chatModel(request, settings.Spec.Model)
+func requestSpec(request ChatRequest, profile RuntimeProfile, attachments map[partLocation]api.AttachmentRef) (api.ResolvedSpec, error) {
+	override, err := chatModel(request, profile.Resolved.Spec.Model)
 	if err != nil {
-		return api.Spec{}, err
+		return api.ResolvedSpec{}, err
 	}
-	spec := settings.Spec.Merge(api.Spec{
+	user := api.SpecLayer{Name: "chat request", Scope: api.SpecLayerUser, Spec: api.Spec{
 		Model:           override,
 		Budget:          request.Budget,
 		ToolPreferences: request.ToolPreferences,
 		ToolApproval:    request.ToolApproval,
 		Permissions:     api.Permissions{Mode: request.PermissionMode},
 		SessionID:       request.ProviderSessionID,
-	})
+	}}
+	layers := append([]api.SpecLayer(nil), profile.Resolved.Trace...)
+	resolved, err := api.ResolveSpecLayers(append(layers, user)...)
+	if err != nil {
+		return api.ResolvedSpec{}, fmt.Errorf("resolve chat runtime profile: %w", err)
+	}
+	spec := resolved.Spec
+	baseSystem := strings.TrimSpace(strings.Join([]string{
+		profile.System, spec.Prompt.System, spec.Prompt.AppendSystem,
+	}, "\n\n"))
 	spec.Prompt.User = ""
 	spec.Prompt.System = ""
 	spec.Prompt.AppendSystem = ""
 	spec.Prompt.Attachments = nil
 	if request.ToolApproval == nil {
 		messages, err := canonicalMessages(request.Messages, attachments)
 		if err != nil {
-			return api.Spec{}, err
+			return api.ResolvedSpec{}, err
 		}
-		system, err := requestSystem(settings.System, request)
+		system, err := requestSystem(baseSystem, request)
 		if err != nil {
-			return api.Spec{}, err
+			return api.ResolvedSpec{}, err
 		}
 		if isAgentBackend(spec.Backend) {
 			user, promptAttachments, err := agentPrompt(messages, request.ProviderSessionID != "")
 			if err != nil {
-				return api.Spec{}, err
+				return api.ResolvedSpec{}, err
 			}
 			spec.Messages = nil
 			spec.Prompt.System = system
@@ -113,9 +122,10 @@ func requestSpec(request ChatRequest, settings RuntimeSettings, attachments map[
 		spec.Messages = nil
 	}
 	if err := spec.Validate(); err != nil {
-		return api.Spec{}, err
+		return api.ResolvedSpec{}, err
 	}
-	return spec, nil
+	resolved.Spec = spec
+	return resolved, nil
 }
 
 func chatModel(request ChatRequest, fallback api.Model) (api.Model, error) {