From deb137522d36b24f222725b94f93874188a27f85 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Tue, 30 Jun 2026 19:40:10 +0530 Subject: [PATCH] perf(runtime): cache credential-backed selection state --- credentials/combined.go | 77 +++++++++++++++++- credentials/combined_store_test.go | 108 +++++++++++++++++++++++++ runtime/default_provider.go | 37 +++++---- runtime/selection.go | 125 ++++++++++++++++++++++++++--- runtime/transport.go | 13 +++ 5 files changed, 330 insertions(+), 30 deletions(-) diff --git a/credentials/combined.go b/credentials/combined.go index 35457f3..04ea8ae 100644 --- a/credentials/combined.go +++ b/credentials/combined.go @@ -4,13 +4,25 @@ import ( "context" "fmt" "strings" + "sync" + "time" ) // CombinedStore persists secrets in the OS secret store (macOS Keychain / Linux Secret Service). type CombinedStore struct { Keychain Store + mu sync.RWMutex + cache map[string]combinedCacheEntry } +type combinedCacheEntry struct { + secret string + found bool + cachedAt time.Time +} + +const combinedGetCacheTTL = 2 * time.Second + func NewCombinedStore() *CombinedStore { return &CombinedStore{ Keychain: newPlatformKeyringStore(), @@ -28,6 +40,7 @@ func (c *CombinedStore) Set(ctx context.Context, account, secret string) error { if err := c.Keychain.Set(ctx, account, secret); err != nil { return fmt.Errorf("%w: %s", err, KeyringUnavailableHelp()) } + c.invalidateAccount(account) return nil } @@ -35,12 +48,72 @@ func (c *CombinedStore) Get(ctx context.Context, account string) (string, error) if c.Keychain == nil { return "", ErrNotFound } - return c.Keychain.Get(ctx, account) + if secret, ok := c.cachedGet(account); ok { + if secret == "" { + return "", ErrNotFound + } + return secret, nil + } + secret, err := c.Keychain.Get(ctx, account) + if err == nil { + c.storeCachedGet(account, secret, true) + return secret, nil + } + if err == ErrNotFound { + c.storeCachedGet(account, "", false) + } + return secret, err } func (c *CombinedStore) Delete(ctx context.Context, account string) error { if c.Keychain == nil { return nil } - return c.Keychain.Delete(ctx, account) + err := c.Keychain.Delete(ctx, account) + if err == nil { + c.invalidateAccount(account) + } + return err +} + +func (c *CombinedStore) cachedGet(account string) (string, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + if c.cache == nil { + return "", false + } + entry, ok := c.cache[c.cacheKey(account)] + if !ok || time.Since(entry.cachedAt) >= combinedGetCacheTTL { + return "", false + } + if !entry.found { + return "", true + } + return entry.secret, true +} + +func (c *CombinedStore) storeCachedGet(account, secret string, found bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.cache == nil { + c.cache = make(map[string]combinedCacheEntry) + } + c.cache[c.cacheKey(account)] = combinedCacheEntry{ + secret: strings.TrimSpace(secret), + found: found, + cachedAt: time.Now(), + } +} + +func (c *CombinedStore) invalidateAccount(account string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.cache == nil { + return + } + delete(c.cache, c.cacheKey(account)) +} + +func (c *CombinedStore) cacheKey(account string) string { + return strings.ToLower(strings.TrimSpace(account)) } diff --git a/credentials/combined_store_test.go b/credentials/combined_store_test.go index f9fd66b..3c92de4 100644 --- a/credentials/combined_store_test.go +++ b/credentials/combined_store_test.go @@ -3,9 +3,54 @@ package credentials import ( "context" "errors" + "sync" "testing" ) +type countingStore struct { + mu sync.Mutex + data map[string]string + gets int + deletes int +} + +func (s *countingStore) Set(ctx context.Context, account, secret string) error { + _ = ctx + s.mu.Lock() + defer s.mu.Unlock() + if s.data == nil { + s.data = map[string]string{} + } + s.data[account] = secret + return nil +} + +func (s *countingStore) Get(ctx context.Context, account string) (string, error) { + _ = ctx + s.mu.Lock() + defer s.mu.Unlock() + s.gets++ + if s.data == nil { + return "", ErrNotFound + } + v, ok := s.data[account] + if !ok || v == "" { + return "", ErrNotFound + } + return v, nil +} + +func (s *countingStore) Delete(ctx context.Context, account string) error { + _ = ctx + s.mu.Lock() + defer s.mu.Unlock() + s.deletes++ + if s.data != nil { + delete(s.data, account) + } + return nil +} + // --- CombinedStore edge-case tests (nil keychain, empty secrets) --- func TestCombinedStore_NilKeychain(t *testing.T) { @@ -103,6 +148,69 @@ func TestCombinedStore_FullCycle(t *testing.T) { } } +func TestCombinedStore_GetUsesShortLivedCache(t *testing.T) { + t.Parallel() + backend := &countingStore{data: map[string]string{"account": "secret"}} + cs := &CombinedStore{Keychain: backend} + ctx := context.Background() + + got1, err := cs.Get(ctx, "account") + if err != nil || got1 != "secret" { + t.Fatalf("first Get = (%q, %v), want (secret, nil)", got1, err) + } + got2, err := cs.Get(ctx, "account") + if err != nil || got2 != "secret" { + t.Fatalf("second Get = (%q, %v), want (secret, nil)", got2, err) + } + if backend.gets != 1 { + t.Fatalf("backend Get count = %d, want 1", backend.gets) + } +} + +func TestCombinedStore_SetInvalidatesCache(t *testing.T) { + t.Parallel() + backend := &countingStore{data: map[string]string{"account": "old"}} + cs := &CombinedStore{Keychain: backend} + ctx := context.Background() + + got, err := cs.Get(ctx, "account") + if err != nil || got != "old" { + t.Fatalf("initial Get = (%q, %v), want (old, nil)", got, err) + } + if err := cs.Set(ctx, "account", "new"); err != nil { + t.Fatalf("Set error: %v", err) + } + got, err = cs.Get(ctx, "account") + if err != nil || got != "new" { + t.Fatalf("post-Set Get = (%q, %v), want (new, nil)", got, err) + } + if backend.gets != 2 { + t.Fatalf("backend Get count after invalidation = %d, want 2", backend.gets) + } +} + +func TestCombinedStore_DeleteInvalidatesCache(t *testing.T) { + t.Parallel() + backend := &countingStore{data: map[string]string{"account": "secret"}} + cs := &CombinedStore{Keychain: backend} + ctx := context.Background() + + got, err := cs.Get(ctx, "account") + if err != nil || got != "secret" { + t.Fatalf("initial Get = (%q, %v), want (secret, nil)", got, err) + } + if err := cs.Delete(ctx, "account"); err != nil { + t.Fatalf("Delete error: %v", err) + } + got, err = cs.Get(ctx, "account") + if err != ErrNotFound || got != "" { + t.Fatalf("post-Delete Get = (%q, %v), want (\"\", ErrNotFound)", got, err) + } + if backend.gets != 2 { + t.Fatalf("backend Get count after delete invalidation = %d, want 2", backend.gets) + } +} + // --- DefaultStore / SetDefaultStore tests --- func TestSetDefaultStore_Override(t *testing.T) { diff --git a/runtime/default_provider.go b/runtime/default_provider.go index 650e39d..98f2e26 100644 --- a/runtime/default_provider.go +++ b/runtime/default_provider.go @@ -49,35 +49,37 @@ func PreferredProvider(ctx context.Context) string { if ctx == nil { ctx = context.Background() } - if provider := normalizeRuntimeProviderID(ActiveProvider(ctx)); provider != "" && providerConfigured(ctx, provider) { + return preferredProviderWithState(ctx, newSelectionRuntimeState(ctx)) +} + +func preferredProviderWithState(ctx context.Context, state *selectionRuntimeState) string { + if provider := normalizeRuntimeProviderID(ActiveProvider(ctx)); provider != "" && providerConfiguredWithState(state, provider) { return provider } if model := ActiveModel(ctx); model != "" { - if provider := inferProviderForModel(ctx, model); provider != "" && providerConfigured(ctx, provider) { + if provider := inferProviderForModel(ctx, model); provider != "" && providerConfiguredWithState(state, provider) { return provider } } - if provider := preferredConfiguredProvider(ctx); provider != "" { + if provider := preferredConfiguredProviderWithState(state); provider != "" { return provider } - return preferredDetectedProvider() + return preferredDetectedProviderWithState(state) } -func preferredConfiguredProvider(ctx context.Context) string { - rt, err := Load(ctx) - if err != nil || rt == nil { +func preferredConfiguredProviderWithState(state *selectionRuntimeState) string { + compiled := state.compiledCatalog() + if compiled == nil || compiled.Catalog == nil { return "" } - rows, err := rt.DeploymentRows() - if err != nil || len(rows) == 0 { - return "" - } - configured := make(map[string]struct{}, len(rows)) - for _, row := range rows { - if !row.Configured { + env := state.env() + configured := make(map[string]struct{}, len(compiled.Catalog.Deployments)) + for id, dep := range compiled.Catalog.Deployments { + dc := config.DeploymentConfigFromEnv(dep, env) + if !config.DeploymentConfigured(id, dep, dc) { continue } - if provider := catalog.CanonicalProviderID(row.ProviderID); provider != "" { + if provider := catalog.CanonicalProviderID(dep.ProviderID); provider != "" { configured[provider] = struct{}{} } } @@ -98,7 +100,8 @@ func preferredConfiguredProvider(ctx context.Context) string { return ordered[0] } -func preferredDetectedProvider() string { +func preferredDetectedProviderWithState(state *selectionRuntimeState) string { + env := state.env() for _, provider := range registry.ChatProviderPreferenceOrder() { profile, ok := runtimeProfileForProvider(provider) if !ok { @@ -106,7 +109,7 @@ func preferredDetectedProvider() string { } ready := true for _, envKey := range profile.DetectionEnv { - if runtimeEnvValue(envKey) == "" { + if strings.TrimSpace(env[envKey]) == "" && runtimeEnvValue(envKey) == "" { ready = false break } diff --git a/runtime/selection.go b/runtime/selection.go index 720ec6a..240ca4c 100644 --- a/runtime/selection.go +++ b/runtime/selection.go @@ -12,6 +12,87 @@ import ( "github.com/GrayCodeAI/eyrie/setup" ) +type selectionRuntimeState struct { + ctx context.Context + compiled *catalog.CompiledCatalogV1 + compiledLoaded bool + discoveryEnv map[string]string + discoveryEnvLoaded bool + hasConfiguredDeployment bool + hasConfiguredLoaded bool + providerConfiguredCache map[string]bool +} + +func newSelectionRuntimeState(ctx context.Context) *selectionRuntimeState { + if ctx == nil { + ctx = context.Background() + } + return &selectionRuntimeState{ + ctx: ctx, + providerConfiguredCache: make(map[string]bool), + } +} + +func (s *selectionRuntimeState) compiledCatalog() *catalog.CompiledCatalogV1 { + if s == nil { + return nil + } + if s.compiledLoaded { + return s.compiled + } + s.compiledLoaded = true + compiled, err := catalog.LoadCatalogForDiscovery(s.ctx) + if err == nil { + s.compiled = compiled + } + return s.compiled +} + +func (s *selectionRuntimeState) env() map[string]string { + if s == nil { + return nil + } + if s.discoveryEnvLoaded { + return s.discoveryEnv + } + s.discoveryEnvLoaded = true + s.discoveryEnv = config.DiscoveryEnvMap(s.ctx) + return s.discoveryEnv +} + +func (s *selectionRuntimeState) hasAnyConfiguredDeployment() bool { + if s == nil { + return false + } + if s.hasConfiguredLoaded { + return s.hasConfiguredDeployment + } + s.hasConfiguredLoaded = true + env := s.env() + compiled := s.compiledCatalog() + if compiled == nil || compiled.Catalog == nil { + s.hasConfiguredDeployment = runtimeAnyNonemptyCredentialEnv(env) + return s.hasConfiguredDeployment + } + for id, dep := range compiled.Catalog.Deployments { + dc := config.DeploymentConfigFromEnv(dep, env) + if config.DeploymentConfigured(id, dep, dc) { + s.hasConfiguredDeployment = true + return true + } + } + return false +} + +func runtimeAnyNonemptyCredentialEnv(env map[string]string) bool { + for _, v := range env { + if !config.LooksLikePlaceholderSecret(v) { + return true + } + } + return false +} + // ActiveModel returns the selected model from provider.json. func ActiveModel(ctx context.Context) string { cfg := config.LoadProviderConfig("") @@ -112,7 +193,11 @@ func SyncSelectionWithCredentials(ctx context.Context) { if ctx == nil { ctx = context.Background() } - if !HasConfiguredDeployment(ctx) { + syncSelectionWithCredentials(ctx, newSelectionRuntimeState(ctx)) +} + +func syncSelectionWithCredentials(ctx context.Context, state *selectionRuntimeState) { + if !state.hasAnyConfiguredDeployment() { if strings.TrimSpace(ActiveModel(ctx)) != "" || strings.TrimSpace(ActiveProvider(ctx)) != "" { _ = ClearActiveSelection(ctx) } @@ -122,7 +207,7 @@ func SyncSelectionWithCredentials(ctx context.Context) { if gateway == "" { return } - if !providerConfigured(ctx, gateway) { + if !providerConfiguredWithState(state, gateway) { _ = ClearActiveSelection(ctx) } } @@ -134,10 +219,11 @@ func EffectiveSelection(ctx context.Context, opts SelectionOpts) SelectionState if ctx == nil { ctx = context.Background() } - SyncSelectionWithCredentials(ctx) + stateCache := newSelectionRuntimeState(ctx) + syncSelectionWithCredentials(ctx, stateCache) cfg := config.LoadProviderConfig("") state := SelectionState{ - HasConfiguredDeployment: HasConfiguredDeployment(ctx), + HasConfiguredDeployment: stateCache.hasAnyConfiguredDeployment(), DeploymentRouting: useDeploymentRouting(cfg, opts.DeploymentRoutingOverride), } @@ -156,7 +242,7 @@ func EffectiveSelection(ctx context.Context, opts SelectionOpts) SelectionState } if provider == "" { - provider = PreferredProvider(ctx) + provider = preferredProviderWithState(ctx, stateCache) } if !state.HasConfiguredDeployment { @@ -165,8 +251,8 @@ func EffectiveSelection(ctx context.Context, opts SelectionOpts) SelectionState return state } - if provider != "" && !providerConfigured(ctx, provider) { - if detected := normalizeRuntimeProviderID(client.DetectProvider()); detected != "" && providerConfigured(ctx, detected) { + if provider != "" && !providerConfiguredWithState(stateCache, provider) { + if detected := normalizeRuntimeProviderID(client.DetectProvider()); detected != "" && providerConfiguredWithState(stateCache, detected) { provider = detected model = "" } @@ -257,25 +343,42 @@ func activeGateway(ctx context.Context) string { } func providerConfigured(ctx context.Context, provider string) bool { + return providerConfiguredWithState(newSelectionRuntimeState(ctx), provider) +} + +func providerConfiguredWithState(state *selectionRuntimeState, provider string) bool { provider = normalizeRuntimeProviderID(provider) if provider == "" { return false } + if state != nil { + if cached, ok := state.providerConfiguredCache[provider]; ok { + return cached + } + } spec, ok := registry.SpecByProviderID(provider) if !ok { return false } - compiled, err := catalog.LoadCatalogForDiscovery(ctx) - if err != nil || compiled == nil || compiled.Catalog == nil { + var compiled *catalog.CompiledCatalogV1 + var env map[string]string + if state != nil { + compiled = state.compiledCatalog() + env = state.env() + } + if compiled == nil || compiled.Catalog == nil { return false } dep, ok := compiled.Catalog.Deployments[spec.DeploymentID] if !ok { return false } - env := config.DiscoveryEnvMap(ctx) dc := config.DeploymentConfigFromEnv(dep, env) - return config.DeploymentConfigured(spec.DeploymentID, dep, dc) + configured := config.DeploymentConfigured(spec.DeploymentID, dep, dc) + if state != nil { + state.providerConfiguredCache[provider] = configured + } + return configured } func normalizeRuntimeProviderID(provider string) string { diff --git a/runtime/transport.go b/runtime/transport.go index a8b17c2..2d41d42 100644 --- a/runtime/transport.go +++ b/runtime/transport.go @@ -28,6 +28,19 @@ func ResolveChatTransport(ctx context.Context, opts ChatTransportOpts) (ChatTran ctx = context.Background() } selection := EffectiveSelection(ctx, opts.Selection) + return resolveChatTransportSelection(ctx, selection) +} + +// ResolveChatTransportFromSelection constructs a transport from an already +// resolved selection state, avoiding redundant provider/model discovery. +func ResolveChatTransportFromSelection(ctx context.Context, selection SelectionState) (ChatTransport, error) { + if ctx == nil { + ctx = context.Background() + } + return resolveChatTransportSelection(ctx, selection) +} + +func resolveChatTransportSelection(ctx context.Context, selection SelectionState) (ChatTransport, error) { transport := ChatTransport{Selection: selection} if !selection.DeploymentRouting { transport.Provider = directChatProvider(ctx, selection.Provider)