diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts index 05ebde1ad..c0bad969f 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts @@ -534,10 +534,11 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { ); }); - it('infers selected-model-unavailable callbacks as action-required failures', async () => { + it('reclassifies preflight unavailable-model callbacks as atomic model-not-found cancellations', async () => { const errorMessage = 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}'; mockGetCodeReviewById.mockResolvedValue(makeReview()); + mockFindKiloReviewComment.mockResolvedValue(null); const response = await POST( makeRequest({ @@ -550,43 +551,78 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { expect(response.status).toBe(200); expect(mockUpdateCodeReviewAttemptForCallback).toHaveBeenCalledWith( expect.objectContaining({ - status: 'failed', + status: 'cancelled', errorMessage, - terminalReason: 'selected_model_unavailable', + terminalReason: 'model_not_found', }) ); - expect(mockUpdateCodeReviewStatus).toHaveBeenCalledWith( + expect(mockUpdateCodeReviewStatusIfNonTerminal).toHaveBeenCalledWith( REVIEW_ID, - 'failed', + 'cancelled', expect.objectContaining({ errorMessage, - terminalReason: 'selected_model_unavailable', + terminalReason: 'model_not_found', }) ); - expect(mockUpdateCodeReviewStatusIfNonTerminal).not.toHaveBeenCalled(); + expect(mockUpdateCodeReviewStatus).not.toHaveBeenCalled(); expect(mockCreateInfraRetryAttemptIfMissing).not.toHaveBeenCalled(); expect(mockRetryReviewFresh).not.toHaveBeenCalled(); - expect(mockDisableCodeReviewForActionRequiredFailure).toHaveBeenCalledWith( - expect.objectContaining({ - owner: { type: 'user', id: 'user-1', userId: 'user-1' }, - platform: 'github', - reviewId: REVIEW_ID, - reason: 'selected_model_unavailable', - errorMessage, - }) - ); + expect(mockDisableCodeReviewForActionRequiredFailure).not.toHaveBeenCalled(); expect(mockUpdateCheckRun).toHaveBeenCalledWith( 'inst-1', 'owner', 'repo', 12345, expect.objectContaining({ - conclusion: 'action_required', - output: expect.objectContaining({ title: 'Selected model unavailable' }), + status: 'completed', + conclusion: 'cancelled', + output: expect.objectContaining({ title: 'Selected model is no longer available' }), }), 'standard' ); - expect(mockFindKiloReviewComment).not.toHaveBeenCalled(); + expect(mockCreatePRComment).toHaveBeenCalledWith( + 'inst-1', + 'owner', + 'repo', + 1, + expect.stringContaining('Choose another model in Kilo Code review settings'), + 'standard' + ); + }); + + it('reclassifies rolling-deploy selected-model-unavailable preflight callbacks as cancellations', async () => { + const errorMessage = + 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}'; + mockGetCodeReviewById.mockResolvedValue(makeReview()); + mockFindKiloReviewComment.mockResolvedValue(null); + + const response = await POST( + makeRequest({ + status: 'failed', + errorMessage, + terminalReason: 'selected_model_unavailable', + }), + makeParams(REVIEW_ID) + ); + + expect(response.status).toBe(200); + expect(mockUpdateCodeReviewAttemptForCallback).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'cancelled', + errorMessage, + terminalReason: 'model_not_found', + }) + ); + expect(mockUpdateCodeReviewStatusIfNonTerminal).toHaveBeenCalledWith( + REVIEW_ID, + 'cancelled', + expect.objectContaining({ + errorMessage, + terminalReason: 'model_not_found', + }) + ); + expect(mockUpdateCodeReviewStatus).not.toHaveBeenCalled(); + expect(mockDisableCodeReviewForActionRequiredFailure).not.toHaveBeenCalled(); }); it('infers model-not-allowed callbacks as action-required failures', async () => { diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts index ede6a3a0e..79bb9ae2d 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts @@ -75,6 +75,7 @@ import { type CodeReviewActionRequiredReason, } from '@/lib/code-reviews/action-required'; import type { Owner } from '@/lib/code-reviews/core'; +import { isCodeReviewModelUnavailableFailure } from '@/lib/code-reviews/model-unavailable'; /** * Payload from the orchestrator DO (legacy format). @@ -169,7 +170,7 @@ function normalizePayload(raw: StatusUpdatePayload): { if ( (raw.status === 'failed' || raw.status === 'interrupted') && - isModelNotFoundCodeReviewTerminalReason(terminalReason, raw.errorMessage) + isCodeReviewModelUnavailableFailure(terminalReason, raw.errorMessage) ) { status = 'cancelled'; terminalReason = 'model_not_found'; @@ -207,17 +208,6 @@ function isBillingCodeReviewTerminalReason( ); } -function isModelNotFoundCodeReviewTerminalReason( - terminalReason?: CodeReviewTerminalReason, - errorMessage?: string | null -): boolean { - if (terminalReason === 'model_not_found') { - return true; - } - - return /\bmodel\s+not\s+found\b/i.test(errorMessage ?? ''); -} - function getActionRequiredTerminalReason( terminalReason?: CodeReviewTerminalReason, errorMessage?: string | null @@ -239,7 +229,7 @@ function isRetryableInfraFailure( if (isCodeReviewActionRequiredReason(terminalReason)) return false; if (classifyCodeReviewActionRequiredFailure(errorMessage)) return false; if (isBillingCodeReviewTerminalReason(terminalReason, errorMessage)) return false; - if (isModelNotFoundCodeReviewTerminalReason(terminalReason, errorMessage)) return false; + if (isCodeReviewModelUnavailableFailure(terminalReason, errorMessage)) return false; const message = errorMessage?.toLowerCase(); if (!message) return false; @@ -431,7 +421,7 @@ function mapStatusToCheckRun( : null; const modelNotFoundCancellation = reviewStatus === 'cancelled' && - isModelNotFoundCodeReviewTerminalReason(terminalReason, errorMessage); + isCodeReviewModelUnavailableFailure(terminalReason, errorMessage); const actionRequiredCopy = actionRequiredReason ? getCodeReviewActionRequiredCopy(actionRequiredReason) : null; @@ -508,7 +498,7 @@ function getGitLabStatusDescription( if (reviewStatus === 'completed') return 'Kilo Code Review completed'; if ( reviewStatus === 'cancelled' && - isModelNotFoundCodeReviewTerminalReason(terminalReason, errorMessage) + isCodeReviewModelUnavailableFailure(terminalReason, errorMessage) ) { return MODEL_NOT_FOUND_GITLAB_DESCRIPTION; } @@ -988,8 +978,7 @@ export async function POST( : undefined, }; const isModelNotFoundCancellation = - status === 'cancelled' && - isModelNotFoundCodeReviewTerminalReason(terminalReason, errorMessage); + status === 'cancelled' && isCodeReviewModelUnavailableFailure(terminalReason, errorMessage); if (isModelNotFoundCancellation) { const claimedTerminalUpdate = await updateCodeReviewStatusIfNonTerminal( diff --git a/apps/web/src/lib/code-reviews/action-required.test.ts b/apps/web/src/lib/code-reviews/action-required.test.ts index 14b2633fc..44cfc6d1a 100644 --- a/apps/web/src/lib/code-reviews/action-required.test.ts +++ b/apps/web/src/lib/code-reviews/action-required.test.ts @@ -41,18 +41,6 @@ describe('classifyCodeReviewActionRequiredFailure', () => { ) ).toBe('github_ip_allow_list'); - expect( - classifyCodeReviewActionRequiredFailure( - 'Selected model is not available for this cloud agent session' - ) - ).toBe('selected_model_unavailable'); - - expect( - classifyCodeReviewActionRequiredFailure( - 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}' - ) - ).toBe('selected_model_unavailable'); - expect( classifyCodeReviewActionRequiredFailure( 'Not Found: The requested model is not allowed for your team.' @@ -66,13 +54,23 @@ describe('classifyCodeReviewActionRequiredFailure', () => { ).toBe('selected_model_unavailable'); }); - it('does not classify unrelated auth, rate-limit, or BYOK quota failures', () => { + it('does not classify unrelated auth, rate-limit, BYOK quota, or preflight model failures', () => { expect(classifyCodeReviewActionRequiredFailure('GitHub returned 401 Unauthorized')).toBeNull(); expect(classifyCodeReviewActionRequiredFailure('GitHub returned 403 Forbidden')).toBeNull(); expect(classifyCodeReviewActionRequiredFailure('Rate limit exceeded: 429')).toBeNull(); expect( classifyCodeReviewActionRequiredFailure('[BYOK] Your account quota is exhausted.') ).toBeNull(); + expect( + classifyCodeReviewActionRequiredFailure( + 'Selected model is not available for this cloud agent session' + ) + ).toBeNull(); + expect( + classifyCodeReviewActionRequiredFailure( + 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}' + ) + ).toBeNull(); }); it('routes selected model recovery to Code Reviewer settings', () => { diff --git a/apps/web/src/lib/code-reviews/action-required.ts b/apps/web/src/lib/code-reviews/action-required.ts index c3da23be9..fffcbd4da 100644 --- a/apps/web/src/lib/code-reviews/action-required.ts +++ b/apps/web/src/lib/code-reviews/action-required.ts @@ -36,8 +36,6 @@ const CodeReviewActionRequiredStateSchema = z.object({ emailSentAt: z.string().optional(), }); -const SELECTED_MODEL_UNAVAILABLE_MESSAGE = - 'selected model is not available for this cloud agent session'; const REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE = 'the requested model is not allowed for your team'; @@ -107,10 +105,7 @@ export function classifyCodeReviewActionRequiredFailure( return 'github_ip_allow_list'; } - if ( - normalized.includes(SELECTED_MODEL_UNAVAILABLE_MESSAGE) || - normalized.includes(REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE) - ) { + if (normalized.includes(REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE)) { return 'selected_model_unavailable'; } diff --git a/apps/web/src/lib/code-reviews/alerting/detectors.test.ts b/apps/web/src/lib/code-reviews/alerting/detectors.test.ts index 9b50e5df2..2c536b8e5 100644 --- a/apps/web/src/lib/code-reviews/alerting/detectors.test.ts +++ b/apps/web/src/lib/code-reviews/alerting/detectors.test.ts @@ -276,6 +276,34 @@ describe('code review alert detectors', () => { await expect(evaluateErrorSpike(db)).resolves.toEqual({ tripped: false }); }); + it('excludes legacy preflight model-unavailable failed rows without terminal reasons', async () => { + await insertReviews([ + reviewValues({ status: 'failed', terminal_reason: null, error_message: 'Checkout failed' }), + reviewValues({ status: 'failed', terminal_reason: null, error_message: 'Checkout failed' }), + reviewValues({ status: 'failed', terminal_reason: null, error_message: 'Checkout failed' }), + reviewValues({ status: 'failed', terminal_reason: null, error_message: 'Checkout failed' }), + reviewValues({ + status: 'failed', + terminal_reason: null, + error_message: + 'prepareSession failed (400): Selected model is not available for this cloud agent session', + }), + ...Array.from({ length: 15 }, () => reviewValues()), + ]); + + await expect(evaluateErrorSpike(db)).resolves.toMatchObject({ + tripped: true, + details: { + kind: 'error_spike', + startedCount: 20, + errorCount: 4, + rate: 0.2, + topReason: 'unknown', + topReasonCount: 4, + }, + }); + }); + it('continues counting non-model not-found failures as errors', async () => { await insertReviews([ reviewValues({ diff --git a/apps/web/src/lib/code-reviews/alerting/detectors.ts b/apps/web/src/lib/code-reviews/alerting/detectors.ts index 280cfb488..5f7ef4f7e 100644 --- a/apps/web/src/lib/code-reviews/alerting/detectors.ts +++ b/apps/web/src/lib/code-reviews/alerting/detectors.ts @@ -2,6 +2,7 @@ import type { db as defaultDb } from '@/lib/drizzle'; import { sql } from '@/lib/drizzle'; import { cloud_agent_code_reviews } from '@kilocode/db/schema'; import { CODE_REVIEW_BENIGN_TERMINAL_REASONS } from '@kilocode/db/schema-types'; +import { CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL } from '@/lib/code-reviews/model-unavailable'; import { ERROR_SPIKE_RATE_THRESHOLD, ERROR_SPIKE_WINDOW_MINUTES, @@ -61,6 +62,7 @@ const systemFailureSql = sql`( const modelNotFoundSql = sql`( COALESCE(terminal_reason, '') = 'model_not_found' OR COALESCE(error_message, '') ILIKE '%model not found%' + OR COALESCE(error_message, '') ILIKE ${sql.raw(CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL)} )`; const startedReviewsCteSql = sql` diff --git a/apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.test.ts b/apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.test.ts index 092107d2e..c274cdaa6 100644 --- a/apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.test.ts +++ b/apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.test.ts @@ -49,6 +49,7 @@ import { cloud_agent_code_reviews, kilocode_users, organizations, + platform_integrations, type User, } from '@kilocode/db/schema'; import { eq } from 'drizzle-orm'; @@ -125,6 +126,14 @@ describe('tryDispatchPendingReviews', () => { eq(agent_configs.owned_by_organization_id, testOrganizationId) ) ); + await db + .delete(platform_integrations) + .where( + or( + eq(platform_integrations.owned_by_user_id, testUser.id), + eq(platform_integrations.owned_by_organization_id, testOrganizationId) + ) + ); mockDispatchReview.mockReset(); mockGetReviewStatus.mockReset(); mockGetAgentConfigForOwner.mockReset(); @@ -369,12 +378,21 @@ describe('tryDispatchPendingReviews', () => { expect(mockSendCodeReviewDisabledEmail).toHaveBeenCalledTimes(1); }); - it('disables Code Reviewer for selected-model worker status failures', async () => { + it('normalizes old-worker preflight unavailable-model status without disabling Code Reviewer', async () => { const recentTimestamp = minutesAgo(1); const owner = { type: 'user', id: testUser.id } satisfies ReviewOwner; const errorMessage = 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session"}}'; const agentConfig = await insertAgentConfigForUser(); + const [integration] = await db + .insert(platform_integrations) + .values({ + owned_by_user_id: testUser.id, + platform: 'github', + integration_type: 'app', + platform_installation_id: 'selected-model-installation', + }) + .returning({ id: platform_integrations.id }); mockGetAgentConfigForOwner.mockResolvedValue(agentConfig); mockDispatchReview.mockRejectedValue( new Error("Dispatch returned terminal status 'failed' for review selected-model-review") @@ -386,6 +404,72 @@ describe('tryDispatchPendingReviews', () => { terminalReason: 'selected_model_unavailable', }); + const [review] = await db + .insert(cloud_agent_code_reviews) + .values({ + ...reviewValues({ + owner, + status: 'pending', + createdAt: recentTimestamp, + updatedAt: recentTimestamp, + }), + check_run_id: 123, + platform_integration_id: integration.id, + }) + .returning({ id: cloud_agent_code_reviews.id }); + + const result = await tryDispatchPendingReviews({ + type: 'user', + id: testUser.id, + userId: testUser.id, + }); + + const storedReview = await getStoredReview(review.id); + const storedAttempt = await db.query.cloud_agent_code_review_attempts.findFirst({ + where: eq(cloud_agent_code_review_attempts.code_review_id, review.id), + }); + const storedConfig = await db.query.agent_configs.findFirst({ + where: eq(agent_configs.id, agentConfig.id), + }); + + expect(result).toEqual({ dispatched: 1, notDispatched: 0, activeCount: 1 }); + expect(storedReview).toEqual( + expect.objectContaining({ + status: 'cancelled', + terminalReason: 'model_not_found', + errorMessage, + }) + ); + expect(storedAttempt).toEqual( + expect.objectContaining({ + status: 'cancelled', + terminal_reason: 'model_not_found', + error_message: errorMessage, + }) + ); + expect(storedConfig?.is_enabled).toBe(true); + expect(mockSendCodeReviewDisabledEmail).not.toHaveBeenCalled(); + expect(mockGetIntegrationById).not.toHaveBeenCalled(); + expect(mockUpdateCheckRun).not.toHaveBeenCalled(); + }); + + it('disables Code Reviewer for team-policy selected-model worker status failures', async () => { + const recentTimestamp = minutesAgo(1); + const owner = { type: 'user', id: testUser.id } satisfies ReviewOwner; + const errorMessage = + 'prepareSession failed (404): {"error":{"message":"Not Found: The requested model is not allowed for your team."}}'; + const agentConfig = await insertAgentConfigForUser(); + mockGetAgentConfigForOwner.mockResolvedValue(agentConfig); + mockDispatchReview.mockRejectedValue( + new Error("Dispatch returned terminal status 'failed' for review team-policy-review") + ); + mockGetReviewStatus.mockResolvedValue({ + reviewId: 'unused', + status: 'failed', + errorMessage, + terminalReason: 'selected_model_unavailable', + }); + const [review] = await db .insert(cloud_agent_code_reviews) .values( @@ -405,6 +489,9 @@ describe('tryDispatchPendingReviews', () => { }); const storedReview = await getStoredReview(review.id); + const storedAttempt = await db.query.cloud_agent_code_review_attempts.findFirst({ + where: eq(cloud_agent_code_review_attempts.code_review_id, review.id), + }); const storedConfig = await db.query.agent_configs.findFirst({ where: eq(agent_configs.id, agentConfig.id), }); @@ -417,6 +504,13 @@ describe('tryDispatchPendingReviews', () => { errorMessage, }) ); + expect(storedAttempt).toEqual( + expect.objectContaining({ + status: 'failed', + terminal_reason: 'selected_model_unavailable', + error_message: errorMessage, + }) + ); expect(storedConfig?.is_enabled).toBe(false); expect(mockSendCodeReviewDisabledEmail).toHaveBeenCalledTimes(1); }); diff --git a/apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.ts b/apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.ts index 8c35d6cc6..255d4e748 100644 --- a/apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.ts +++ b/apps/web/src/lib/code-reviews/dispatch/dispatch-pending-reviews.ts @@ -49,6 +49,7 @@ import { isCodeReviewActionRequiredReason, type CodeReviewActionRequiredReason, } from '../action-required'; +import { isCodeReviewModelUnavailableFailure } from '../model-unavailable'; import { activeCodeReviewWorkCondition, reconsiderableCodeReviewWorkCondition, @@ -589,11 +590,22 @@ async function handleAmbiguousDispatchFailure( const completedAt = workerStatus.completedAt ? new Date(workerStatus.completedAt) : undefined; const workerTerminalReason = parseTerminalReason(workerStatus.terminalReason); - const classifiedReason = classifyCodeReviewActionRequiredFailure(workerStatus.errorMessage); - const terminalReason = workerTerminalReason ?? classifiedReason ?? undefined; - const actionRequiredReason = isCodeReviewActionRequiredReason(workerTerminalReason) - ? workerTerminalReason - : classifiedReason; + const isModelUnavailable = isCodeReviewModelUnavailableFailure( + workerTerminalReason, + workerStatus.errorMessage + ); + const status = isModelUnavailable ? 'cancelled' : workerStatus.status; + const classifiedReason = isModelUnavailable + ? null + : classifyCodeReviewActionRequiredFailure(workerStatus.errorMessage); + const terminalReason = isModelUnavailable + ? 'model_not_found' + : (workerTerminalReason ?? classifiedReason ?? undefined); + const actionRequiredReason = isModelUnavailable + ? null + : isCodeReviewActionRequiredReason(workerTerminalReason) + ? workerTerminalReason + : classifiedReason; if (actionRequiredReason) { try { @@ -621,7 +633,7 @@ async function handleAmbiguousDispatchFailure( await updateCodeReviewAttemptForCallback({ codeReviewId: review.id, attemptId, - status: workerStatus.status, + status, sessionId: workerStatus.sessionId, cliSessionId: workerStatus.cliSessionId, errorMessage: workerStatus.errorMessage, @@ -630,7 +642,7 @@ async function handleAmbiguousDispatchFailure( }); const parentUpdated = await updateCodeReviewStatusIfNonTerminal( review.id, - workerStatus.status, + status, { sessionId: workerStatus.sessionId, cliSessionId: workerStatus.cliSessionId, @@ -644,7 +656,7 @@ async function handleAmbiguousDispatchFailure( logExceptInTest('[dispatchReview] Worker returned terminal status for fresh dispatch', { reviewId: review.id, attemptId, - status: workerStatus.status, + status, parentUpdated, }); return true; diff --git a/apps/web/src/lib/code-reviews/model-unavailable.test.ts b/apps/web/src/lib/code-reviews/model-unavailable.test.ts new file mode 100644 index 000000000..1a624e81d --- /dev/null +++ b/apps/web/src/lib/code-reviews/model-unavailable.test.ts @@ -0,0 +1,39 @@ +import { isCodeReviewModelUnavailableFailure } from './model-unavailable'; + +describe('isCodeReviewModelUnavailableFailure', () => { + it('classifies explicit model-not-found terminal reasons', () => { + expect(isCodeReviewModelUnavailableFailure('model_not_found')).toBe(true); + }); + + it('classifies runtime model-not-found messages case-insensitively', () => { + expect( + isCodeReviewModelUnavailableFailure(undefined, 'MODEL NOT FOUND: kilo/retired-model') + ).toBe(true); + }); + + it('classifies direct and wrapped preflight unavailable-model messages', () => { + expect( + isCodeReviewModelUnavailableFailure( + undefined, + 'Selected model is not available for this cloud agent session' + ) + ).toBe(true); + expect( + isCodeReviewModelUnavailableFailure( + undefined, + 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session","code":-32600,"data":{"code":"BAD_REQUEST","httpStatus":400,"path":"prepareSession"}}}' + ) + ).toBe(true); + }); + + it('does not classify team-policy, generic not-found, or reason-only unavailable-model failures', () => { + expect( + isCodeReviewModelUnavailableFailure( + undefined, + 'Not Found: The requested model is not allowed for your team.' + ) + ).toBe(false); + expect(isCodeReviewModelUnavailableFailure(undefined, 'Repository not found')).toBe(false); + expect(isCodeReviewModelUnavailableFailure('selected_model_unavailable')).toBe(false); + }); +}); diff --git a/apps/web/src/lib/code-reviews/model-unavailable.ts b/apps/web/src/lib/code-reviews/model-unavailable.ts new file mode 100644 index 000000000..f8badb9f4 --- /dev/null +++ b/apps/web/src/lib/code-reviews/model-unavailable.ts @@ -0,0 +1,25 @@ +export const CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_MESSAGE = + 'Selected model is not available for this cloud agent session'; +export const CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL = `'%${CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_MESSAGE}%'`; + +const NORMALIZED_CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_MESSAGE = + CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_MESSAGE.toLowerCase(); + +export function isCodeReviewModelUnavailableFailure( + terminalReason?: string | null, + errorMessage?: string | null +): boolean { + if (terminalReason === 'model_not_found') { + return true; + } + + const message = errorMessage?.toLowerCase(); + if (!message) { + return false; + } + + return ( + /\bmodel\s+not\s+found\b/i.test(message) || + message.includes(NORMALIZED_CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_MESSAGE) + ); +} diff --git a/apps/web/src/routers/admin-code-reviews-router.test.ts b/apps/web/src/routers/admin-code-reviews-router.test.ts index 1720b7827..20c48fa57 100644 --- a/apps/web/src/routers/admin-code-reviews-router.test.ts +++ b/apps/web/src/routers/admin-code-reviews-router.test.ts @@ -13,6 +13,8 @@ import { eq } from 'drizzle-orm'; const REPO = `test-org/admin-code-review-wait-${Date.now()}`; const START_DATE = '2035-01-01T00:00:00.000Z'; const END_DATE = '2035-01-20T00:00:00.000Z'; +const PREFLIGHT_MODEL_UNAVAILABLE_ERROR = + 'prepareSession failed (400): Selected model is not available for this cloud agent session'; type ReviewOwner = { type: 'user'; id: string } | { type: 'org'; id: string }; type FilterInput = { @@ -398,7 +400,7 @@ describe('adminCodeReviewsRouter', () => { expect(segmentation.ownershipBreakdown[0]).toMatchObject({ failed: 1 }); }); - it('buckets selected-model-unavailable terminal reasons as action required', async () => { + it('classifies historical preflight model-unavailable outcomes as cancellations', async () => { const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner; const [review] = await db .insert(cloud_agent_code_reviews) @@ -408,7 +410,7 @@ describe('adminCodeReviewsRouter', () => { status: 'failed', createdAt: timestamp(760), terminalReason: 'selected_model_unavailable', - errorMessage: 'Selected model is not available for this cloud agent session', + errorMessage: PREFLIGHT_MODEL_UNAVAILABLE_ERROR, }) ) .returning({ id: cloud_agent_code_reviews.id }); @@ -418,7 +420,57 @@ describe('adminCodeReviewsRouter', () => { attempt_number: 1, status: 'failed', terminal_reason: 'selected_model_unavailable', - error_message: 'Selected model is not available for this cloud agent session', + error_message: PREFLIGHT_MODEL_UNAVAILABLE_ERROR, + created_at: timestamp(761), + started_at: timestamp(762), + completed_at: timestamp(763), + }); + + const caller = await createCallerForUser(adminUser.id); + for (const retryAccountingMode of ['final_outcome', 'all_attempts'] as const) { + const input = filterInput({ retryAccountingMode }); + const overview = await caller.admin.codeReviews.getOverviewStats(input); + const daily = await caller.admin.codeReviews.getDailyStats(input); + const cancellations = await caller.admin.codeReviews.getCancellationAnalysis(input); + const errors = await caller.admin.codeReviews.getErrorAnalysis(input); + const sessions = await caller.admin.codeReviews.getErrorSessions({ + ...input, + errorMessage: PREFLIGHT_MODEL_UNAVAILABLE_ERROR, + }); + const segmentation = await caller.admin.codeReviews.getUserSegmentation(input); + + expect(overview).toMatchObject({ failedCount: 0, cancelledCount: 1 }); + expect(daily).toEqual([expect.objectContaining({ failed: 0, cancelled: 1 })]); + expect(cancellations).toEqual([ + expect.objectContaining({ reason: 'Model no longer available', count: 1 }), + ]); + expect(errors).toEqual({ categories: [], details: [] }); + expect(sessions).toEqual([]); + expect(segmentation.ownershipBreakdown[0]).toMatchObject({ failed: 0 }); + } + }); + + it('buckets team-policy selected-model-unavailable terminal reasons as action required', async () => { + const owner = { type: 'user', id: adminUser.id } satisfies ReviewOwner; + const [review] = await db + .insert(cloud_agent_code_reviews) + .values( + reviewValues({ + owner, + status: 'failed', + createdAt: timestamp(760), + terminalReason: 'selected_model_unavailable', + errorMessage: 'Not Found: The requested model is not allowed for your team.', + }) + ) + .returning({ id: cloud_agent_code_reviews.id }); + + await db.insert(cloud_agent_code_review_attempts).values({ + code_review_id: review.id, + attempt_number: 1, + status: 'failed', + terminal_reason: 'selected_model_unavailable', + error_message: 'Not Found: The requested model is not allowed for your team.', created_at: timestamp(761), started_at: timestamp(762), completed_at: timestamp(763), diff --git a/apps/web/src/routers/admin-code-reviews-router.ts b/apps/web/src/routers/admin-code-reviews-router.ts index 4a47afad6..889df130e 100644 --- a/apps/web/src/routers/admin-code-reviews-router.ts +++ b/apps/web/src/routers/admin-code-reviews-router.ts @@ -13,6 +13,7 @@ import { staleQueuedCodeReviewCutoffSql, staleRunningCodeReviewCutoffSql, } from '@/lib/code-reviews/dispatch/dispatch-constants'; +import { CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL } from '@/lib/code-reviews/model-unavailable'; /** * SQL condition that identifies billing/credits errors (402 Payment Required). @@ -39,11 +40,13 @@ const isBillingAttemptError = sql`( const isModelNotFound = sql`( ${cloud_agent_code_reviews.terminal_reason} = 'model_not_found' OR ${cloud_agent_code_reviews.error_message} ILIKE '%model not found%' + OR ${cloud_agent_code_reviews.error_message} ILIKE ${sql.raw(CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL)} )`; const isModelNotFoundAttempt = sql`( ${cloud_agent_code_review_attempts.terminal_reason} = 'model_not_found' OR ${cloud_agent_code_review_attempts.error_message} ILIKE '%model not found%' + OR ${cloud_agent_code_review_attempts.error_message} ILIKE ${sql.raw(CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL)} )`; /** @@ -63,10 +66,12 @@ const excludeBillingAttemptErrors = sql`COALESCE(${cloud_agent_code_review_attem AND COALESCE(${cloud_agent_code_review_attempts.error_message}, '') NOT ILIKE '%Credits Required%'`; const excludeModelNotFound = sql`COALESCE(${cloud_agent_code_reviews.terminal_reason}, '') <> 'model_not_found' - AND COALESCE(${cloud_agent_code_reviews.error_message}, '') NOT ILIKE '%model not found%'`; + AND COALESCE(${cloud_agent_code_reviews.error_message}, '') NOT ILIKE '%model not found%' + AND COALESCE(${cloud_agent_code_reviews.error_message}, '') NOT ILIKE ${sql.raw(CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL)}`; const excludeModelNotFoundAttempt = sql`COALESCE(${cloud_agent_code_review_attempts.terminal_reason}, '') <> 'model_not_found' - AND COALESCE(${cloud_agent_code_review_attempts.error_message}, '') NOT ILIKE '%model not found%'`; + AND COALESCE(${cloud_agent_code_review_attempts.error_message}, '') NOT ILIKE '%model not found%' + AND COALESCE(${cloud_agent_code_review_attempts.error_message}, '') NOT ILIKE ${sql.raw(CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL)}`; /** * Categorize error messages into high-level buckets via SQL CASE WHEN. @@ -481,7 +486,7 @@ export const adminCodeReviewsRouter = createTRPCRouter({ ] as SQL[]; const cancellationReasonExpr = sql`CASE - WHEN ${statusTable.terminal_reason} = 'model_not_found' OR ${statusTable.error_message} ILIKE '%model not found%' THEN 'Model no longer available' + WHEN ${statusTable.terminal_reason} = 'model_not_found' OR ${statusTable.error_message} ILIKE '%model not found%' OR ${statusTable.error_message} ILIKE ${sql.raw(CODE_REVIEW_MODEL_UNAVAILABLE_PREFLIGHT_SQL_LITERAL)} THEN 'Model no longer available' WHEN ${statusTable.terminal_reason} = 'superseded' OR ${statusTable.error_message} ILIKE '%superseded%' THEN 'Superseded by new commit' WHEN ${statusTable.error_message} ILIKE '%stream timeout%' THEN 'Stream timeout' WHEN ${statusTable.terminal_reason} = 'user_cancelled' OR ${statusTable.error_message} ILIKE '%cancelled%' OR ${statusTable.error_message} ILIKE '%canceled%' THEN 'Explicitly cancelled' diff --git a/services/code-review-infra/src/code-review-orchestrator.ts b/services/code-review-infra/src/code-review-orchestrator.ts index ffc0a3e1e..bcc93cec3 100644 --- a/services/code-review-infra/src/code-review-orchestrator.ts +++ b/services/code-review-infra/src/code-review-orchestrator.ts @@ -739,8 +739,12 @@ export class CodeReviewOrchestrator extends DurableObject { if ( message.includes(SELECTED_MODEL_UNAVAILABLE_MESSAGE) || - message.includes(REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE) + /\bmodel\s+not\s+found\b/i.test(message) ) { + return 'model_not_found'; + } + + if (message.includes(REQUESTED_MODEL_NOT_ALLOWED_FOR_TEAM_MESSAGE)) { return 'selected_model_unavailable'; } @@ -1253,8 +1257,9 @@ export class CodeReviewOrchestrator extends DurableObject { ); const terminalReason = this.getTerminalReason(error); + const status = terminalReason === 'model_not_found' ? 'cancelled' : 'failed'; - await this.updateStatus('failed', { errorMessage, terminalReason }); + await this.updateStatus(status, { errorMessage, terminalReason }); console.error('[CodeReviewOrchestrator] Review failed (cloud-agent-next):', { reviewId: this.state.reviewId, diff --git a/services/code-review-infra/test/integration/code-review-orchestrator.test.ts b/services/code-review-infra/test/integration/code-review-orchestrator.test.ts index b60b03d45..972722830 100644 --- a/services/code-review-infra/test/integration/code-review-orchestrator.test.ts +++ b/services/code-review-infra/test/integration/code-review-orchestrator.test.ts @@ -886,7 +886,7 @@ describe('CodeReviewOrchestrator recovery', () => { expect(stored?.sandboxRetryAttempted).toBeUndefined(); }); - it('maps selected-model prepareSession 400 failures to action-required terminal reason', async () => { + it('maps selected-model prepareSession 400 failures to model-not-found cancellation', async () => { const stub = getReviewStub(); const fetchMock = vi.fn(async (request: RequestInfo | URL) => { const url = String(request); @@ -913,18 +913,80 @@ describe('CodeReviewOrchestrator recovery', () => { expect(ran).toBe(true); await expect(stub.status()).resolves.toMatchObject({ - status: 'failed', - terminalReason: 'selected_model_unavailable', + status: 'cancelled', + terminalReason: 'model_not_found', + errorMessage: expect.stringContaining( + 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session"' + ), }); expect(fetchCalls(fetchMock, '/trpc/prepareSession')).toHaveLength(1); expect(fetchCalls(fetchMock, '/trpc/initiateFromKilocodeSessionV2')).toHaveLength(0); expect(lastStatusUpdateBody(fetchMock)).toMatchObject({ - status: 'failed', - terminalReason: 'selected_model_unavailable', + status: 'cancelled', + terminalReason: 'model_not_found', + errorMessage: expect.stringContaining( + 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session"' + ), }); await expect(storedReview(stub)).resolves.toMatchObject({ - status: 'failed', - terminalReason: 'selected_model_unavailable', + status: 'cancelled', + terminalReason: 'model_not_found', + errorMessage: expect.stringContaining( + 'prepareSession failed (400): {"error":{"message":"Selected model is not available for this cloud agent session"' + ), + }); + }); + + it('maps model-not-found initiation failures to cancellation', async () => { + const stub = getReviewStub(); + const fetchMock = vi.fn(async (request: RequestInfo | URL) => { + const url = String(request); + if (url.includes('/api/internal/code-review-status/')) { + return Response.json({ success: true }); + } + if (url.includes('/trpc/prepareSession')) { + return trpcSuccess({ + cloudAgentSessionId: 'agent-retired-model', + kiloSessionId: 'ses_retired_model', + }); + } + if (url.includes('/trpc/initiateFromKilocodeSessionV2')) { + return trpcError(404, 'Model not found: kilo/retired-model', 'NOT_FOUND'); + } + return new Response('unexpected fetch', { status: 500 }); + }); + globalThis.fetch = fetchMock; + + await runInDurableObject(stub, async (_instance: CodeReviewOrchestrator, state) => { + await state.storage.put('state', codeReview()); + await state.storage.setAlarm(Date.now() + 30_000); + }); + + const ran = await runDurableObjectAlarm(stub); + + expect(ran).toBe(true); + await expect(stub.status()).resolves.toMatchObject({ + status: 'cancelled', + terminalReason: 'model_not_found', + errorMessage: expect.stringContaining( + 'initiateFromKilocodeSessionV2 failed (404): {"error":{"message":"Model not found: kilo/retired-model"' + ), + }); + expect(fetchCalls(fetchMock, '/trpc/prepareSession')).toHaveLength(1); + expect(fetchCalls(fetchMock, '/trpc/initiateFromKilocodeSessionV2')).toHaveLength(1); + expect(lastStatusUpdateBody(fetchMock)).toMatchObject({ + status: 'cancelled', + terminalReason: 'model_not_found', + errorMessage: expect.stringContaining( + 'initiateFromKilocodeSessionV2 failed (404): {"error":{"message":"Model not found: kilo/retired-model"' + ), + }); + await expect(storedReview(stub)).resolves.toMatchObject({ + status: 'cancelled', + terminalReason: 'model_not_found', + errorMessage: expect.stringContaining( + 'initiateFromKilocodeSessionV2 failed (404): {"error":{"message":"Model not found: kilo/retired-model"' + ), }); });