From fcc664839bf2c4fee7f8d1a20a2a3b31e0465ac4 Mon Sep 17 00:00:00 2001
From: Nicolas Bouliol <nicolas.bouliol@forestadmin.com>
Date: Mon, 13 Jul 2026 17:16:12 +0200
Subject: [PATCH] test(ai-proxy): retry transient provider errors in model
 tool-support check

The 'all models should support tool calls' integration test treated a
transient Anthropic 529 overloaded_error as a model capability failure,
turning a provider blip into a red CI run. The dispatcher uses
maxRetries: 0 by design, so nothing absorbs the blip.

Retry transient errors (429/5xx/529/timeouts) with backoff; if still
failing, skip the model (logged) instead of counting it as a failure.
Genuine capability failures and hard infra errors (auth/DNS) stay
blocking.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../ai-proxy/test/llm.integration.test.ts     | 82 +++++++++++++------
 1 file changed, 59 insertions(+), 23 deletions(-)

diff --git a/packages/ai-proxy/test/llm.integration.test.ts b/packages/ai-proxy/test/llm.integration.test.ts
index 5bc7f22626..27c97e0740 100644
--- a/packages/ai-proxy/test/llm.integration.test.ts
+++ b/packages/ai-proxy/test/llm.integration.test.ts
@@ -26,6 +26,23 @@ const { OPENAI_API_KEY, ANTHROPIC_API_KEY } = process.env;
 const describeWithOpenAI = OPENAI_API_KEY ? describe : describe.skip;
 const describeWithAnthropic = ANTHROPIC_API_KEY ? describe : describe.skip;
 
+const HARD_INFRA_ERROR_PATTERN = /401|403|Authentication|invalid x-api-key|ECONNREFUSED|getaddrinfo/i;
+const TRANSIENT_ERROR_PATTERN =
+  /429|500|502|503|504|529|rate limit|overloaded|ETIMEDOUT|ECONNRESET|ECONNABORTED|socket hang up/i;
+const MAX_MODEL_QUERY_ATTEMPTS = 3;
+
+function classifyProviderError(message: string): 'hard' | 'transient' | 'capability' {
+  if (HARD_INFRA_ERROR_PATTERN.test(message)) return 'hard';
+  if (TRANSIENT_ERROR_PATTERN.test(message)) return 'transient';
+
+  return 'capability';
+}
+
+const sleep = (ms: number): Promise<void> =>
+  new Promise(resolve => {
+    setTimeout(resolve, ms);
+  });
+
 async function fetchChatModelsFromOpenAI(): Promise<string[]> {
   const openai = new OpenAI({ apiKey: OPENAI_API_KEY });
 
@@ -410,18 +427,16 @@ providers.forEach(
           console.log(`Testing ${modelsToTest.length} ${label} models:`, modelsToTest);
         });
 
-        it('all models should support tool calls', async () => {
-          const results: { model: string; success: boolean; error?: string }[] = [];
-
-          for (const model of modelsToTest) {
-            const modelRouter = new Router({
-              aiConfigurations: [
-                { name: 'test', provider: aiConfig.provider, model, apiKey: aiConfig.apiKey },
-              ],
-            });
+        const queryModelToolSupport = async (model: string): Promise<ChatCompletionResponse> => {
+          const modelRouter = new Router({
+            aiConfigurations: [
+              { name: 'test', provider: aiConfig.provider, model, apiKey: aiConfig.apiKey },
+            ],
+          });
 
+          for (let attempt = 1; ; attempt += 1) {
             try {
-              const response = (await modelRouter.route({
+              return (await modelRouter.route({
                 route: 'ai-query',
                 body: {
                   messages: [{ role: 'user', content: 'What is 2+2?' }],
@@ -442,6 +457,23 @@ providers.forEach(
                   parallel_tool_calls: false,
                 },
               })) as ChatCompletionResponse;
+            } catch (error) {
+              const isTransient = classifyProviderError(String(error)) === 'transient';
+
+              if (!isTransient || attempt >= MAX_MODEL_QUERY_ATTEMPTS) throw error;
+
+              await sleep(2000 * attempt);
+            }
+          }
+        };
+
+        it('all models should support tool calls', async () => {
+          const results: { model: string; success: boolean; error?: string }[] = [];
+          const skipped: { model: string; error: string }[] = [];
+
+          for (const model of modelsToTest) {
+            try {
+              const response = await queryModelToolSupport(model);
 
               const success =
                 response.choices[0].finish_reason === 'tool_calls' &&
@@ -451,23 +483,27 @@ providers.forEach(
             } catch (error) {
               const errorMessage = String(error);
 
-              const isInfrastructureError =
-                errorMessage.includes('rate limit') ||
-                errorMessage.includes('429') ||
-                errorMessage.includes('401') ||
-                errorMessage.includes('Authentication') ||
-                errorMessage.includes('ECONNREFUSED') ||
-                errorMessage.includes('ETIMEDOUT') ||
-                errorMessage.includes('getaddrinfo');
-
-              if (isInfrastructureError) {
-                throw new Error(`Infrastructure error testing model ${model}: ${errorMessage}`);
+              switch (classifyProviderError(errorMessage)) {
+                case 'hard':
+                  throw new Error(`Infrastructure error testing model ${model}: ${errorMessage}`);
+                case 'transient':
+                  skipped.push({ model, error: errorMessage });
+                  break;
+                default:
+                  results.push({ model, success: false, error: errorMessage });
               }
-
-              results.push({ model, success: false, error: errorMessage });
             }
           }
 
+          if (skipped.length > 0) {
+            const skippedModelNames = skipped.map(s => s.model).join(', ');
+            // eslint-disable-next-line no-console
+            console.warn(
+              `\n⚠️ ${skipped.length} ${label} model(s) skipped after transient provider errors: ${skippedModelNames}\n`,
+              skipped,
+            );
+          }
+
           const failures = results.filter(r => !r.success);
 
           if (failures.length > 0) {