From e48bf6f3a17ab3722f881180fb0b8ffbbd9fcdf7 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 20:28:08 +0100 Subject: [PATCH 01/22] Add github action to update model list dynamically --- .github/workflows/sync-ai-models.yml | 116 +++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 .github/workflows/sync-ai-models.yml diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml new file mode 100644 index 0000000000..7118eae74e --- /dev/null +++ b/.github/workflows/sync-ai-models.yml @@ -0,0 +1,116 @@ +name: Sync AI Models + +on: + schedule: + - cron: '0 0 * * 0' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + sync-models: + name: Sync AI Provider Models + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6.0.2 + + - name: Restore node_modules cache + id: node-modules-cache + uses: actions/cache/restore@v5.0.3 + with: + path: node_modules + key: node-modules-cache-${{ hashFiles('package-lock.json', '.npmrc') }} + + - name: Install dependencies + if: steps.node-modules-cache.outputs.cache-hit != 'true' + run: npm ci --no-audit --no-fund + + - name: Check for model updates + id: check-models + continue-on-error: true + run: | + npx tsx packages/openops/src/lib/ai/sync-models.ts --update > sync-output.txt 2>&1 + echo "exit_code=$?" >> $GITHUB_OUTPUT + cat sync-output.txt + + - name: Upload sync output + if: steps.check-models.outputs.exit_code == '1' + uses: actions/upload-artifact@v4 + with: + name: sync-output + path: sync-output.txt + retention-days: 7 + + - name: Create Pull Request + if: steps.check-models.outputs.exit_code == '1' + id: create-pr + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: 'Update AI provider models from models.dev' + title: 'Update AI Provider Models' + body: | + ## šŸ¤– Automated Model Sync + + This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). + + ### Changes Detected + + The sync script found differences between our current model lists and the latest available models. + + ### What to Review + + - āœ… Verify that new models are legitimate and available + - āœ… Check that removed models are actually deprecated + - āœ… Ensure model IDs match the provider's API documentation + + ### Testing + + Run the sync script locally to verify: + ```bash + npx tsx packages/openops/src/lib/ai/sync-models.ts + ``` + + ### Sync Output + + Full details will be added in a comment below. + + --- + + šŸ”— **Source**: https://models.dev/api.json + šŸ“… **Run**: ${{ github.run_id }} + branch: sync-ai-models-${{ github.run_number }} + delete-branch: true + labels: | + dependencies + automated + + - name: Comment on PR with details + if: steps.create-pr.outputs.pull-request-number + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const output = fs.readFileSync('sync-output.txt', 'utf8'); + + github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ steps.create-pr.outputs.pull-request-number }}, + body: `### šŸ“Š Model Sync Details\n\n\`\`\`\n${output}\n\`\`\`` + }); + + - name: Summary + if: always() + run: | + if [ "${{ steps.check-models.outputs.exit_code }}" == "0" ]; then + echo "āœ… All AI provider models are up to date!" >> $GITHUB_STEP_SUMMARY + elif [ "${{ steps.check-models.outputs.exit_code }}" == "1" ]; then + echo "šŸ”„ Model updates detected and PR created" >> $GITHUB_STEP_SUMMARY + cat sync-output.txt >> $GITHUB_STEP_SUMMARY + else + echo "āŒ Sync script failed with unexpected error" >> $GITHUB_STEP_SUMMARY + exit 1 + fi From 712d76ea433dd177fb1c438b43259866b9fc34bb Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 20:29:48 +0100 Subject: [PATCH 02/22] Actually add script to github lol --- packages/openops/src/lib/ai/sync-models.ts | 308 +++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 packages/openops/src/lib/ai/sync-models.ts diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts new file mode 100644 index 0000000000..26b1838f6e --- /dev/null +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -0,0 +1,308 @@ +/** + * Syncs AI provider model lists from models.dev + * + * This script fetches the latest AI model data from models.dev and updates + * our provider files accordingly. It filters for text-only models and excludes + * embedding models. + * + * Data source: https://models.dev (MIT License) + * API endpoint: https://models.dev/api.json + * GitHub: https://github.com/anomalyco/models.dev + * + * Usage: + * npx tsx sync-models.ts # Check for differences + * npx tsx sync-models.ts --update # Update provider files + */ + +import fs from 'fs'; +import path from 'path'; +import { AiProviderEnum } from '@openops/shared'; + +interface ModelData { + id: string; + name: string; + family: string; + modalities: { + input: string[]; + output: string[]; + }; + cost?: { + input: number; + output: number; + }; + limit?: { + context: number; + output: number; + }; +} + +interface ProviderData { + id: string; + name: string; + models: Record; +} + +interface ModelsDevAPI { + [providerKey: string]: ProviderData; +} + +const MODELS_DEV_KEYS: Partial> = { + [AiProviderEnum.ANTHROPIC]: 'anthropic', + [AiProviderEnum.CEREBRAS]: 'cerebras', + [AiProviderEnum.COHERE]: 'cohere', + [AiProviderEnum.DEEPINFRA]: 'deepinfra', + [AiProviderEnum.DEEPSEEK]: 'deepseek', + [AiProviderEnum.GOOGLE]: 'google', + [AiProviderEnum.GOOGLE_VERTEX]: 'google-vertex', + [AiProviderEnum.GROQ]: 'groq', + [AiProviderEnum.MISTRAL]: 'mistral', + [AiProviderEnum.OPENAI]: 'openai', + [AiProviderEnum.PERPLEXITY]: 'perplexity', + [AiProviderEnum.TOGETHER_AI]: 'togetherai', + [AiProviderEnum.XAI]: 'xai', +}; + +function normalizeProviderKey(key: string): string { + return key.toLowerCase().replace(/[-_]/g, ''); +} + +function getProviderFiles(): string[] { + const providersDir = path.join(__dirname, 'providers'); + return fs + .readdirSync(providersDir) + .filter((file) => file.endsWith('.ts') && file !== 'index.ts') + .map((file) => file.replace('.ts', '')); +} + +function findMatchingProviderFile(modelsDevKey: string): string | null { + const providerFiles = getProviderFiles(); + const normalizedKey = normalizeProviderKey(modelsDevKey); + + return ( + providerFiles.find( + (file) => normalizeProviderKey(file) === normalizedKey, + ) || null + ); +} + +async function fetchModelsDevData(): Promise { + const response = await fetch('https://models.dev/api.json'); + if (!response.ok) { + throw new Error(`Failed to fetch models.dev API: ${response.statusText}`); + } + return response.json(); +} + +function isEmbeddingModel(modelId: string): boolean { + return modelId.toLowerCase().includes('embedding'); +} + +function filterTextOnlyModels(models: Record): string[] { + return Object.values(models) + .filter((model) => { + const outputModalities = model.modalities?.output || []; + const isTextOnly = + outputModalities.length === 1 && outputModalities[0] === 'text'; + + if (!isTextOnly) return false; + + if (isEmbeddingModel(model.id)) return false; + + return true; + }) + .map((model) => model.id) + .sort(); +} + +function getProviderFilePath(providerFileName: string): string { + return path.join(__dirname, 'providers', `${providerFileName}.ts`); +} + +function getCurrentModels(providerFileName: string): string[] { + const filePath = getProviderFilePath(providerFileName); + + if (!fs.existsSync(filePath)) { + console.warn(`āš ļø Provider file not found: ${filePath}`); + return []; + } + + const content = fs.readFileSync(filePath, 'utf-8'); + + const modelsArrayMatch = content.match( + /const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/, + ); + + if (!modelsArrayMatch) { + console.warn(`āš ļø Could not find models array in ${providerFileName}.ts`); + return []; + } + + const modelsString = modelsArrayMatch[1]; + const models = modelsString + .split(',') + .map((line) => { + const match = line.match(/['"]([^'"]+)['"]/); + return match ? match[1] : null; + }) + .filter((model): model is string => model !== null); + + return models.sort(); +} + +function updateProviderFile( + providerFileName: string, + newModels: string[], +): void { + const filePath = getProviderFilePath(providerFileName); + + if (!fs.existsSync(filePath)) { + console.warn(`āš ļø Cannot update: Provider file not found: ${filePath}`); + return; + } + + const content = fs.readFileSync(filePath, 'utf-8'); + + const modelsArrayMatch = content.match( + /const\s+(\w+Models)\s*=\s*\[([\s\S]*?)\];/, + ); + + if (!modelsArrayMatch) { + console.warn( + `āš ļø Cannot update: Could not find models array in ${providerFileName}.ts`, + ); + return; + } + + const arrayName = modelsArrayMatch[1]; + + const formattedModels = newModels.map((model) => ` '${model}',`).join('\n'); + + const newArray = `const ${arrayName} = [\n${formattedModels}\n];`; + + const updatedContent = content.replace( + /const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/, + newArray, + ); + + fs.writeFileSync(filePath, updatedContent, 'utf-8'); + + console.log(` āœ… Updated ${providerFileName}.ts`); +} + +function compareModels( + current: string[], + latest: string[], +): { + added: string[]; + removed: string[]; + unchanged: string[]; +} { + const currentSet = new Set(current); + const latestSet = new Set(latest); + + const added = latest.filter((model) => !currentSet.has(model)); + const removed = current.filter((model) => !latestSet.has(model)); + const unchanged = current.filter((model) => latestSet.has(model)); + + return { added, removed, unchanged }; +} + +async function main() { + const shouldUpdate = process.argv.includes('--update'); + + console.log('šŸ”„ Fetching models from models.dev...\n'); + + const modelsDevData = await fetchModelsDevData(); + + console.log('šŸ“Š Comparing models for each provider:\n'); + console.log('='.repeat(80)); + + let totalAdded = 0; + let totalRemoved = 0; + let hasChanges = false; + + for (const [provider, modelsDevKey] of Object.entries(MODELS_DEV_KEYS)) { + if (!modelsDevKey) continue; + + const providerData = modelsDevData[modelsDevKey]; + + if (!providerData) { + console.log(`\nāŒ ${provider}`); + console.log(` Provider "${modelsDevKey}" not found in models.dev`); + continue; + } + + const providerFile = findMatchingProviderFile(modelsDevKey); + + if (!providerFile) { + console.log(`\nāŒ ${provider}`); + console.log( + ` No matching provider file found for "${modelsDevKey}"`, + ); + console.log( + ` Available files: ${getProviderFiles().join(', ')}`, + ); + continue; + } + + const latestModels = filterTextOnlyModels(providerData.models); + const currentModels = getCurrentModels(providerFile); + + const diff = compareModels(currentModels, latestModels); + + const hasProviderChanges = diff.added.length > 0 || diff.removed.length > 0; + + if (hasProviderChanges) { + hasChanges = true; + } + + const icon = hasProviderChanges ? 'šŸ”„' : 'āœ…'; + console.log(`\n${icon} ${provider} (${modelsDevKey})`); + console.log(` File: ${providerFile}.ts`); + console.log(` Current: ${currentModels.length} models`); + console.log(` Latest: ${latestModels.length} models`); + + if (diff.added.length > 0) { + console.log(` āž• Added (${diff.added.length}):`); + diff.added.forEach((model) => console.log(` - ${model}`)); + totalAdded += diff.added.length; + } + + if (diff.removed.length > 0) { + console.log(` āž– Removed (${diff.removed.length}):`); + diff.removed.forEach((model) => console.log(` - ${model}`)); + totalRemoved += diff.removed.length; + } + + if (!hasProviderChanges) { + console.log(` āœ“ No changes`); + } + + if (hasProviderChanges && shouldUpdate) { + updateProviderFile(providerFile, latestModels); + } + } + + console.log('\n' + '='.repeat(80)); + console.log('\nšŸ“ˆ Summary:'); + console.log(` Total models added: ${totalAdded}`); + console.log(` Total models removed: ${totalRemoved}`); + + if (hasChanges) { + if (shouldUpdate) { + console.log('\nāœ… Provider files updated successfully!'); + console.log(' Please review the changes and commit.'); + } else { + console.log('\nāš ļø Changes detected! Run with --update to apply them.'); + } + process.exit(1); + } else { + console.log('\nāœ… All providers are up to date!'); + process.exit(0); + } +} + +main().catch((error) => { + console.error('āŒ Error:', error); + process.exit(1); +}); From 9154a21a483131b854d22ac2b7968a2432525df9 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 20:48:54 +0100 Subject: [PATCH 03/22] Quick test --- .github/workflows/sync-ai-models.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 7118eae74e..220399581a 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -1,6 +1,10 @@ name: Sync AI Models on: + pull_request: + paths: + - '.github/workflows/sync-ai-models.yml' + - 'packages/openops/src/lib/ai/sync-models.ts' schedule: - cron: '0 0 * * 0' workflow_dispatch: From 18862db36582ba4b68dec0eb35cda268f0b00876 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 20:51:20 +0100 Subject: [PATCH 04/22] test again --- .github/workflows/sync-ai-models.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 220399581a..5e1797aec8 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -35,9 +35,13 @@ jobs: id: check-models continue-on-error: true run: | + set +e npx tsx packages/openops/src/lib/ai/sync-models.ts --update > sync-output.txt 2>&1 - echo "exit_code=$?" >> $GITHUB_OUTPUT + EXIT_CODE=$? + set -e + echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT cat sync-output.txt + exit 0 - name: Upload sync output if: steps.check-models.outputs.exit_code == '1' From fe698f0c4f6af53888ff8ead7283b4db2bd14e94 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 20:52:50 +0100 Subject: [PATCH 05/22] Add base branch to PR creation --- .github/workflows/sync-ai-models.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 5e1797aec8..7b5c75a9b5 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -57,6 +57,7 @@ jobs: uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.GITHUB_TOKEN }} + base: main commit-message: 'Update AI provider models from models.dev' title: 'Update AI Provider Models' body: | From 2fe017469721b061ee550f7939dbaef89e7069bc Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 20:56:35 +0100 Subject: [PATCH 06/22] Simplify PR creation and fix errors --- .github/workflows/sync-ai-models.yml | 45 +++++----------------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 7b5c75a9b5..1cc113feae 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -43,14 +43,6 @@ jobs: cat sync-output.txt exit 0 - - name: Upload sync output - if: steps.check-models.outputs.exit_code == '1' - uses: actions/upload-artifact@v4 - with: - name: sync-output - path: sync-output.txt - retention-days: 7 - - name: Create Pull Request if: steps.check-models.outputs.exit_code == '1' id: create-pr @@ -65,52 +57,29 @@ jobs: This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). - ### Changes Detected - - The sync script found differences between our current model lists and the latest available models. + **Review the file changes** to see which models were added or removed for each provider. - ### What to Review + ### What to Check - - āœ… Verify that new models are legitimate and available - - āœ… Check that removed models are actually deprecated - - āœ… Ensure model IDs match the provider's API documentation + - āœ… New models are legitimate and available + - āœ… Removed models are actually deprecated/retired + - āœ… Model IDs match provider documentation - ### Testing + ### Testing Locally - Run the sync script locally to verify: ```bash npx tsx packages/openops/src/lib/ai/sync-models.ts ``` - ### Sync Output - - Full details will be added in a comment below. - --- - šŸ”— **Source**: https://models.dev/api.json - šŸ“… **Run**: ${{ github.run_id }} + šŸ”— **Source**: https://models.dev/api.json (MIT License) branch: sync-ai-models-${{ github.run_number }} delete-branch: true labels: | dependencies automated - - name: Comment on PR with details - if: steps.create-pr.outputs.pull-request-number - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const output = fs.readFileSync('sync-output.txt', 'utf8'); - - github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: ${{ steps.create-pr.outputs.pull-request-number }}, - body: `### šŸ“Š Model Sync Details\n\n\`\`\`\n${output}\n\`\`\`` - }); - - name: Summary if: always() run: | From ca1b87fb5896abc191fa1b34e42ee199ede612f7 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 21:07:44 +0100 Subject: [PATCH 07/22] More testing --- .github/workflows/sync-ai-models.yml | 108 ++++++++++++++------- packages/openops/src/lib/ai/sync-models.ts | 38 ++++++++ 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 1cc113feae..dae7903f19 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -31,6 +31,9 @@ jobs: if: steps.node-modules-cache.outputs.cache-hit != 'true' run: npm ci --no-audit --no-fund + - name: Build shared package + run: npx nx run shared:build + - name: Check for model updates id: check-models continue-on-error: true @@ -43,42 +46,76 @@ jobs: cat sync-output.txt exit 0 - - name: Create Pull Request + - name: Read warnings if: steps.check-models.outputs.exit_code == '1' - id: create-pr - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ secrets.GITHUB_TOKEN }} - base: main - commit-message: 'Update AI provider models from models.dev' - title: 'Update AI Provider Models' - body: | - ## šŸ¤– Automated Model Sync - - This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). - - **Review the file changes** to see which models were added or removed for each provider. - - ### What to Check - - - āœ… New models are legitimate and available - - āœ… Removed models are actually deprecated/retired - - āœ… Model IDs match provider documentation - - ### Testing Locally - - ```bash - npx tsx packages/openops/src/lib/ai/sync-models.ts - ``` - - --- + id: warnings + run: | + if [ -f warnings.txt ]; then + echo "HAS_WARNINGS=true" >> $GITHUB_OUTPUT + { + echo 'WARNINGS<> $GITHUB_OUTPUT + else + echo "HAS_WARNINGS=false" >> $GITHUB_OUTPUT + fi - šŸ”— **Source**: https://models.dev/api.json (MIT License) - branch: sync-ai-models-${{ github.run_number }} - delete-branch: true - labels: | - dependencies - automated + - name: Commit and push to branch + if: steps.check-models.outputs.exit_code == '1' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b sync-ai-models-${{ github.run_number }} + git add packages/openops/src/lib/ai/providers/ + git commit -m "Update AI provider models from models.dev" + git push origin sync-ai-models-${{ github.run_number }} + + # Commented out for testing - uncomment when ready to auto-create PRs + # - name: Create Pull Request + # if: steps.check-models.outputs.exit_code == '1' + # id: create-pr + # uses: peter-evans/create-pull-request@v7 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # base: main + # commit-message: 'Update AI provider models from models.dev' + # title: 'Update AI Provider Models' + # body: | + # ## šŸ¤– Automated Model Sync + # + # This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). + # + # **Review the file changes** to see which models were added or removed for each provider. + # + # ${{ steps.warnings.outputs.HAS_WARNINGS == 'true' && format(' + # + # --- + # + # {0} + # + # ', steps.warnings.outputs.WARNINGS) || '' }} + # + # ### What to Check + # + # - āœ… New models are legitimate and available + # - āœ… Removed models are actually deprecated/retired + # - āœ… Model IDs match provider documentation + # + # ### Testing Locally + # + # ```bash + # npx tsx packages/openops/src/lib/ai/sync-models.ts + # ``` + # + # --- + # + # šŸ”— **Source**: https://models.dev/api.json (MIT License) + # branch: sync-ai-models-${{ github.run_number }} + # delete-branch: true + # labels: | + # dependencies + # automated - name: Summary if: always() @@ -86,8 +123,7 @@ jobs: if [ "${{ steps.check-models.outputs.exit_code }}" == "0" ]; then echo "āœ… All AI provider models are up to date!" >> $GITHUB_STEP_SUMMARY elif [ "${{ steps.check-models.outputs.exit_code }}" == "1" ]; then - echo "šŸ”„ Model updates detected and PR created" >> $GITHUB_STEP_SUMMARY - cat sync-output.txt >> $GITHUB_STEP_SUMMARY + echo "šŸ”„ Model updates detected and branch created: sync-ai-models-${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY else echo "āŒ Sync script failed with unexpected error" >> $GITHUB_STEP_SUMMARY exit 1 diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index 26b1838f6e..541474f69c 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -207,11 +207,49 @@ function compareModels( return { added, removed, unchanged }; } +function checkForUnmappedProviders(): string[] { + const providerFiles = getProviderFiles(); + const mappedKeys = Object.values(MODELS_DEV_KEYS).filter( + (key): key is string => key !== undefined, + ); + + const unmappedFiles = providerFiles.filter((file) => { + const normalizedFile = normalizeProviderKey(file); + return !mappedKeys.some( + (key) => normalizeProviderKey(key) === normalizedFile, + ); + }); + + return unmappedFiles; +} + async function main() { const shouldUpdate = process.argv.includes('--update'); console.log('šŸ”„ Fetching models from models.dev...\n'); + const unmappedProviders = checkForUnmappedProviders(); + if (unmappedProviders.length > 0) { + console.log('āš ļø Warning: Found provider files not in MODELS_DEV_KEYS:'); + unmappedProviders.forEach((file) => + console.log(` - ${file}.ts (not synced)`), + ); + console.log( + ' Add these to MODELS_DEV_KEYS if they should be synced.\n', + ); + + if (shouldUpdate) { + const warningMessage = unmappedProviders + .map((file) => `- \`${file}.ts\``) + .join('\n'); + fs.writeFileSync( + 'warnings.txt', + `āš ļø **Warning**: The following provider files are not in \`MODELS_DEV_KEYS\` and were not synced:\n\n${warningMessage}\n\nIf these should be synced with models.dev, add them to \`MODELS_DEV_KEYS\` in \`sync-models.ts\`.`, + 'utf-8', + ); + } + } + const modelsDevData = await fetchModelsDevData(); console.log('šŸ“Š Comparing models for each provider:\n'); From 1c62f3b3698d481323ca052311a2b58099731622 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 21:10:10 +0100 Subject: [PATCH 08/22] testy test --- .github/workflows/sync-ai-models.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index dae7903f19..81ec3dc88d 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -39,11 +39,10 @@ jobs: continue-on-error: true run: | set +e - npx tsx packages/openops/src/lib/ai/sync-models.ts --update > sync-output.txt 2>&1 + npx tsx packages/openops/src/lib/ai/sync-models.ts --update EXIT_CODE=$? set -e echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT - cat sync-output.txt exit 0 - name: Read warnings @@ -68,6 +67,10 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" git checkout -b sync-ai-models-${{ github.run_number }} git add packages/openops/src/lib/ai/providers/ + if git diff --staged --quiet; then + echo "No changes to commit" + exit 0 + fi git commit -m "Update AI provider models from models.dev" git push origin sync-ai-models-${{ github.run_number }} From 40a3c7e8740baf168b5586859843fb3066fc00d1 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 21:13:22 +0100 Subject: [PATCH 09/22] Hilfe --- .github/workflows/sync-ai-models.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 81ec3dc88d..3f2c141d74 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -31,15 +31,17 @@ jobs: if: steps.node-modules-cache.outputs.cache-hit != 'true' run: npm ci --no-audit --no-fund - - name: Build shared package - run: npx nx run shared:build + - name: Build packages + run: | + npx nx run shared:build + npx nx run openops:build - name: Check for model updates id: check-models continue-on-error: true run: | set +e - npx tsx packages/openops/src/lib/ai/sync-models.ts --update + node packages/openops/dist/lib/ai/sync-models.js --update EXIT_CODE=$? set -e echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT From fd2aa4a6835e22c3e6bc856f7e23c6a9a97056f6 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 21:16:33 +0100 Subject: [PATCH 10/22] please --- .github/workflows/sync-ai-models.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 3f2c141d74..6058d76e6f 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -32,9 +32,11 @@ jobs: run: npm ci --no-audit --no-fund - name: Build packages + env: + NX_REJECT_UNKNOWN_LOCAL_CACHE: 0 run: | - npx nx run shared:build - npx nx run openops:build + npm run prepare + npx nx run openops-common:build - name: Check for model updates id: check-models From d3af81317dad2136c8f71a88741084fff0bb74d6 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 21:18:36 +0100 Subject: [PATCH 11/22] on god --- .github/workflows/sync-ai-models.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 6058d76e6f..bed4ce1aeb 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -43,7 +43,7 @@ jobs: continue-on-error: true run: | set +e - node packages/openops/dist/lib/ai/sync-models.js --update + node dist/packages/openops/lib/ai/sync-models.js --update EXIT_CODE=$? set -e echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT From 72d076d8eb7994e10a867db43dc6c95bd75677f0 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 21:24:00 +0100 Subject: [PATCH 12/22] ahh --- .github/workflows/sync-ai-models.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index bed4ce1aeb..76af46b874 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -31,19 +31,15 @@ jobs: if: steps.node-modules-cache.outputs.cache-hit != 'true' run: npm ci --no-audit --no-fund - - name: Build packages - env: - NX_REJECT_UNKNOWN_LOCAL_CACHE: 0 - run: | - npm run prepare - npx nx run openops-common:build + - name: Build shared package + run: npx nx run shared:build - name: Check for model updates id: check-models continue-on-error: true run: | set +e - node dist/packages/openops/lib/ai/sync-models.js --update + npx tsx --tsconfig tsconfig.base.json packages/openops/src/lib/ai/sync-models.ts --update EXIT_CODE=$? set -e echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT From 1bc679334a15f1ae5ebf3b53c8a3692d32c7c76d Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 21:25:23 +0100 Subject: [PATCH 13/22] ahhh --- .github/workflows/sync-ai-models.yml | 23 ---------------------- packages/openops/src/lib/ai/sync-models.ts | 11 ----------- 2 files changed, 34 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 76af46b874..8d1bd14eda 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -45,21 +45,6 @@ jobs: echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT exit 0 - - name: Read warnings - if: steps.check-models.outputs.exit_code == '1' - id: warnings - run: | - if [ -f warnings.txt ]; then - echo "HAS_WARNINGS=true" >> $GITHUB_OUTPUT - { - echo 'WARNINGS<> $GITHUB_OUTPUT - else - echo "HAS_WARNINGS=false" >> $GITHUB_OUTPUT - fi - - name: Commit and push to branch if: steps.check-models.outputs.exit_code == '1' run: | @@ -91,14 +76,6 @@ jobs: # # **Review the file changes** to see which models were added or removed for each provider. # - # ${{ steps.warnings.outputs.HAS_WARNINGS == 'true' && format(' - # - # --- - # - # {0} - # - # ', steps.warnings.outputs.WARNINGS) || '' }} - # # ### What to Check # # - āœ… New models are legitimate and available diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index 541474f69c..2afb5da213 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -237,17 +237,6 @@ async function main() { console.log( ' Add these to MODELS_DEV_KEYS if they should be synced.\n', ); - - if (shouldUpdate) { - const warningMessage = unmappedProviders - .map((file) => `- \`${file}.ts\``) - .join('\n'); - fs.writeFileSync( - 'warnings.txt', - `āš ļø **Warning**: The following provider files are not in \`MODELS_DEV_KEYS\` and were not synced:\n\n${warningMessage}\n\nIf these should be synced with models.dev, add them to \`MODELS_DEV_KEYS\` in \`sync-models.ts\`.`, - 'utf-8', - ); - } } const modelsDevData = await fetchModelsDevData(); From 5e307f7a3c62eb3eed987147fe79383a19930dfd Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 21:29:11 +0100 Subject: [PATCH 14/22] Add back pr creation step --- .github/workflows/sync-ai-models.yml | 87 ++++++++++++---------------- 1 file changed, 36 insertions(+), 51 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 8d1bd14eda..59824a55cb 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -45,57 +45,42 @@ jobs: echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT exit 0 - - name: Commit and push to branch + - name: Create Pull Request if: steps.check-models.outputs.exit_code == '1' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git checkout -b sync-ai-models-${{ github.run_number }} - git add packages/openops/src/lib/ai/providers/ - if git diff --staged --quiet; then - echo "No changes to commit" - exit 0 - fi - git commit -m "Update AI provider models from models.dev" - git push origin sync-ai-models-${{ github.run_number }} - - # Commented out for testing - uncomment when ready to auto-create PRs - # - name: Create Pull Request - # if: steps.check-models.outputs.exit_code == '1' - # id: create-pr - # uses: peter-evans/create-pull-request@v7 - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # base: main - # commit-message: 'Update AI provider models from models.dev' - # title: 'Update AI Provider Models' - # body: | - # ## šŸ¤– Automated Model Sync - # - # This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). - # - # **Review the file changes** to see which models were added or removed for each provider. - # - # ### What to Check - # - # - āœ… New models are legitimate and available - # - āœ… Removed models are actually deprecated/retired - # - āœ… Model IDs match provider documentation - # - # ### Testing Locally - # - # ```bash - # npx tsx packages/openops/src/lib/ai/sync-models.ts - # ``` - # - # --- - # - # šŸ”— **Source**: https://models.dev/api.json (MIT License) - # branch: sync-ai-models-${{ github.run_number }} - # delete-branch: true - # labels: | - # dependencies - # automated + id: create-pr + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + base: main + commit-message: 'Update AI provider models from models.dev' + title: 'Update AI Provider Models' + body: | + ## šŸ¤– Automated Model Sync + + This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). + + **Review the file changes** to see which models were added or removed for each provider. + + ### What to Check + + - āœ… New models are legitimate and available + - āœ… Removed models are actually deprecated/retired + - āœ… Model IDs match provider documentation + + ### Testing Locally + + ```bash + npx tsx packages/openops/src/lib/ai/sync-models.ts + ``` + + --- + + šŸ”— **Source**: https://models.dev/api.json (MIT License) + branch: sync-ai-models-${{ github.run_number }} + delete-branch: true + labels: | + dependencies + automated - name: Summary if: always() @@ -103,7 +88,7 @@ jobs: if [ "${{ steps.check-models.outputs.exit_code }}" == "0" ]; then echo "āœ… All AI provider models are up to date!" >> $GITHUB_STEP_SUMMARY elif [ "${{ steps.check-models.outputs.exit_code }}" == "1" ]; then - echo "šŸ”„ Model updates detected and branch created: sync-ai-models-${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY + echo "šŸ”„ Model updates detected and PR created" >> $GITHUB_STEP_SUMMARY else echo "āŒ Sync script failed with unexpected error" >> $GITHUB_STEP_SUMMARY exit 1 From 7d47b3bf7f75642b5c45f914c18af7572a042fe5 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 23:33:51 +0100 Subject: [PATCH 15/22] Clean --- packages/openops/src/lib/ai/sync-models.ts | 28 ++++++------- packages/openops/test/ai/sync-models.test.ts | 44 ++++++++++++++++++++ 2 files changed, 56 insertions(+), 16 deletions(-) create mode 100644 packages/openops/test/ai/sync-models.test.ts diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index 2afb5da213..214c29936a 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -14,9 +14,9 @@ * npx tsx sync-models.ts --update # Update provider files */ +import { AiProviderEnum } from '@openops/shared'; import fs from 'fs'; import path from 'path'; -import { AiProviderEnum } from '@openops/shared'; interface ModelData { id: string; @@ -46,11 +46,11 @@ interface ModelsDevAPI { [providerKey: string]: ProviderData; } -const MODELS_DEV_KEYS: Partial> = { +export const MODELS_DEV_KEYS: Partial> = { [AiProviderEnum.ANTHROPIC]: 'anthropic', [AiProviderEnum.CEREBRAS]: 'cerebras', [AiProviderEnum.COHERE]: 'cohere', - [AiProviderEnum.DEEPINFRA]: 'deepinfra', + // [AiProviderEnum.DEEPINFRA]: 'deepinfra', // Temporarily disabled until models.dev PR is merged [AiProviderEnum.DEEPSEEK]: 'deepseek', [AiProviderEnum.GOOGLE]: 'google', [AiProviderEnum.GOOGLE_VERTEX]: 'google-vertex', @@ -234,9 +234,7 @@ async function main() { unmappedProviders.forEach((file) => console.log(` - ${file}.ts (not synced)`), ); - console.log( - ' Add these to MODELS_DEV_KEYS if they should be synced.\n', - ); + console.log(' Add these to MODELS_DEV_KEYS if they should be synced.\n'); } const modelsDevData = await fetchModelsDevData(); @@ -263,12 +261,8 @@ async function main() { if (!providerFile) { console.log(`\nāŒ ${provider}`); - console.log( - ` No matching provider file found for "${modelsDevKey}"`, - ); - console.log( - ` Available files: ${getProviderFiles().join(', ')}`, - ); + console.log(` No matching provider file found for "${modelsDevKey}"`); + console.log(` Available files: ${getProviderFiles().join(', ')}`); continue; } @@ -329,7 +323,9 @@ async function main() { } } -main().catch((error) => { - console.error('āŒ Error:', error); - process.exit(1); -}); +if (require.main === module) { + main().catch((error) => { + console.error('āŒ Error:', error); + process.exit(1); + }); +} diff --git a/packages/openops/test/ai/sync-models.test.ts b/packages/openops/test/ai/sync-models.test.ts new file mode 100644 index 0000000000..1b0fa9a4ba --- /dev/null +++ b/packages/openops/test/ai/sync-models.test.ts @@ -0,0 +1,44 @@ +import { AiProviderEnum } from '@openops/shared'; +import { MODELS_DEV_KEYS } from '../../src/lib/ai/sync-models'; + +const EXCLUDED_PROVIDERS = [ + AiProviderEnum.AZURE_OPENAI, + AiProviderEnum.DEEPINFRA, // Temporarily disabled until models.dev PR is merged + AiProviderEnum.OPENAI_COMPATIBLE, +]; + +describe('sync-models.ts provider mapping', () => { + it('should have all AI providers mapped in MODELS_DEV_KEYS or explicitly excluded', () => { + const allProviders = Object.values(AiProviderEnum); + const unmappedProviders: string[] = []; + + for (const provider of allProviders) { + if (EXCLUDED_PROVIDERS.includes(provider)) { + continue; + } + + const enumKey = Object.keys(AiProviderEnum).find( + (key) => + AiProviderEnum[key as keyof typeof AiProviderEnum] === provider, + ); + + if (!enumKey) { + continue; + } + + const isMapped = MODELS_DEV_KEYS[provider] !== undefined; + + if (!isMapped) { + unmappedProviders.push(provider); + } + } + + expect(unmappedProviders).toEqual([]); + }); + + it('should exclude providers that should not be synced from models.dev', () => { + expect(EXCLUDED_PROVIDERS).toContain(AiProviderEnum.AZURE_OPENAI); + expect(EXCLUDED_PROVIDERS).toContain(AiProviderEnum.DEEPINFRA); + expect(EXCLUDED_PROVIDERS).toContain(AiProviderEnum.OPENAI_COMPATIBLE); + }); +}); From 04709a93c0b29455559d1dadb906a071a43aba1f Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 23:43:27 +0100 Subject: [PATCH 16/22] fix time --- .github/workflows/sync-ai-models.yml | 68 ++++++++++++---------- packages/openops/src/lib/ai/sync-models.ts | 2 +- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml index 59824a55cb..b34258a33f 100644 --- a/.github/workflows/sync-ai-models.yml +++ b/.github/workflows/sync-ai-models.yml @@ -1,12 +1,8 @@ name: Sync AI Models on: - pull_request: - paths: - - '.github/workflows/sync-ai-models.yml' - - 'packages/openops/src/lib/ai/sync-models.ts' schedule: - - cron: '0 0 * * 0' + - cron: '0 12 * * 1' workflow_dispatch: permissions: @@ -45,42 +41,50 @@ jobs: echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT exit 0 - - name: Create Pull Request + - name: Create branch and commit changes if: steps.check-models.outputs.exit_code == '1' - id: create-pr - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ secrets.GITHUB_TOKEN }} - base: main - commit-message: 'Update AI provider models from models.dev' - title: 'Update AI Provider Models' - body: | - ## šŸ¤– Automated Model Sync + id: create-branch + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git switch -C "sync-ai-models-${{ github.run_number }}" + git add packages/openops/src/lib/ai/providers/ + git commit -m "Update AI provider models from models.dev" + echo "has_changes=true" >> $GITHUB_OUTPUT + + - name: Push changes and create PR + if: steps.create-branch.outputs.has_changes == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + git push origin "sync-ai-models-${{ github.run_number }}" + gh pr create \ + --title "Update AI Provider Models" \ + --body "## šŸ¤– Automated Model Sync - This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). + This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). - **Review the file changes** to see which models were added or removed for each provider. + **Review the file changes** to see which models were added or removed for each provider. - ### What to Check + ### What to Check - - āœ… New models are legitimate and available - - āœ… Removed models are actually deprecated/retired - - āœ… Model IDs match provider documentation + - āœ… New models are legitimate and available + - āœ… Removed models are actually deprecated/retired + - āœ… Model IDs match provider documentation - ### Testing Locally + ### Testing Locally - ```bash - npx tsx packages/openops/src/lib/ai/sync-models.ts - ``` + \`\`\`bash + npx tsx packages/openops/src/lib/ai/sync-models.ts + \`\`\` - --- + --- - šŸ”— **Source**: https://models.dev/api.json (MIT License) - branch: sync-ai-models-${{ github.run_number }} - delete-branch: true - labels: | - dependencies - automated + šŸ”— **Source**: https://models.dev/api.json (MIT License)" \ + --base main \ + --head "sync-ai-models-${{ github.run_number }}" \ + --label dependencies \ + --label automated - name: Summary if: always() diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index 214c29936a..8ae47a287b 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -111,7 +111,7 @@ function filterTextOnlyModels(models: Record): string[] { return true; }) .map((model) => model.id) - .sort(); + .sort((a, b) => a.localeCompare(b)); } function getProviderFilePath(providerFileName: string): string { From 24f25d221f3ed1432f902cd4372f8a8cb1cd9b42 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 23:54:49 +0100 Subject: [PATCH 17/22] clean up --- packages/openops/src/lib/ai/sync-models.ts | 272 ++++----------------- 1 file changed, 43 insertions(+), 229 deletions(-) diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index 8ae47a287b..b58c54dba7 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -1,44 +1,16 @@ -/** - * Syncs AI provider model lists from models.dev - * - * This script fetches the latest AI model data from models.dev and updates - * our provider files accordingly. It filters for text-only models and excludes - * embedding models. - * - * Data source: https://models.dev (MIT License) - * API endpoint: https://models.dev/api.json - * GitHub: https://github.com/anomalyco/models.dev - * - * Usage: - * npx tsx sync-models.ts # Check for differences - * npx tsx sync-models.ts --update # Update provider files - */ - import { AiProviderEnum } from '@openops/shared'; -import fs from 'fs'; -import path from 'path'; +import fs from 'node:fs'; +import path from 'node:path'; interface ModelData { id: string; - name: string; - family: string; modalities: { input: string[]; output: string[]; }; - cost?: { - input: number; - output: number; - }; - limit?: { - context: number; - output: number; - }; } interface ProviderData { - id: string; - name: string; models: Record; } @@ -50,7 +22,6 @@ export const MODELS_DEV_KEYS: Partial> = { [AiProviderEnum.ANTHROPIC]: 'anthropic', [AiProviderEnum.CEREBRAS]: 'cerebras', [AiProviderEnum.COHERE]: 'cohere', - // [AiProviderEnum.DEEPINFRA]: 'deepinfra', // Temporarily disabled until models.dev PR is merged [AiProviderEnum.DEEPSEEK]: 'deepseek', [AiProviderEnum.GOOGLE]: 'google', [AiProviderEnum.GOOGLE_VERTEX]: 'google-vertex', @@ -62,29 +33,6 @@ export const MODELS_DEV_KEYS: Partial> = { [AiProviderEnum.XAI]: 'xai', }; -function normalizeProviderKey(key: string): string { - return key.toLowerCase().replace(/[-_]/g, ''); -} - -function getProviderFiles(): string[] { - const providersDir = path.join(__dirname, 'providers'); - return fs - .readdirSync(providersDir) - .filter((file) => file.endsWith('.ts') && file !== 'index.ts') - .map((file) => file.replace('.ts', '')); -} - -function findMatchingProviderFile(modelsDevKey: string): string | null { - const providerFiles = getProviderFiles(); - const normalizedKey = normalizeProviderKey(modelsDevKey); - - return ( - providerFiles.find( - (file) => normalizeProviderKey(file) === normalizedKey, - ) || null - ); -} - async function fetchModelsDevData(): Promise { const response = await fetch('https://models.dev/api.json'); if (!response.ok) { @@ -93,234 +41,100 @@ async function fetchModelsDevData(): Promise { return response.json(); } -function isEmbeddingModel(modelId: string): boolean { - return modelId.toLowerCase().includes('embedding'); -} - function filterTextOnlyModels(models: Record): string[] { return Object.values(models) .filter((model) => { const outputModalities = model.modalities?.output || []; const isTextOnly = outputModalities.length === 1 && outputModalities[0] === 'text'; - - if (!isTextOnly) return false; - - if (isEmbeddingModel(model.id)) return false; - - return true; + const isEmbedding = model.id.toLowerCase().includes('embedding'); + return isTextOnly && !isEmbedding; }) .map((model) => model.id) .sort((a, b) => a.localeCompare(b)); } -function getProviderFilePath(providerFileName: string): string { - return path.join(__dirname, 'providers', `${providerFileName}.ts`); -} - -function getCurrentModels(providerFileName: string): string[] { - const filePath = getProviderFilePath(providerFileName); - - if (!fs.existsSync(filePath)) { - console.warn(`āš ļø Provider file not found: ${filePath}`); - return []; - } +function getCurrentModels(providerFile: string): string[] { + const filePath = path.join(__dirname, 'providers', `${providerFile}.ts`); + if (!fs.existsSync(filePath)) return []; const content = fs.readFileSync(filePath, 'utf-8'); + const match = content.match(/const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/); + if (!match) return []; - const modelsArrayMatch = content.match( - /const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/, - ); - - if (!modelsArrayMatch) { - console.warn(`āš ļø Could not find models array in ${providerFileName}.ts`); - return []; - } - - const modelsString = modelsArrayMatch[1]; - const models = modelsString + return match[1] .split(',') - .map((line) => { - const match = line.match(/['"]([^'"]+)['"]/); - return match ? match[1] : null; - }) - .filter((model): model is string => model !== null); - - return models.sort(); + .map((line) => line.match(/['"]([^'"]+)['"]/)?.[1]) + .filter((model): model is string => model !== null) + .sort(); } -function updateProviderFile( - providerFileName: string, - newModels: string[], -): void { - const filePath = getProviderFilePath(providerFileName); - - if (!fs.existsSync(filePath)) { - console.warn(`āš ļø Cannot update: Provider file not found: ${filePath}`); - return; - } - +function updateProviderFile(providerFile: string, models: string[]): void { + const filePath = path.join(__dirname, 'providers', `${providerFile}.ts`); const content = fs.readFileSync(filePath, 'utf-8'); + const match = content.match(/const\s+(\w+Models)\s*=\s*\[([\s\S]*?)\];/); + if (!match) return; - const modelsArrayMatch = content.match( - /const\s+(\w+Models)\s*=\s*\[([\s\S]*?)\];/, - ); - - if (!modelsArrayMatch) { - console.warn( - `āš ļø Cannot update: Could not find models array in ${providerFileName}.ts`, - ); - return; - } - - const arrayName = modelsArrayMatch[1]; - - const formattedModels = newModels.map((model) => ` '${model}',`).join('\n'); - + const arrayName = match[1]; + const formattedModels = models.map((model) => ` '${model}',`).join('\n'); const newArray = `const ${arrayName} = [\n${formattedModels}\n];`; - const updatedContent = content.replace( /const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/, newArray, ); fs.writeFileSync(filePath, updatedContent, 'utf-8'); - - console.log(` āœ… Updated ${providerFileName}.ts`); } -function compareModels( - current: string[], - latest: string[], -): { - added: string[]; - removed: string[]; - unchanged: string[]; -} { - const currentSet = new Set(current); - const latestSet = new Set(latest); - - const added = latest.filter((model) => !currentSet.has(model)); - const removed = current.filter((model) => !latestSet.has(model)); - const unchanged = current.filter((model) => latestSet.has(model)); - - return { added, removed, unchanged }; -} +function findProviderFile(modelsDevKey: string): string | null { + const providersDir = path.join(__dirname, 'providers'); + const files = fs + .readdirSync(providersDir) + .filter((file) => file.endsWith('.ts') && file !== 'index.ts') + .map((file) => file.replace('.ts', '')); -function checkForUnmappedProviders(): string[] { - const providerFiles = getProviderFiles(); - const mappedKeys = Object.values(MODELS_DEV_KEYS).filter( - (key): key is string => key !== undefined, + const normalizedKey = modelsDevKey.toLowerCase().replaceAll(/[-_]/g, ''); + return ( + files.find( + (file) => file.toLowerCase().replaceAll(/[-_]/g, '') === normalizedKey, + ) || null ); - - const unmappedFiles = providerFiles.filter((file) => { - const normalizedFile = normalizeProviderKey(file); - return !mappedKeys.some( - (key) => normalizeProviderKey(key) === normalizedFile, - ); - }); - - return unmappedFiles; } async function main() { const shouldUpdate = process.argv.includes('--update'); - - console.log('šŸ”„ Fetching models from models.dev...\n'); - - const unmappedProviders = checkForUnmappedProviders(); - if (unmappedProviders.length > 0) { - console.log('āš ļø Warning: Found provider files not in MODELS_DEV_KEYS:'); - unmappedProviders.forEach((file) => - console.log(` - ${file}.ts (not synced)`), - ); - console.log(' Add these to MODELS_DEV_KEYS if they should be synced.\n'); - } - const modelsDevData = await fetchModelsDevData(); - console.log('šŸ“Š Comparing models for each provider:\n'); - console.log('='.repeat(80)); - - let totalAdded = 0; - let totalRemoved = 0; let hasChanges = false; for (const [provider, modelsDevKey] of Object.entries(MODELS_DEV_KEYS)) { if (!modelsDevKey) continue; const providerData = modelsDevData[modelsDevKey]; + if (!providerData) continue; - if (!providerData) { - console.log(`\nāŒ ${provider}`); - console.log(` Provider "${modelsDevKey}" not found in models.dev`); - continue; - } - - const providerFile = findMatchingProviderFile(modelsDevKey); - - if (!providerFile) { - console.log(`\nāŒ ${provider}`); - console.log(` No matching provider file found for "${modelsDevKey}"`); - console.log(` Available files: ${getProviderFiles().join(', ')}`); - continue; - } + const providerFile = findProviderFile(modelsDevKey); + if (!providerFile) continue; const latestModels = filterTextOnlyModels(providerData.models); const currentModels = getCurrentModels(providerFile); - const diff = compareModels(currentModels, latestModels); - - const hasProviderChanges = diff.added.length > 0 || diff.removed.length > 0; + const added = latestModels.filter((m) => !currentModels.includes(m)); + const removed = currentModels.filter((m) => !latestModels.includes(m)); - if (hasProviderChanges) { + if (added.length > 0 || removed.length > 0) { hasChanges = true; - } + console.log(`${provider}:`); + if (added.length > 0) console.log(` +${added.length}`); + if (removed.length > 0) console.log(` -${removed.length}`); - const icon = hasProviderChanges ? 'šŸ”„' : 'āœ…'; - console.log(`\n${icon} ${provider} (${modelsDevKey})`); - console.log(` File: ${providerFile}.ts`); - console.log(` Current: ${currentModels.length} models`); - console.log(` Latest: ${latestModels.length} models`); - - if (diff.added.length > 0) { - console.log(` āž• Added (${diff.added.length}):`); - diff.added.forEach((model) => console.log(` - ${model}`)); - totalAdded += diff.added.length; - } - - if (diff.removed.length > 0) { - console.log(` āž– Removed (${diff.removed.length}):`); - diff.removed.forEach((model) => console.log(` - ${model}`)); - totalRemoved += diff.removed.length; - } - - if (!hasProviderChanges) { - console.log(` āœ“ No changes`); - } - - if (hasProviderChanges && shouldUpdate) { - updateProviderFile(providerFile, latestModels); + if (shouldUpdate) { + updateProviderFile(providerFile, latestModels); + } } } - console.log('\n' + '='.repeat(80)); - console.log('\nšŸ“ˆ Summary:'); - console.log(` Total models added: ${totalAdded}`); - console.log(` Total models removed: ${totalRemoved}`); - - if (hasChanges) { - if (shouldUpdate) { - console.log('\nāœ… Provider files updated successfully!'); - console.log(' Please review the changes and commit.'); - } else { - console.log('\nāš ļø Changes detected! Run with --update to apply them.'); - } - process.exit(1); - } else { - console.log('\nāœ… All providers are up to date!'); - process.exit(0); - } + process.exit(hasChanges ? 1 : 0); } if (require.main === module) { From 06454c4082560b05b30e63c85c64b1ff5e07ac0a Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Fri, 13 Mar 2026 23:55:17 +0100 Subject: [PATCH 18/22] add back --- packages/openops/src/lib/ai/sync-models.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index b58c54dba7..ca2710c9e6 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -1,3 +1,19 @@ +/** + * Syncs AI provider model lists from models.dev + * + * This script fetches the latest AI model data from models.dev and updates + * our provider files accordingly. It filters for text-only models and excludes + * embedding models. + * + * Data source: https://models.dev (MIT License) + * API endpoint: https://models.dev/api.json + * GitHub: https://github.com/anomalyco/models.dev + * + * Usage: + * npx tsx sync-models.ts # Check for differences + * npx tsx sync-models.ts --update # Update provider files + */ + import { AiProviderEnum } from '@openops/shared'; import fs from 'node:fs'; import path from 'node:path'; @@ -22,6 +38,7 @@ export const MODELS_DEV_KEYS: Partial> = { [AiProviderEnum.ANTHROPIC]: 'anthropic', [AiProviderEnum.CEREBRAS]: 'cerebras', [AiProviderEnum.COHERE]: 'cohere', + // [AiProviderEnum.DEEPINFRA]: 'deepinfra', // Temporarily disabled until models.dev PR is merged [AiProviderEnum.DEEPSEEK]: 'deepseek', [AiProviderEnum.GOOGLE]: 'google', [AiProviderEnum.GOOGLE_VERTEX]: 'google-vertex', From 939a7d742caad1affd105f3a75daed4d6c6ad862 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Sat, 14 Mar 2026 14:49:05 +0100 Subject: [PATCH 19/22] rm github action --- .github/workflows/sync-ai-models.yml | 99 ------- packages/openops/src/lib/ai/sync-models.ts | 317 ++++++++++++++------- 2 files changed, 219 insertions(+), 197 deletions(-) delete mode 100644 .github/workflows/sync-ai-models.yml diff --git a/.github/workflows/sync-ai-models.yml b/.github/workflows/sync-ai-models.yml deleted file mode 100644 index b34258a33f..0000000000 --- a/.github/workflows/sync-ai-models.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Sync AI Models - -on: - schedule: - - cron: '0 12 * * 1' - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -jobs: - sync-models: - name: Sync AI Provider Models - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.2 - - - name: Restore node_modules cache - id: node-modules-cache - uses: actions/cache/restore@v5.0.3 - with: - path: node_modules - key: node-modules-cache-${{ hashFiles('package-lock.json', '.npmrc') }} - - - name: Install dependencies - if: steps.node-modules-cache.outputs.cache-hit != 'true' - run: npm ci --no-audit --no-fund - - - name: Build shared package - run: npx nx run shared:build - - - name: Check for model updates - id: check-models - continue-on-error: true - run: | - set +e - npx tsx --tsconfig tsconfig.base.json packages/openops/src/lib/ai/sync-models.ts --update - EXIT_CODE=$? - set -e - echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT - exit 0 - - - name: Create branch and commit changes - if: steps.check-models.outputs.exit_code == '1' - id: create-branch - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git switch -C "sync-ai-models-${{ github.run_number }}" - git add packages/openops/src/lib/ai/providers/ - git commit -m "Update AI provider models from models.dev" - echo "has_changes=true" >> $GITHUB_OUTPUT - - - name: Push changes and create PR - if: steps.create-branch.outputs.has_changes == 'true' - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - git push origin "sync-ai-models-${{ github.run_number }}" - gh pr create \ - --title "Update AI Provider Models" \ - --body "## šŸ¤– Automated Model Sync - - This PR updates AI provider model lists based on the latest data from [models.dev](https://models.dev). - - **Review the file changes** to see which models were added or removed for each provider. - - ### What to Check - - - āœ… New models are legitimate and available - - āœ… Removed models are actually deprecated/retired - - āœ… Model IDs match provider documentation - - ### Testing Locally - - \`\`\`bash - npx tsx packages/openops/src/lib/ai/sync-models.ts - \`\`\` - - --- - - šŸ”— **Source**: https://models.dev/api.json (MIT License)" \ - --base main \ - --head "sync-ai-models-${{ github.run_number }}" \ - --label dependencies \ - --label automated - - - name: Summary - if: always() - run: | - if [ "${{ steps.check-models.outputs.exit_code }}" == "0" ]; then - echo "āœ… All AI provider models are up to date!" >> $GITHUB_STEP_SUMMARY - elif [ "${{ steps.check-models.outputs.exit_code }}" == "1" ]; then - echo "šŸ”„ Model updates detected and PR created" >> $GITHUB_STEP_SUMMARY - else - echo "āŒ Sync script failed with unexpected error" >> $GITHUB_STEP_SUMMARY - exit 1 - fi diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index ca2710c9e6..5a9216c51f 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -1,13 +1,9 @@ /** - * Syncs AI provider model lists from models.dev + * Syncs AI provider model lists from @ai-sdk type definitions * - * This script fetches the latest AI model data from models.dev and updates - * our provider files accordingly. It filters for text-only models and excludes - * embedding models. - * - * Data source: https://models.dev (MIT License) - * API endpoint: https://models.dev/api.json - * GitHub: https://github.com/anomalyco/models.dev + * This script fetches the latest TypeScript type definitions from unpkg for each + * @ai-sdk provider package and extracts the valid model IDs from the union types. + * Models confirmed to work with the AI SDK are included. * * Usage: * npx tsx sync-models.ts # Check for differences @@ -18,136 +14,261 @@ import { AiProviderEnum } from '@openops/shared'; import fs from 'node:fs'; import path from 'node:path'; -interface ModelData { - id: string; - modalities: { - input: string[]; - output: string[]; - }; -} - -interface ProviderData { - models: Record; +interface AiSdkConfig { + package: string; + typeName: string; + providerFile: string; + distPath?: string; + arrayName?: string; + excludedModels?: string[]; + additionalArrays?: Array<{ + distPath: string; + typeName: string; + arrayName: string; + excludedModels?: string[]; + }>; } -interface ModelsDevAPI { - [providerKey: string]: ProviderData; -} - -export const MODELS_DEV_KEYS: Partial> = { - [AiProviderEnum.ANTHROPIC]: 'anthropic', - [AiProviderEnum.CEREBRAS]: 'cerebras', - [AiProviderEnum.COHERE]: 'cohere', - // [AiProviderEnum.DEEPINFRA]: 'deepinfra', // Temporarily disabled until models.dev PR is merged - [AiProviderEnum.DEEPSEEK]: 'deepseek', - [AiProviderEnum.GOOGLE]: 'google', - [AiProviderEnum.GOOGLE_VERTEX]: 'google-vertex', - [AiProviderEnum.GROQ]: 'groq', - [AiProviderEnum.MISTRAL]: 'mistral', - [AiProviderEnum.OPENAI]: 'openai', - [AiProviderEnum.PERPLEXITY]: 'perplexity', - [AiProviderEnum.TOGETHER_AI]: 'togetherai', - [AiProviderEnum.XAI]: 'xai', +export const AI_SDK_CONFIGS: Partial> = { + [AiProviderEnum.ANTHROPIC]: { + package: 'anthropic', + typeName: 'AnthropicMessagesModelId', + providerFile: 'anthropic', + }, + [AiProviderEnum.CEREBRAS]: { + package: 'cerebras', + typeName: 'CerebrasChatModelId', + providerFile: 'cerebras', + }, + [AiProviderEnum.COHERE]: { + package: 'cohere', + typeName: 'CohereChatModelId', + providerFile: 'cohere', + }, + [AiProviderEnum.DEEPSEEK]: { + package: 'deepseek', + typeName: 'DeepSeekChatModelId', + providerFile: 'deep-seek', + }, + [AiProviderEnum.GOOGLE]: { + package: 'google', + typeName: 'GoogleGenerativeAIModelId', + providerFile: 'google', + }, + [AiProviderEnum.GOOGLE_VERTEX]: { + package: 'google-vertex', + typeName: 'GoogleVertexModelId', + providerFile: 'google-vertex', + excludedModels: [ + 'gemini-1.0-pro', + 'gemini-1.0-pro-001', + 'gemini-1.0-pro-002', + 'gemini-1.0-pro-vision-001', + 'gemini-1.5-flash-001', + 'gemini-1.5-flash-002', + 'gemini-1.5-pro-001', + 'gemini-1.5-pro-002', + ], + additionalArrays: [ + { + distPath: 'dist/anthropic/index.d.ts', + typeName: 'GoogleVertexAnthropicMessagesModelId', + arrayName: 'googleVertexClaudeModels', + }, + ], + }, + [AiProviderEnum.GROQ]: { + package: 'groq', + typeName: 'GroqChatModelId', + providerFile: 'groq', + }, + [AiProviderEnum.MISTRAL]: { + package: 'mistral', + typeName: 'MistralChatModelId', + providerFile: 'mistral', + }, + [AiProviderEnum.OPENAI]: { + package: 'openai', + typeName: 'OpenAIChatModelId', + providerFile: 'openai', + }, + [AiProviderEnum.PERPLEXITY]: { + package: 'perplexity', + typeName: 'PerplexityLanguageModelId', + providerFile: 'perplexity', + }, + [AiProviderEnum.TOGETHER_AI]: { + package: 'togetherai', + typeName: 'TogetherAIChatModelId', + providerFile: 'together-ai', + }, + [AiProviderEnum.XAI]: { + package: 'xai', + typeName: 'XaiChatModelId', + providerFile: 'xai', + }, }; -async function fetchModelsDevData(): Promise { - const response = await fetch('https://models.dev/api.json'); +const NON_CHAT_KEYWORDS = [ + 'guard', + 'embed', + 'audio', + 'tts', + 'native-audio', + 'imagen', + 'search-preview', + 'aqa', + 'robotics', + 'computer-use', + 'nano-banana', + 'veo', + '-image', +]; + +async function fetchAiSdkModels( + pkg: string, + typeName: string, + distPath = 'dist/index.d.ts', + excludedModels: string[] = [], +): Promise { + const url = `https://unpkg.com/@ai-sdk/${pkg}@latest/${distPath}`; + const response = await fetch(url); if (!response.ok) { - throw new Error(`Failed to fetch models.dev API: ${response.statusText}`); + throw new Error( + `Failed to fetch @ai-sdk/${pkg} types: ${response.statusText}`, + ); + } + + const dts = await response.text(); + const pattern = new RegExp(`type\\s+${typeName}\\s*=\\s*([^;]+);`, 's'); + const match = dts.match(pattern); + if (!match) { + throw new Error(`Could not find type ${typeName} in @ai-sdk/${pkg}`); } - return response.json(); -} -function filterTextOnlyModels(models: Record): string[] { - return Object.values(models) - .filter((model) => { - const outputModalities = model.modalities?.output || []; - const isTextOnly = - outputModalities.length === 1 && outputModalities[0] === 'text'; - const isEmbedding = model.id.toLowerCase().includes('embedding'); - return isTextOnly && !isEmbedding; - }) - .map((model) => model.id) + return [...match[1].matchAll(/'([^']+)'/g)] + .map((m) => m[1]) + .filter( + (id) => + !NON_CHAT_KEYWORDS.some((kw) => id.toLowerCase().includes(kw)) && + !excludedModels.includes(id), + ) .sort((a, b) => a.localeCompare(b)); } -function getCurrentModels(providerFile: string): string[] { +function getCurrentModels(providerFile: string, arrayName?: string): string[] { const filePath = path.join(__dirname, 'providers', `${providerFile}.ts`); if (!fs.existsSync(filePath)) return []; const content = fs.readFileSync(filePath, 'utf-8'); - const match = content.match(/const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/); + const pattern = arrayName + ? new RegExp(`const\\s+${arrayName}\\s*=\\s*\\[([\\s\\S]*?)\\];`) + : /const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/; + const match = content.match(pattern); if (!match) return []; return match[1] .split(',') .map((line) => line.match(/['"]([^'"]+)['"]/)?.[1]) - .filter((model): model is string => model !== null) + .filter((model): model is string => model != null) .sort(); } -function updateProviderFile(providerFile: string, models: string[]): void { +function updateProviderFile( + providerFile: string, + models: string[], + arrayName?: string, +): void { const filePath = path.join(__dirname, 'providers', `${providerFile}.ts`); const content = fs.readFileSync(filePath, 'utf-8'); - const match = content.match(/const\s+(\w+Models)\s*=\s*\[([\s\S]*?)\];/); + const pattern = arrayName + ? new RegExp(`const\\s+(${arrayName})\\s*=\\s*\\[([\\s\\S]*?)\\];`) + : /const\s+(\w+Models)\s*=\s*\[([\s\S]*?)\];/; + const match = content.match(pattern); if (!match) return; - const arrayName = match[1]; + const resolvedArrayName = match[1]; const formattedModels = models.map((model) => ` '${model}',`).join('\n'); - const newArray = `const ${arrayName} = [\n${formattedModels}\n];`; - const updatedContent = content.replace( - /const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/, - newArray, - ); + const newArray = `const ${resolvedArrayName} = [\n${formattedModels}\n];`; + const updatedContent = content.replace(pattern, newArray); fs.writeFileSync(filePath, updatedContent, 'utf-8'); } -function findProviderFile(modelsDevKey: string): string | null { - const providersDir = path.join(__dirname, 'providers'); - const files = fs - .readdirSync(providersDir) - .filter((file) => file.endsWith('.ts') && file !== 'index.ts') - .map((file) => file.replace('.ts', '')); - - const normalizedKey = modelsDevKey.toLowerCase().replaceAll(/[-_]/g, ''); - return ( - files.find( - (file) => file.toLowerCase().replaceAll(/[-_]/g, '') === normalizedKey, - ) || null - ); -} +async function syncConfig( + label: string, + pkg: string, + typeName: string, + providerFile: string, + distPath: string | undefined, + arrayName: string | undefined, + excludedModels: string[] | undefined, + shouldUpdate: boolean, +): Promise { + let latestModels: string[]; + try { + latestModels = await fetchAiSdkModels( + pkg, + typeName, + distPath, + excludedModels, + ); + } catch (error) { + console.error(`Skipping ${label}: ${(error as Error).message}`); + return false; + } -async function main() { - const shouldUpdate = process.argv.includes('--update'); - const modelsDevData = await fetchModelsDevData(); + const currentModels = getCurrentModels(providerFile, arrayName); + const added = latestModels.filter((m) => !currentModels.includes(m)); + const removed = currentModels.filter((m) => !latestModels.includes(m)); - let hasChanges = false; + if (added.length === 0 && removed.length === 0) { + return false; + } + + console.log(`${label}:`); + if (added.length > 0) console.log(` +${added.length}`); + if (removed.length > 0) console.log(` -${removed.length}`); - for (const [provider, modelsDevKey] of Object.entries(MODELS_DEV_KEYS)) { - if (!modelsDevKey) continue; + if (shouldUpdate) { + updateProviderFile(providerFile, latestModels, arrayName); + } - const providerData = modelsDevData[modelsDevKey]; - if (!providerData) continue; + return true; +} - const providerFile = findProviderFile(modelsDevKey); - if (!providerFile) continue; +async function main() { + const shouldUpdate = process.argv.includes('--update'); - const latestModels = filterTextOnlyModels(providerData.models); - const currentModels = getCurrentModels(providerFile); + let hasChanges = false; - const added = latestModels.filter((m) => !currentModels.includes(m)); - const removed = currentModels.filter((m) => !latestModels.includes(m)); + for (const [provider, config] of Object.entries(AI_SDK_CONFIGS)) { + if (!config) continue; - if (added.length > 0 || removed.length > 0) { - hasChanges = true; - console.log(`${provider}:`); - if (added.length > 0) console.log(` +${added.length}`); - if (removed.length > 0) console.log(` -${removed.length}`); + const changed = await syncConfig( + provider, + config.package, + config.typeName, + config.providerFile, + config.distPath, + config.arrayName, + config.excludedModels, + shouldUpdate, + ); + if (changed) hasChanges = true; - if (shouldUpdate) { - updateProviderFile(providerFile, latestModels); - } + for (const extra of config.additionalArrays ?? []) { + const extraChanged = await syncConfig( + `${provider} (${extra.arrayName})`, + config.package, + extra.typeName, + config.providerFile, + extra.distPath, + extra.arrayName, + extra.excludedModels, + shouldUpdate, + ); + if (extraChanged) hasChanges = true; } } From c29db4b6a18ffdfc1a73a26b2385983f8040ff12 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Sat, 14 Mar 2026 14:50:48 +0100 Subject: [PATCH 20/22] rm --- packages/openops/test/ai/sync-models.test.ts | 44 -------------------- 1 file changed, 44 deletions(-) delete mode 100644 packages/openops/test/ai/sync-models.test.ts diff --git a/packages/openops/test/ai/sync-models.test.ts b/packages/openops/test/ai/sync-models.test.ts deleted file mode 100644 index 1b0fa9a4ba..0000000000 --- a/packages/openops/test/ai/sync-models.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { AiProviderEnum } from '@openops/shared'; -import { MODELS_DEV_KEYS } from '../../src/lib/ai/sync-models'; - -const EXCLUDED_PROVIDERS = [ - AiProviderEnum.AZURE_OPENAI, - AiProviderEnum.DEEPINFRA, // Temporarily disabled until models.dev PR is merged - AiProviderEnum.OPENAI_COMPATIBLE, -]; - -describe('sync-models.ts provider mapping', () => { - it('should have all AI providers mapped in MODELS_DEV_KEYS or explicitly excluded', () => { - const allProviders = Object.values(AiProviderEnum); - const unmappedProviders: string[] = []; - - for (const provider of allProviders) { - if (EXCLUDED_PROVIDERS.includes(provider)) { - continue; - } - - const enumKey = Object.keys(AiProviderEnum).find( - (key) => - AiProviderEnum[key as keyof typeof AiProviderEnum] === provider, - ); - - if (!enumKey) { - continue; - } - - const isMapped = MODELS_DEV_KEYS[provider] !== undefined; - - if (!isMapped) { - unmappedProviders.push(provider); - } - } - - expect(unmappedProviders).toEqual([]); - }); - - it('should exclude providers that should not be synced from models.dev', () => { - expect(EXCLUDED_PROVIDERS).toContain(AiProviderEnum.AZURE_OPENAI); - expect(EXCLUDED_PROVIDERS).toContain(AiProviderEnum.DEEPINFRA); - expect(EXCLUDED_PROVIDERS).toContain(AiProviderEnum.OPENAI_COMPATIBLE); - }); -}); From 0aee84332bc251a4e0d9288479fd6c9db9b5b05f Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Sat, 14 Mar 2026 14:57:24 +0100 Subject: [PATCH 21/22] rm extra array --- packages/openops/src/lib/ai/sync-models.ts | 207 ++++++++------------- 1 file changed, 82 insertions(+), 125 deletions(-) diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index 5a9216c51f..e789109826 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -14,98 +14,96 @@ import { AiProviderEnum } from '@openops/shared'; import fs from 'node:fs'; import path from 'node:path'; -interface AiSdkConfig { - package: string; +interface TypeSource { typeName: string; - providerFile: string; distPath?: string; - arrayName?: string; excludedModels?: string[]; - additionalArrays?: Array<{ - distPath: string; - typeName: string; - arrayName: string; - excludedModels?: string[]; - }>; +} + +interface AiSdkConfig { + package: string; + providerFile: string; + typeSources: TypeSource[]; } export const AI_SDK_CONFIGS: Partial> = { [AiProviderEnum.ANTHROPIC]: { package: 'anthropic', - typeName: 'AnthropicMessagesModelId', providerFile: 'anthropic', + typeSources: [{ typeName: 'AnthropicMessagesModelId' }], }, [AiProviderEnum.CEREBRAS]: { package: 'cerebras', - typeName: 'CerebrasChatModelId', providerFile: 'cerebras', + typeSources: [{ typeName: 'CerebrasChatModelId' }], }, [AiProviderEnum.COHERE]: { package: 'cohere', - typeName: 'CohereChatModelId', providerFile: 'cohere', + typeSources: [{ typeName: 'CohereChatModelId' }], }, [AiProviderEnum.DEEPSEEK]: { package: 'deepseek', - typeName: 'DeepSeekChatModelId', providerFile: 'deep-seek', + typeSources: [{ typeName: 'DeepSeekChatModelId' }], }, [AiProviderEnum.GOOGLE]: { package: 'google', - typeName: 'GoogleGenerativeAIModelId', providerFile: 'google', + typeSources: [{ typeName: 'GoogleGenerativeAIModelId' }], }, [AiProviderEnum.GOOGLE_VERTEX]: { package: 'google-vertex', - typeName: 'GoogleVertexModelId', providerFile: 'google-vertex', - excludedModels: [ - 'gemini-1.0-pro', - 'gemini-1.0-pro-001', - 'gemini-1.0-pro-002', - 'gemini-1.0-pro-vision-001', - 'gemini-1.5-flash-001', - 'gemini-1.5-flash-002', - 'gemini-1.5-pro-001', - 'gemini-1.5-pro-002', - ], - additionalArrays: [ + typeSources: [ + { + typeName: 'GoogleVertexModelId', + excludedModels: [ + 'gemini-1.0-pro', + 'gemini-1.0-pro-001', + 'gemini-1.0-pro-002', + 'gemini-1.0-pro-vision-001', + 'gemini-1.5-flash-001', + 'gemini-1.5-flash-002', + 'gemini-1.5-pro-001', + 'gemini-1.5-pro-002', + ], + }, { - distPath: 'dist/anthropic/index.d.ts', typeName: 'GoogleVertexAnthropicMessagesModelId', - arrayName: 'googleVertexClaudeModels', + distPath: 'dist/anthropic/index.d.ts', }, ], }, [AiProviderEnum.GROQ]: { package: 'groq', - typeName: 'GroqChatModelId', providerFile: 'groq', + typeSources: [{ typeName: 'GroqChatModelId' }], }, [AiProviderEnum.MISTRAL]: { package: 'mistral', - typeName: 'MistralChatModelId', providerFile: 'mistral', + typeSources: [{ typeName: 'MistralChatModelId' }], }, [AiProviderEnum.OPENAI]: { package: 'openai', - typeName: 'OpenAIChatModelId', providerFile: 'openai', + typeSources: [{ typeName: 'OpenAIChatModelId' }], }, [AiProviderEnum.PERPLEXITY]: { package: 'perplexity', - typeName: 'PerplexityLanguageModelId', providerFile: 'perplexity', + typeSources: [{ typeName: 'PerplexityLanguageModelId' }], }, [AiProviderEnum.TOGETHER_AI]: { package: 'togetherai', - typeName: 'TogetherAIChatModelId', providerFile: 'together-ai', + typeSources: [{ typeName: 'TogetherAIChatModelId' }], }, [AiProviderEnum.XAI]: { package: 'xai', - typeName: 'XaiChatModelId', providerFile: 'xai', + typeSources: [{ typeName: 'XaiChatModelId' }], }, }; @@ -127,10 +125,9 @@ const NON_CHAT_KEYWORDS = [ async function fetchAiSdkModels( pkg: string, - typeName: string, - distPath = 'dist/index.d.ts', - excludedModels: string[] = [], + source: TypeSource, ): Promise { + const distPath = source.distPath ?? 'dist/index.d.ts'; const url = `https://unpkg.com/@ai-sdk/${pkg}@latest/${distPath}`; const response = await fetch(url); if (!response.ok) { @@ -140,31 +137,31 @@ async function fetchAiSdkModels( } const dts = await response.text(); - const pattern = new RegExp(`type\\s+${typeName}\\s*=\\s*([^;]+);`, 's'); + const pattern = new RegExp( + `type\\s+${source.typeName}\\s*=\\s*([^;]+);`, + 's', + ); const match = dts.match(pattern); if (!match) { - throw new Error(`Could not find type ${typeName} in @ai-sdk/${pkg}`); + throw new Error(`Could not find type ${source.typeName} in @ai-sdk/${pkg}`); } + const excluded = source.excludedModels ?? []; return [...match[1].matchAll(/'([^']+)'/g)] .map((m) => m[1]) .filter( (id) => !NON_CHAT_KEYWORDS.some((kw) => id.toLowerCase().includes(kw)) && - !excludedModels.includes(id), - ) - .sort((a, b) => a.localeCompare(b)); + !excluded.includes(id), + ); } -function getCurrentModels(providerFile: string, arrayName?: string): string[] { +function getCurrentModels(providerFile: string): string[] { const filePath = path.join(__dirname, 'providers', `${providerFile}.ts`); if (!fs.existsSync(filePath)) return []; const content = fs.readFileSync(filePath, 'utf-8'); - const pattern = arrayName - ? new RegExp(`const\\s+${arrayName}\\s*=\\s*\\[([\\s\\S]*?)\\];`) - : /const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/; - const match = content.match(pattern); + const match = content.match(/const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/); if (!match) return []; return match[1] @@ -174,69 +171,23 @@ function getCurrentModels(providerFile: string, arrayName?: string): string[] { .sort(); } -function updateProviderFile( - providerFile: string, - models: string[], - arrayName?: string, -): void { +function updateProviderFile(providerFile: string, models: string[]): void { const filePath = path.join(__dirname, 'providers', `${providerFile}.ts`); const content = fs.readFileSync(filePath, 'utf-8'); - const pattern = arrayName - ? new RegExp(`const\\s+(${arrayName})\\s*=\\s*\\[([\\s\\S]*?)\\];`) - : /const\s+(\w+Models)\s*=\s*\[([\s\S]*?)\];/; - const match = content.match(pattern); + const match = content.match(/const\s+(\w+Models)\s*=\s*\[([\s\S]*?)\];/); if (!match) return; - const resolvedArrayName = match[1]; + const arrayName = match[1]; const formattedModels = models.map((model) => ` '${model}',`).join('\n'); - const newArray = `const ${resolvedArrayName} = [\n${formattedModels}\n];`; - const updatedContent = content.replace(pattern, newArray); + const newArray = `const ${arrayName} = [\n${formattedModels}\n];`; + const updatedContent = content.replace( + /const\s+\w+Models\s*=\s*\[([\s\S]*?)\];/, + newArray, + ); fs.writeFileSync(filePath, updatedContent, 'utf-8'); } -async function syncConfig( - label: string, - pkg: string, - typeName: string, - providerFile: string, - distPath: string | undefined, - arrayName: string | undefined, - excludedModels: string[] | undefined, - shouldUpdate: boolean, -): Promise { - let latestModels: string[]; - try { - latestModels = await fetchAiSdkModels( - pkg, - typeName, - distPath, - excludedModels, - ); - } catch (error) { - console.error(`Skipping ${label}: ${(error as Error).message}`); - return false; - } - - const currentModels = getCurrentModels(providerFile, arrayName); - const added = latestModels.filter((m) => !currentModels.includes(m)); - const removed = currentModels.filter((m) => !latestModels.includes(m)); - - if (added.length === 0 && removed.length === 0) { - return false; - } - - console.log(`${label}:`); - if (added.length > 0) console.log(` +${added.length}`); - if (removed.length > 0) console.log(` -${removed.length}`); - - if (shouldUpdate) { - updateProviderFile(providerFile, latestModels, arrayName); - } - - return true; -} - async function main() { const shouldUpdate = process.argv.includes('--update'); @@ -245,30 +196,36 @@ async function main() { for (const [provider, config] of Object.entries(AI_SDK_CONFIGS)) { if (!config) continue; - const changed = await syncConfig( - provider, - config.package, - config.typeName, - config.providerFile, - config.distPath, - config.arrayName, - config.excludedModels, - shouldUpdate, - ); - if (changed) hasChanges = true; - - for (const extra of config.additionalArrays ?? []) { - const extraChanged = await syncConfig( - `${provider} (${extra.arrayName})`, - config.package, - extra.typeName, - config.providerFile, - extra.distPath, - extra.arrayName, - extra.excludedModels, - shouldUpdate, + let latestModels: string[]; + try { + const results = await Promise.all( + config.typeSources.map((source) => + fetchAiSdkModels(config.package, source), + ), + ); + latestModels = [...new Set(results.flat())].sort((a, b) => + a.localeCompare(b), ); - if (extraChanged) hasChanges = true; + } catch (error) { + console.error(`Skipping ${provider}: ${(error as Error).message}`); + continue; + } + + const currentModels = getCurrentModels(config.providerFile); + const added = latestModels.filter((m) => !currentModels.includes(m)); + const removed = currentModels.filter((m) => !latestModels.includes(m)); + + if (added.length === 0 && removed.length === 0) { + continue; + } + + hasChanges = true; + console.log(`${provider}:`); + if (added.length > 0) console.log(` +${added.length}`); + if (removed.length > 0) console.log(` -${removed.length}`); + + if (shouldUpdate) { + updateProviderFile(config.providerFile, latestModels); } } From 5a624c2160949eec3e8e9395f8e6fdd797941812 Mon Sep 17 00:00:00 2001 From: BigFluffyCookie Date: Sat, 14 Mar 2026 19:53:50 +0100 Subject: [PATCH 22/22] clean --- packages/openops/src/lib/ai/sync-models.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/openops/src/lib/ai/sync-models.ts b/packages/openops/src/lib/ai/sync-models.ts index e789109826..6c33be131f 100644 --- a/packages/openops/src/lib/ai/sync-models.ts +++ b/packages/openops/src/lib/ai/sync-models.ts @@ -138,10 +138,10 @@ async function fetchAiSdkModels( const dts = await response.text(); const pattern = new RegExp( - `type\\s+${source.typeName}\\s*=\\s*([^;]+);`, + String.raw`type\s+${source.typeName}\s*=\s*([^;]+);`, 's', ); - const match = dts.match(pattern); + const match = pattern.exec(dts); if (!match) { throw new Error(`Could not find type ${source.typeName} in @ai-sdk/${pkg}`); }