From 3fb2dc3988e9743e602dbb3529307ae49fcad18a Mon Sep 17 00:00:00 2001 From: jottakka Date: Tue, 3 Mar 2026 17:21:33 -0300 Subject: [PATCH 1/8] Some tweaks in tool docs --- .github/workflows/check-toolkit-coverage.yml | 161 ++++++++ .../integrations/create-category-meta.ts | 48 +++ .../integrations/customer-support/_meta.tsx | 40 +- .../integrations/databases/_meta.tsx | 36 +- .../integrations/development/_meta.tsx | 104 ++--- .../integrations/entertainment/_meta.tsx | 16 +- .../resources/integrations/payments/_meta.tsx | 28 +- .../integrations/productivity/_meta.tsx | 152 ++++--- app/en/resources/integrations/sales/_meta.tsx | 56 +-- .../resources/integrations/search/_meta.tsx | 60 +-- .../resources/integrations/social/_meta.tsx | 44 ++- package.json | 3 +- .../data/toolkits/complextools.json | 275 ------------- .../data/toolkits/deepwiki.json | 145 ------- .../data/toolkits/index.json | 31 +- .../data/toolkits/test2.json | 357 ----------------- .../scripts/check-toolkit-coverage.ts | 371 ++++++++++++++++++ .../scripts/check-toolkit-coverage.test.ts | 320 +++++++++++++++ 18 files changed, 1221 insertions(+), 1026 deletions(-) create mode 100644 .github/workflows/check-toolkit-coverage.yml create mode 100644 app/en/resources/integrations/create-category-meta.ts delete mode 100644 toolkit-docs-generator/data/toolkits/complextools.json delete mode 100644 toolkit-docs-generator/data/toolkits/deepwiki.json delete mode 100644 toolkit-docs-generator/data/toolkits/test2.json create mode 100644 toolkit-docs-generator/scripts/check-toolkit-coverage.ts create mode 100644 toolkit-docs-generator/tests/scripts/check-toolkit-coverage.test.ts diff --git a/.github/workflows/check-toolkit-coverage.yml b/.github/workflows/check-toolkit-coverage.yml new file mode 100644 index 000000000..d2d8a37d5 --- /dev/null +++ b/.github/workflows/check-toolkit-coverage.yml @@ -0,0 +1,161 @@ +name: Check toolkit coverage +# Runs on every PR to main. Never fails ā€” surfaces gaps as GitHub warnings +# and a PR comment so authors are informed without being blocked. +# +# Checks: +# 1. API toolkits missing a JSON data file in toolkit-docs-generator/data/toolkits/ +# 2. JSON data files with no matching API toolkit (stale orphans) +# 3. API toolkits present in JSON but absent from any sidebar _meta.tsx + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + +permissions: + contents: read + pull-requests: write + +jobs: + coverage: + name: Toolkit coverage check + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: pnpm + + - name: Install dependencies + run: pnpm install + + - name: Run toolkit coverage check + id: coverage + run: | + pnpm run check:toolkit-coverage 2>&1 | tee coverage-output.txt + if grep -q "āš ļø Toolkit coverage warnings" coverage-output.txt; then + echo "has_warnings=true" >> "$GITHUB_OUTPUT" + else + echo "has_warnings=false" >> "$GITHUB_OUTPUT" + fi + env: + ENGINE_API_URL: ${{ secrets.ENGINE_API_URL }} + ENGINE_API_KEY: ${{ secrets.ENGINE_API_KEY }} + + - name: Emit GitHub warning annotations + if: steps.coverage.outputs.has_warnings == 'true' + run: | + # Emit one ::warning:: annotation per gap category so GitHub surfaces + # a yellow warning badge on the check without marking it as failed. + if grep -q "šŸ“ In API but missing JSON" coverage-output.txt; then + missing=$(grep -A 100 "šŸ“ In API but missing JSON" coverage-output.txt \ + | grep "^ [A-Z]" | sed 's/^ //' | paste -sd ', ') + echo "::warning::Toolkits in API with no JSON data file: ${missing}" + fi + + if grep -q "šŸ—‘ļø JSON files with no matching" coverage-output.txt; then + stale=$(grep -A 100 "šŸ—‘ļø JSON files with no matching" coverage-output.txt \ + | grep "^ [a-z]" | sed 's/^ //' | paste -sd ', ') + echo "::warning::Stale JSON files (no matching API toolkit): ${stale}" + fi + + if grep -q "šŸ—‚ļø In API + JSON but missing from sidebar" coverage-output.txt; then + sidebar=$(grep -A 100 "šŸ—‚ļø In API + JSON but missing from sidebar" coverage-output.txt \ + | grep "^ [A-Z]" | sed 's/^ //' | paste -sd ', ') + echo "::warning::Toolkits missing from sidebar _meta.tsx: ${sidebar}" + fi + + - name: Write job summary + if: always() + run: | + if [ "$(cat coverage-output.txt | grep -c 'āš ļø Toolkit coverage warnings')" -gt 0 ]; then + echo "## āš ļø Toolkit coverage ā€” gaps found" >> "$GITHUB_STEP_SUMMARY" + else + echo "## āœ… Toolkit coverage ā€” all clear" >> "$GITHUB_STEP_SUMMARY" + fi + echo "" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + cat coverage-output.txt >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + + - name: Post or update PR comment + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const output = fs.readFileSync('coverage-output.txt', 'utf8'); + const hasWarnings = output.includes('āš ļø Toolkit coverage warnings'); + const marker = ''; + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const existing = comments.find(c => + c.user.type === 'Bot' && c.body.includes(marker) + ); + + if (!hasWarnings) { + if (existing) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + }); + } + return; + } + + const body = `${marker} +## āš ļø Toolkit coverage gaps detected + +The coverage check found gaps between the Arcade API and the local toolkit data files. These are **warnings only** ā€” this PR is not blocked from merging. + +
+šŸ“‹ Full report + +\`\`\` +${output} +\`\`\` + +
+ +### What each warning means + +| Warning | Cause | Fix | +|---|---|---| +| **In API but missing JSON** | A toolkit is published in the API but has no docs data file | Run the toolkit docs generator for those toolkits | +| **Stale JSON files** | A JSON data file exists locally but has no matching toolkit in the API | Delete the file or verify the toolkit is still published | +| **Missing from sidebar** | A toolkit has JSON but isn't listed in any \`_meta.tsx\` | Run \`npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts\` | + +--- +*This comment updates automatically when the PR is pushed. It is removed when all gaps are resolved.*`; + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } diff --git a/app/en/resources/integrations/create-category-meta.ts b/app/en/resources/integrations/create-category-meta.ts new file mode 100644 index 000000000..0ee8d4720 --- /dev/null +++ b/app/en/resources/integrations/create-category-meta.ts @@ -0,0 +1,48 @@ +import type { MetaRecord } from "nextra"; + +type ToolkitType = + | "arcade" + | "arcade_starter" + | "community" + | "verified" + | "auth"; + +export type CategoryEntry = { + slug: string; + title: string; + href: string; + type: ToolkitType; +}; + +/** + * Builds a Nextra MetaRecord for an integration category, automatically + * inserting "Optimized" and "Starter" separator headings based on toolkit type. + * + * - "Optimized" group: any entry whose type is not "arcade_starter" + * - "Starter" group: entries with type "arcade_starter" + * + * A separator is only emitted when its group is non-empty, so categories + * with a single type do not show an unnecessary heading. + */ +export function createCategoryMeta(entries: CategoryEntry[]): MetaRecord { + const optimized = entries.filter((e) => e.type !== "arcade_starter"); + const starter = entries.filter((e) => e.type === "arcade_starter"); + + const result: MetaRecord = {}; + + if (optimized.length > 0) { + result["-- Optimized"] = { type: "separator", title: "Optimized" }; + for (const entry of optimized) { + result[entry.slug] = { title: entry.title, href: entry.href }; + } + } + + if (starter.length > 0) { + result["-- Starter"] = { type: "separator", title: "Starter" }; + for (const entry of starter) { + result[entry.slug] = { title: entry.title, href: entry.href }; + } + } + + return result; +} diff --git a/app/en/resources/integrations/customer-support/_meta.tsx b/app/en/resources/integrations/customer-support/_meta.tsx index b713002bb..cb46cb15b 100644 --- a/app/en/resources/integrations/customer-support/_meta.tsx +++ b/app/en/resources/integrations/customer-support/_meta.tsx @@ -1,38 +1,40 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - "-- Optimized": { - type: "separator", - title: "Optimized", - }, - zendesk: { +export default createCategoryMeta([ + { + slug: "zendesk", title: "Zendesk", href: "/en/resources/integrations/customer-support/zendesk", + type: "arcade", }, - "-- Starter": { - type: "separator", - title: "Starter", - }, - "customerio-api": { + { + slug: "customerio-api", title: "Customer.io API", href: "/en/resources/integrations/customer-support/customerio-api", + type: "arcade_starter", }, - "customerio-pipelines-api": { + { + slug: "customerio-pipelines-api", title: "Customer.io Pipelines API", href: "/en/resources/integrations/customer-support/customerio-pipelines-api", + type: "arcade_starter", }, - "customerio-track-api": { + { + slug: "customerio-track-api", title: "Customer.io Track API", href: "/en/resources/integrations/customer-support/customerio-track-api", + type: "arcade_starter", }, - "freshservice-api": { + { + slug: "freshservice-api", title: "Freshservice API", href: "/en/resources/integrations/customer-support/freshservice-api", + type: "arcade_starter", }, - "intercom-api": { + { + slug: "intercom-api", title: "Intercom API", href: "/en/resources/integrations/customer-support/intercom-api", + type: "arcade_starter", }, -}; - -export default meta; +]); diff --git a/app/en/resources/integrations/databases/_meta.tsx b/app/en/resources/integrations/databases/_meta.tsx index d397cbee0..07ab0020c 100644 --- a/app/en/resources/integrations/databases/_meta.tsx +++ b/app/en/resources/integrations/databases/_meta.tsx @@ -1,34 +1,34 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - "-- Optimized": { - type: "separator", - title: "Optimized", - }, - clickhouse: { +export default createCategoryMeta([ + { + slug: "clickhouse", title: "Clickhouse", href: "/en/resources/integrations/databases/clickhouse", + type: "community", }, - mongodb: { + { + slug: "mongodb", title: "MongoDB", href: "/en/resources/integrations/databases/mongodb", + type: "community", }, - postgres: { + { + slug: "postgres", title: "Postgres", href: "/en/resources/integrations/databases/postgres", + type: "community", }, - "-- Starter": { - type: "separator", - title: "Starter", - }, - "weaviate-api": { + { + slug: "weaviate-api", title: "Weaviate API", href: "/en/resources/integrations/databases/weaviate-api", + type: "arcade_starter", }, - yugabytedb: { + { + slug: "yugabytedb", title: "YugabyteDB", href: "/en/resources/integrations/databases/yugabytedb", + type: "arcade_starter", }, -}; - -export default meta; +]); diff --git a/app/en/resources/integrations/development/_meta.tsx b/app/en/resources/integrations/development/_meta.tsx index 310c71a19..692b2cd9b 100644 --- a/app/en/resources/integrations/development/_meta.tsx +++ b/app/en/resources/integrations/development/_meta.tsx @@ -1,102 +1,118 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - "-- Optimized": { - type: "separator", - title: "Optimized", - }, - brightdata: { +export default createCategoryMeta([ + { + slug: "brightdata", title: "Bright Data", href: "/en/resources/integrations/development/brightdata", + type: "community", }, - complextools: { - title: "ComplexTools", - href: "/en/resources/integrations/development/complextools", - }, - daytona: { + { + slug: "daytona", title: "Daytona", href: "/en/resources/integrations/development/daytona", + type: "arcade", }, - deepwiki: { - title: "Deepwiki", - href: "/en/resources/integrations/development/deepwiki", - }, - e2b: { + { + slug: "e2b", title: "E2B", href: "/en/resources/integrations/development/e2b", + type: "arcade", }, - figma: { + { + slug: "figma", title: "Figma", href: "/en/resources/integrations/development/figma", + type: "arcade", }, - firecrawl: { + { + slug: "firecrawl", title: "Firecrawl", href: "/en/resources/integrations/development/firecrawl", + type: "arcade", }, - github: { + { + slug: "github", title: "GitHub", href: "/en/resources/integrations/development/github", + type: "arcade", }, - math: { + { + slug: "math", title: "Math", href: "/en/resources/integrations/development/math", + type: "arcade", }, - pagerduty: { + { + slug: "pagerduty", title: "Pagerduty", href: "/en/resources/integrations/development/pagerduty", + type: "arcade", }, - pylon: { + { + slug: "pylon", title: "Pylon", href: "/en/resources/integrations/development/pylon", + type: "arcade", }, - search: { + { + slug: "search", title: "Search", href: "/en/resources/integrations/development/search", + type: "arcade", }, - test2: { - title: "Test2", - href: "/en/resources/integrations/development/test2", - }, - web: { + { + slug: "web", title: "Web", href: "/en/resources/integrations/development/web", + type: "arcade", }, - "-- Starter": { - type: "separator", - title: "Starter", - }, - "arcade-engine-api": { + { + slug: "arcade-engine-api", title: "Arcade Engine API", href: "/en/resources/integrations/development/arcade-engine-api", + type: "arcade_starter", }, - "cursor-agents-api": { + { + slug: "cursor-agents-api", title: "Cursor Agents API", href: "/en/resources/integrations/development/cursor-agents-api", + type: "arcade_starter", }, - "datadog-api": { + { + slug: "datadog-api", title: "Datadog API", href: "/en/resources/integrations/development/datadog-api", + type: "arcade_starter", }, - "github-api": { + { + slug: "github-api", title: "GitHub API", href: "/en/resources/integrations/development/github-api", + type: "arcade_starter", }, - "pagerduty-api": { + { + slug: "pagerduty-api", title: "PagerDuty API", href: "/en/resources/integrations/development/pagerduty-api", + type: "arcade_starter", }, - "posthog-api": { + { + slug: "posthog-api", title: "PostHog API", href: "/en/resources/integrations/development/posthog-api", + type: "arcade_starter", }, - pylonapi: { + { + slug: "pylonapi", title: "PylonApi", href: "/en/resources/integrations/development/pylonapi", + type: "arcade_starter", }, - "vercel-api": { + { + slug: "vercel-api", title: "Vercel API", href: "/en/resources/integrations/development/vercel-api", + type: "arcade_starter", }, -}; - -export default meta; +]); diff --git a/app/en/resources/integrations/entertainment/_meta.tsx b/app/en/resources/integrations/entertainment/_meta.tsx index 2b7eebcd9..fe7743af2 100644 --- a/app/en/resources/integrations/entertainment/_meta.tsx +++ b/app/en/resources/integrations/entertainment/_meta.tsx @@ -1,14 +1,16 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - imgflip: { +export default createCategoryMeta([ + { + slug: "imgflip", title: "Imgflip", href: "/en/resources/integrations/entertainment/imgflip", + type: "arcade", }, - spotify: { + { + slug: "spotify", title: "Spotify", href: "/en/resources/integrations/entertainment/spotify", + type: "arcade", }, -}; - -export default meta; +]); diff --git a/app/en/resources/integrations/payments/_meta.tsx b/app/en/resources/integrations/payments/_meta.tsx index 64de9299e..d05281818 100644 --- a/app/en/resources/integrations/payments/_meta.tsx +++ b/app/en/resources/integrations/payments/_meta.tsx @@ -1,26 +1,22 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - "-- Optimized": { - type: "separator", - title: "Optimized", - }, - stripe: { +export default createCategoryMeta([ + { + slug: "stripe", title: "Stripe", href: "/en/resources/integrations/payments/stripe", + type: "arcade", }, - "-- Starter": { - type: "separator", - title: "Starter", - }, - stripe_api: { + { + slug: "stripe_api", title: "Stripe API", href: "/en/resources/integrations/payments/stripe_api", + type: "arcade_starter", }, - "zoho-books-api": { + { + slug: "zoho-books-api", title: "Zoho Books API", href: "/en/resources/integrations/payments/zoho-books-api", + type: "arcade_starter", }, -}; - -export default meta; +]); diff --git a/app/en/resources/integrations/productivity/_meta.tsx b/app/en/resources/integrations/productivity/_meta.tsx index 27b3a0d2e..34e371387 100644 --- a/app/en/resources/integrations/productivity/_meta.tsx +++ b/app/en/resources/integrations/productivity/_meta.tsx @@ -1,150 +1,208 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - "-- Optimized": { - type: "separator", - title: "Optimized", - }, - asana: { +export default createCategoryMeta([ + { + slug: "asana", title: "Asana", href: "/en/resources/integrations/productivity/asana", + type: "arcade", }, - clickup: { + { + slug: "clickup", title: "ClickUp", href: "/en/resources/integrations/productivity/clickup", + type: "arcade", }, - confluence: { + { + slug: "confluence", title: "Confluence", href: "/en/resources/integrations/productivity/confluence", + type: "arcade", }, - dropbox: { + { + slug: "dropbox", title: "Dropbox", href: "/en/resources/integrations/productivity/dropbox", + type: "arcade", }, - gmail: { + { + slug: "gmail", title: "Gmail", href: "/en/resources/integrations/productivity/gmail", + type: "arcade", }, - "google-calendar": { + { + slug: "google-calendar", title: "Google Calendar", href: "/en/resources/integrations/productivity/google-calendar", + type: "arcade", }, - "google-contacts": { + { + slug: "google-contacts", title: "Google Contacts", href: "/en/resources/integrations/productivity/google-contacts", + type: "arcade", }, - "google-docs": { + { + slug: "google-docs", title: "Google Docs", href: "/en/resources/integrations/productivity/google-docs", + type: "arcade", }, - "google-drive": { + { + slug: "google-drive", title: "Google Drive", href: "/en/resources/integrations/productivity/google-drive", + type: "arcade", }, - "google-sheets": { + { + slug: "google-sheets", title: "Google Sheets", href: "/en/resources/integrations/productivity/google-sheets", + type: "arcade", }, - "google-slides": { + { + slug: "google-slides", title: "Google Slides", href: "/en/resources/integrations/productivity/google-slides", + type: "arcade", }, - jira: { + { + slug: "jira", title: "Jira", href: "/en/resources/integrations/productivity/jira", + type: "auth", }, - linear: { + { + slug: "linear", title: "Linear", href: "/en/resources/integrations/productivity/linear", + type: "arcade", }, - "microsoft-excel": { + { + slug: "microsoft-excel", title: "Microsoft Excel", href: "/en/resources/integrations/productivity/microsoft-excel", + type: "arcade", }, - "microsoft-onedrive": { + { + slug: "microsoft-onedrive", title: "Microsoft OneDrive", href: "/en/resources/integrations/productivity/microsoft-onedrive", + type: "arcade", }, - "microsoft-powerpoint": { + { + slug: "microsoft-powerpoint", title: "Microsoft PowerPoint", href: "/en/resources/integrations/productivity/microsoft-powerpoint", + type: "arcade", }, - sharepoint: { + { + slug: "sharepoint", title: "Microsoft SharePoint", href: "/en/resources/integrations/productivity/sharepoint", + type: "arcade", }, - "microsoft-word": { + { + slug: "microsoft-word", title: "Microsoft Word", href: "/en/resources/integrations/productivity/microsoft-word", + type: "arcade", }, - "outlook-calendar": { + { + slug: "outlook-calendar", title: "Outlook Calendar", href: "/en/resources/integrations/productivity/outlook-calendar", + type: "arcade", }, - "outlook-mail": { + { + slug: "outlook-mail", title: "Outlook Mail", href: "/en/resources/integrations/productivity/outlook-mail", + type: "arcade", }, - "-- Starter": { - type: "separator", - title: "Starter", - }, - "airtable-api": { + { + slug: "airtable-api", title: "Airtable API", href: "/en/resources/integrations/productivity/airtable-api", + type: "arcade_starter", }, - "asana-api": { + { + slug: "asana-api", title: "Asana API", href: "/en/resources/integrations/productivity/asana-api", + type: "arcade_starter", }, - "ashby-api": { + { + slug: "ashby-api", title: "Ashby API", href: "/en/resources/integrations/productivity/ashby-api", + type: "arcade_starter", }, - "box-api": { + { + slug: "box-api", title: "Box API", href: "/en/resources/integrations/productivity/box-api", + type: "arcade_starter", }, - "calendly-api": { + { + slug: "calendly-api", title: "Calendly API", href: "/en/resources/integrations/productivity/calendly-api", + type: "arcade_starter", }, - "clickup-api": { + { + slug: "clickup-api", title: "ClickUp API", href: "/en/resources/integrations/productivity/clickup-api", + type: "arcade_starter", }, - "figma-api": { + { + slug: "figma-api", title: "Figma API", href: "/en/resources/integrations/productivity/figma-api", + type: "arcade_starter", }, - "luma-api": { + { + slug: "luma-api", title: "Luma API", href: "/en/resources/integrations/productivity/luma-api", + type: "arcade_starter", }, - "mailchimp-api": { + { + slug: "mailchimp-api", title: "Mailchimp API", href: "/en/resources/integrations/productivity/mailchimp-api", + type: "arcade_starter", }, - "miro-api": { + { + slug: "miro-api", title: "Miro API", href: "/en/resources/integrations/productivity/miro-api", + type: "arcade_starter", }, - "squareup-api": { + { + slug: "squareup-api", title: "SquareUp API", href: "/en/resources/integrations/productivity/squareup-api", + type: "arcade_starter", }, - "ticktick-api": { + { + slug: "ticktick-api", title: "TickTick API", href: "/en/resources/integrations/productivity/ticktick-api", + type: "arcade_starter", }, - "trello-api": { + { + slug: "trello-api", title: "Trello API", href: "/en/resources/integrations/productivity/trello-api", + type: "arcade_starter", }, - "xero-api": { + { + slug: "xero-api", title: "Xero API", href: "/en/resources/integrations/productivity/xero-api", + type: "arcade_starter", }, -}; - -export default meta; +]); diff --git a/app/en/resources/integrations/sales/_meta.tsx b/app/en/resources/integrations/sales/_meta.tsx index 07f1f47fb..a55a7359d 100644 --- a/app/en/resources/integrations/sales/_meta.tsx +++ b/app/en/resources/integrations/sales/_meta.tsx @@ -1,54 +1,64 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - "-- Optimized": { - type: "separator", - title: "Optimized", - }, - hubspot: { +export default createCategoryMeta([ + { + slug: "hubspot", title: "HubSpot", href: "/en/resources/integrations/sales/hubspot", + type: "arcade", }, - salesforce: { + { + slug: "salesforce", title: "Salesforce", href: "/en/resources/integrations/sales/salesforce", + type: "arcade", }, - "-- Starter": { - type: "separator", - title: "Starter", - }, - "hubspot-automation-api": { + { + slug: "hubspot-automation-api", title: "HubSpot Automation API", href: "/en/resources/integrations/sales/hubspot-automation-api", + type: "arcade_starter", }, - "hubspot-cms-api": { + { + slug: "hubspot-cms-api", title: "HubSpot CMS API", href: "/en/resources/integrations/sales/hubspot-cms-api", + type: "arcade_starter", }, - "hubspot-conversations-api": { + { + slug: "hubspot-conversations-api", title: "HubSpot Conversations API", href: "/en/resources/integrations/sales/hubspot-conversations-api", + type: "arcade_starter", }, - "hubspot-crm-api": { + { + slug: "hubspot-crm-api", title: "HubSpot CRM API", href: "/en/resources/integrations/sales/hubspot-crm-api", + type: "arcade_starter", }, - "hubspot-events-api": { + { + slug: "hubspot-events-api", title: "HubSpot Events API", href: "/en/resources/integrations/sales/hubspot-events-api", + type: "arcade_starter", }, - "hubspot-marketing-api": { + { + slug: "hubspot-marketing-api", title: "HubSpot Marketing API", href: "/en/resources/integrations/sales/hubspot-marketing-api", + type: "arcade_starter", }, - "hubspot-meetings-api": { + { + slug: "hubspot-meetings-api", title: "HubSpot Meetings API", href: "/en/resources/integrations/sales/hubspot-meetings-api", + type: "arcade_starter", }, - "hubspot-users-api": { + { + slug: "hubspot-users-api", title: "HubSpot Users API", href: "/en/resources/integrations/sales/hubspot-users-api", + type: "arcade_starter", }, -}; - -export default meta; +]); diff --git a/app/en/resources/integrations/search/_meta.tsx b/app/en/resources/integrations/search/_meta.tsx index 1936cef66..f27d773ab 100644 --- a/app/en/resources/integrations/search/_meta.tsx +++ b/app/en/resources/integrations/search/_meta.tsx @@ -1,58 +1,70 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - "-- Optimized": { - type: "separator", - title: "Optimized", - }, - google_finance: { +export default createCategoryMeta([ + { + slug: "google_finance", title: "Google Finance", href: "/en/resources/integrations/search/google_finance", + type: "arcade", }, - google_flights: { + { + slug: "google_flights", title: "Google Flights", href: "/en/resources/integrations/search/google_flights", + type: "arcade", }, - google_hotels: { + { + slug: "google_hotels", title: "Google Hotels", href: "/en/resources/integrations/search/google_hotels", + type: "arcade", }, - google_jobs: { + { + slug: "google_jobs", title: "Google Jobs", href: "/en/resources/integrations/search/google_jobs", + type: "arcade", }, - google_maps: { + { + slug: "google_maps", title: "Google Maps", href: "/en/resources/integrations/search/google_maps", + type: "arcade", }, - google_news: { + { + slug: "google_news", title: "Google News", href: "/en/resources/integrations/search/google_news", + type: "arcade", }, - google_search: { + { + slug: "google_search", title: "Google Search", href: "/en/resources/integrations/search/google_search", + type: "arcade", }, - google_shopping: { + { + slug: "google_shopping", title: "Google Shopping", href: "/en/resources/integrations/search/google_shopping", + type: "arcade", }, - walmart: { + { + slug: "walmart", title: "Walmart", href: "/en/resources/integrations/search/walmart", + type: "arcade", }, - youtube: { + { + slug: "youtube", title: "Youtube", href: "/en/resources/integrations/search/youtube", + type: "arcade", }, - "-- Starter": { - type: "separator", - title: "Starter", - }, - "exa-api": { + { + slug: "exa-api", title: "Exa API", href: "/en/resources/integrations/search/exa-api", + type: "arcade_starter", }, -}; - -export default meta; +]); diff --git a/app/en/resources/integrations/social/_meta.tsx b/app/en/resources/integrations/social/_meta.tsx index c22d37207..e7b648dfc 100644 --- a/app/en/resources/integrations/social/_meta.tsx +++ b/app/en/resources/integrations/social/_meta.tsx @@ -1,42 +1,46 @@ -import type { MetaRecord } from "nextra"; +import { createCategoryMeta } from "../create-category-meta"; -const meta: MetaRecord = { - "-- Optimized": { - type: "separator", - title: "Optimized", - }, - linkedin: { +export default createCategoryMeta([ + { + slug: "linkedin", title: "LinkedIn", href: "/en/resources/integrations/social/linkedin", + type: "arcade", }, - "microsoft-teams": { + { + slug: "microsoft-teams", title: "Microsoft Teams", href: "/en/resources/integrations/social/microsoft-teams", + type: "arcade", }, - reddit: { + { + slug: "reddit", title: "Reddit", href: "/en/resources/integrations/social/reddit", + type: "arcade", }, - slack: { + { + slug: "slack", title: "Slack", href: "/en/resources/integrations/social/slack", + type: "arcade", }, - x: { + { + slug: "x", title: "X", href: "/en/resources/integrations/social/x", + type: "arcade", }, - zoom: { + { + slug: "zoom", title: "Zoom", href: "/en/resources/integrations/social/zoom", + type: "arcade", }, - "-- Starter": { - type: "separator", - title: "Starter", - }, - "slack-api": { + { + slug: "slack-api", title: "Slack API", href: "/en/resources/integrations/social/slack-api", + type: "arcade_starter", }, -}; - -export default meta; +]); diff --git a/package.json b/package.json index befbcb3ec..a445be223 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,8 @@ "check-redirects": "pnpm dlx tsx scripts/check-redirects.ts", "update-links": "pnpm dlx tsx scripts/update-internal-links.ts", "check-meta": "pnpm dlx tsx scripts/check-meta-keys.ts", - "metadata-report": "pnpm dlx tsx toolkit-docs-generator/scripts/report-tool-metadata.ts" + "metadata-report": "pnpm dlx tsx toolkit-docs-generator/scripts/report-tool-metadata.ts", + "check:toolkit-coverage": "pnpm dlx tsx toolkit-docs-generator/scripts/check-toolkit-coverage.ts" }, "repository": { "type": "git", diff --git a/toolkit-docs-generator/data/toolkits/complextools.json b/toolkit-docs-generator/data/toolkits/complextools.json deleted file mode 100644 index bba5a14c4..000000000 --- a/toolkit-docs-generator/data/toolkits/complextools.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "id": "ComplexTools", - "label": "ComplexTools", - "version": "0.1.0", - "description": "", - "metadata": { - "category": "development", - "iconUrl": "https://design-system.arcade.dev/icons/complextools.svg", - "isBYOC": false, - "isPro": false, - "type": "arcade", - "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/complextools", - "isComingSoon": false, - "isHidden": false - }, - "auth": null, - "tools": [ - { - "name": "CreateOrder", - "qualifiedName": "ComplexTools.CreateOrder", - "fullyQualifiedName": "ComplexTools.CreateOrder@0.1.0", - "description": "Create a new order in the e-commerce system.\n\nNote: Understanding the request schema is necessary to properly create the stringified\nJSON input object for execution.\n\nModes:\n- GET_REQUEST_SCHEMA: Returns the schema. Only call if you don't already have it.\n Do NOT call repeatedly if you already received the schema.\n- EXECUTE: Creates the order with request body JSON, priority, and notification_email.\n Priority must be: 'low', 'normal', 'high', or 'urgent'.\n\nIf you need the schema, call with mode='get_request_schema' ONCE, then execute.", - "parameters": [ - { - "name": "mode", - "type": "string", - "required": true, - "description": "Operation mode: 'get_request_schema' returns the OpenAPI spec for the request body, 'execute' performs the actual order creation", - "enum": ["get_request_schema", "execute"], - "inferrable": true - }, - { - "name": "priority", - "type": "string", - "required": true, - "description": "Order priority level. Required when mode is 'execute', ignored when mode is 'get_request_schema'. Valid values: 'low', 'normal', 'high', 'urgent'", - "enum": null, - "inferrable": true - }, - { - "name": "notification_email", - "type": "string", - "required": true, - "description": "Email address for order notifications. Required when mode is 'execute', ignored when mode is 'get_request_schema'", - "enum": null, - "inferrable": true - }, - { - "name": "request_body", - "type": "string", - "required": false, - "description": "Stringified JSON representing the order creation request body. Required when mode is 'execute', ignored when mode is 'get_request_schema'", - "enum": null, - "inferrable": true - }, - { - "name": "gift_message", - "type": "string", - "required": false, - "description": "Optional gift message to include with the order. Only used when mode is 'execute'", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "ComplexTools.CreateOrder", - "parameters": { - "mode": { - "value": "execute", - "type": "string", - "required": true - }, - "priority": { - "value": "normal", - "type": "string", - "required": true - }, - "notification_email": { - "value": "customer@example.com", - "type": "string", - "required": true - }, - "request_body": { - "value": "{\"item_id\": \"12345\", \"quantity\": 2, \"shipping_address\": \"123 Main St, Anytown, USA\"}", - "type": "string", - "required": false - }, - "gift_message": { - "value": "Happy Birthday!", - "type": "string", - "required": false - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "CreateUser", - "qualifiedName": "ComplexTools.CreateUser", - "fullyQualifiedName": "ComplexTools.CreateUser@0.1.0", - "description": "Create a new user in the system.\n\nNote: Understanding the request schema is necessary to properly create the stringified\nJSON input object for execution.\n\nModes:\n- GET_REQUEST_SCHEMA: Returns the schema. Only call if you don't already have it.\n Do NOT call repeatedly if you already received the schema.\n- EXECUTE: Creates the user with the provided request body JSON.\n\nIf you need the schema, call with mode='get_request_schema' ONCE, then execute.", - "parameters": [ - { - "name": "mode", - "type": "string", - "required": true, - "description": "Operation mode: 'get_request_schema' returns the OpenAPI spec for the request body, 'execute' performs the actual user creation", - "enum": ["get_request_schema", "execute"], - "inferrable": true - }, - { - "name": "request_body", - "type": "string", - "required": false, - "description": "Stringified JSON representing the user creation request body. Required when mode is 'execute', ignored when mode is 'get_request_schema'", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "ComplexTools.CreateUser", - "parameters": { - "mode": { - "value": "execute", - "type": "string", - "required": true - }, - "request_body": { - "value": "{\"username\":\"johndoe\",\"email\":\"johndoe@example.com\",\"password\":\"P@ssw0rd123\",\"first_name\":\"John\",\"last_name\":\"Doe\"}", - "type": "string", - "required": false - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "GetAvailableProducts", - "qualifiedName": "ComplexTools.GetAvailableProducts", - "fullyQualifiedName": "ComplexTools.GetAvailableProducts@0.1.0", - "description": "Retrieve a list of available products with their IDs and details.\n\nThis tool returns static product data that can be used when creating orders.\nEach product includes its UUID, name, category, and available customizations.\nUse the product IDs from this response when building order request bodies.", - "parameters": [], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "string", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "ComplexTools.GetAvailableProducts", - "parameters": {}, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "GetCustomerInfo", - "qualifiedName": "ComplexTools.GetCustomerInfo", - "fullyQualifiedName": "ComplexTools.GetCustomerInfo@0.1.0", - "description": "Retrieve customer information by email address.\n\nThis tool returns customer details including their UUID, profile information,\nand preferences. The customer ID can be used when creating orders or users.\nThis is a simplified tool that returns static data for demonstration purposes.", - "parameters": [ - { - "name": "customer_email", - "type": "string", - "required": true, - "description": "Email address of the customer to look up", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "string", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "ComplexTools.GetCustomerInfo", - "parameters": { - "customer_email": { - "value": "example.customer@mail.com", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "GetDiscountCodes", - "qualifiedName": "ComplexTools.GetDiscountCodes", - "fullyQualifiedName": "ComplexTools.GetDiscountCodes@0.1.0", - "description": "Retrieve a list of currently active discount codes.\n\nThis tool returns available discount codes that can be applied to orders,\nincluding their codes, descriptions, and discount percentages.\nUse these codes in the order creation request body.", - "parameters": [], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "string", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "ComplexTools.GetDiscountCodes", - "parameters": {}, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "GetShippingRates", - "qualifiedName": "ComplexTools.GetShippingRates", - "fullyQualifiedName": "ComplexTools.GetShippingRates@0.1.0", - "description": "Get available shipping rates and estimated delivery times for a specific country.\n\nThis tool returns shipping options (standard, express, overnight, international)\nwith their costs and estimated delivery times based on the destination country.\nUse this information to choose an appropriate shipping method when creating orders.", - "parameters": [ - { - "name": "country_code", - "type": "string", - "required": true, - "description": "ISO 3166-1 alpha-2 country code (e.g., 'US', 'GB', 'CA')", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "string", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "ComplexTools.GetShippingRates", - "parameters": { - "country_code": { - "value": "CA", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - } - ], - "documentationChunks": [], - "customImports": [], - "subPages": [], - "generatedAt": "2026-01-26T17:27:46.761Z", - "summary": "ComplexTools provides an extensive toolkit for managing e-commerce operations, enabling seamless order creation, user management, and access to vital product information.\n\n### Capabilities\n- Create orders and manage user accounts efficiently.\n- Retrieve product data, customer information, active discount codes, and shipping rates.\n- Simple integration with static data for demonstration and development purposes.\n- Understand request schemas for effective API usage without redundancy.\n\n### Secrets\n- No secrets or sensitive information are utilized in this toolkit." -} diff --git a/toolkit-docs-generator/data/toolkits/deepwiki.json b/toolkit-docs-generator/data/toolkits/deepwiki.json deleted file mode 100644 index 2bac06bc1..000000000 --- a/toolkit-docs-generator/data/toolkits/deepwiki.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "id": "Deepwiki", - "label": "Deepwiki", - "version": "0.0.1", - "description": "", - "metadata": { - "category": "development", - "iconUrl": "https://design-system.arcade.dev/icons/deepwiki.svg", - "isBYOC": false, - "isPro": false, - "type": "arcade", - "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/deepwiki", - "isComingSoon": false, - "isHidden": false - }, - "auth": null, - "tools": [ - { - "name": "ask_question", - "qualifiedName": "Deepwiki.ask_question", - "fullyQualifiedName": "Deepwiki.ask_question@0.0.1", - "description": "Ask any question about a GitHub repository", - "parameters": [ - { - "name": "question", - "type": "string", - "required": true, - "description": "The question to ask about the repository", - "enum": null, - "inferrable": true - }, - { - "name": "repoName", - "type": "string", - "required": true, - "description": "GitHub repository: owner/repo (e.g. \"facebook/react\")", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "string", - "description": null - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Deepwiki.ask_question", - "parameters": { - "question": { - "value": "What is the main purpose of this repository?", - "type": "string", - "required": true - }, - "repoName": { - "value": "facebook/react", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "read_wiki_contents", - "qualifiedName": "Deepwiki.read_wiki_contents", - "fullyQualifiedName": "Deepwiki.read_wiki_contents@0.0.1", - "description": "View documentation about a GitHub repository", - "parameters": [ - { - "name": "repoName", - "type": "string", - "required": true, - "description": "GitHub repository: owner/repo (e.g. \"facebook/react\")", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "string", - "description": null - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Deepwiki.read_wiki_contents", - "parameters": { - "repoName": { - "value": "facebook/react", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "read_wiki_structure", - "qualifiedName": "Deepwiki.read_wiki_structure", - "fullyQualifiedName": "Deepwiki.read_wiki_structure@0.0.1", - "description": "Get a list of documentation topics for a GitHub repository", - "parameters": [ - { - "name": "repoName", - "type": "string", - "required": true, - "description": "GitHub repository: owner/repo (e.g. \"facebook/react\")", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "string", - "description": null - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Deepwiki.read_wiki_structure", - "parameters": { - "repoName": { - "value": "facebook/react", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - } - ], - "documentationChunks": [], - "customImports": [], - "subPages": [], - "generatedAt": "2026-01-26T17:28:39.914Z", - "summary": "Deepwiki offers a toolkit designed for seamless interaction with GitHub repositories, enabling developers to efficiently access and query documentation as well as repository details. \\n\\n**Capabilities**:\\n- Perform effective queries about GitHub repositories using natural language.\\n- Retrieve comprehensive documentation and wiki content.\\n- Explore the structure of documentation topics within a repository for streamlined navigation.\\n\\n**Secrets**:\\n- No secret types are utilized in this toolkit, enhancing accessibility without requiring sensitive information." -} diff --git a/toolkit-docs-generator/data/toolkits/index.json b/toolkit-docs-generator/data/toolkits/index.json index 1afae6be8..7301ff945 100644 --- a/toolkit-docs-generator/data/toolkits/index.json +++ b/toolkit-docs-generator/data/toolkits/index.json @@ -1,6 +1,4 @@ { - "generatedAt": "2026-03-01T11:12:36.819Z", - "version": "1.0.0", "toolkits": [ { "id": "AirtableApi", @@ -110,15 +108,6 @@ "toolCount": 2, "authType": "none" }, - { - "id": "ComplexTools", - "label": "ComplexTools", - "version": "0.1.0", - "category": "development", - "type": "arcade", - "toolCount": 6, - "authType": "none" - }, { "id": "Confluence", "label": "Confluence", @@ -182,15 +171,6 @@ "toolCount": 46, "authType": "oauth2" }, - { - "id": "Deepwiki", - "label": "Deepwiki", - "version": "0.0.1", - "category": "development", - "type": "arcade", - "toolCount": 3, - "authType": "none" - }, { "id": "Dropbox", "label": "Dropbox", @@ -794,15 +774,6 @@ "toolCount": 220, "authType": "none" }, - { - "id": "Test2", - "label": "Test2", - "version": "0.1.0", - "category": "development", - "type": "arcade", - "toolCount": 6, - "authType": "none" - }, { "id": "TicktickApi", "label": "TickTick API", @@ -930,4 +901,4 @@ "authType": "oauth2" } ] -} \ No newline at end of file +} diff --git a/toolkit-docs-generator/data/toolkits/test2.json b/toolkit-docs-generator/data/toolkits/test2.json deleted file mode 100644 index 818791d54..000000000 --- a/toolkit-docs-generator/data/toolkits/test2.json +++ /dev/null @@ -1,357 +0,0 @@ -{ - "id": "Test2", - "label": "Test2", - "version": "0.1.0", - "description": "asdf", - "metadata": { - "category": "development", - "iconUrl": "https://design-system.arcade.dev/icons/test2.svg", - "isBYOC": false, - "isPro": false, - "type": "arcade", - "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/test2", - "isComingSoon": false, - "isHidden": false - }, - "auth": null, - "tools": [ - { - "name": "CreateEvent", - "qualifiedName": "Test2.CreateEvent", - "fullyQualifiedName": "Test2.CreateEvent@0.1.0", - "description": "Create a calendar event with attendees and location.\n\nNOTE: If you have not already called get_body_scheme('create_event') or are unsure\nabout the body structure, call it first to see the exact schema required.\n\nThe body should include title, startTime, attendees, and location\n(oneOf: physical/virtual/hybrid). Optionally include recurrence and reminders.", - "parameters": [ - { - "name": "body", - "type": "string", - "required": true, - "description": "JSON string containing event data. If unsure about structure, call get_body_scheme('create_event') first.", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Test2.CreateEvent", - "parameters": { - "body": { - "value": "{\"title\": \"Team Meeting\", \"startTime\": \"2023-10-12T10:00:00Z\", \"attendees\": [\"alice@example.com\", \"bob@example.com\"], \"location\": \"virtual\", \"recurrence\": \"weekly\", \"reminders\": [{\"method\": \"email\", \"minutes\": 10}]}", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "CreateOrder", - "qualifiedName": "Test2.CreateOrder", - "fullyQualifiedName": "Test2.CreateOrder@0.1.0", - "description": "Create a new order with items and payment information.\n\nNOTE: If you have not already called get_body_scheme('create_order') or are unsure\nabout the body structure, call it first to see the exact schema required.\n\nThe body should include customerId, items with complex pricing (anyOf: fixed/discounted/tiered),\npayment method (oneOf: credit_card/paypal/bank_transfer), and shipping information.", - "parameters": [ - { - "name": "body", - "type": "string", - "required": true, - "description": "JSON string containing order data. If unsure about structure, call get_body_scheme('create_order') first.", - "enum": null, - "inferrable": true - }, - { - "name": "api_version", - "type": "string", - "required": true, - "description": "API version to use", - "enum": null, - "inferrable": true - }, - { - "name": "idempotency_key", - "type": "string", - "required": true, - "description": "Unique key to prevent duplicate orders", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Test2.CreateOrder", - "parameters": { - "body": { - "value": "{\"customerId\":\"12345\",\"items\":[{\"productId\":\"56789\",\"quantity\":2,\"pricing\":{\"type\":\"fixed\",\"amount\":19.99}},{\"productId\":\"67890\",\"quantity\":1,\"pricing\":{\"type\":\"discounted\",\"amount\":29.99,\"discount\":5}}],\"paymentMethod\":\"credit_card\",\"shipping\":{\"address\":\"123 Main St\",\"city\":\"Anytown\",\"state\":\"CA\",\"zip\":\"90210\"}}", - "type": "string", - "required": true - }, - "api_version": { - "value": "1.0", - "type": "string", - "required": true - }, - "idempotency_key": { - "value": "order-unique-abc123", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "CreateUser", - "qualifiedName": "Test2.CreateUser", - "fullyQualifiedName": "Test2.CreateUser@0.1.0", - "description": "Create a new user with complex profile information.\n\nNOTE: If you have not already called get_body_scheme('create_user') or are unsure\nabout the body structure, call it first to see the exact schema required.\n\nThe body should include email, profile with contact method (oneOf: phone/email/social),\noptional address, and preferences.", - "parameters": [ - { - "name": "body", - "type": "string", - "required": true, - "description": "JSON string containing user data. If unsure about structure, call get_body_scheme('create_user') first.", - "enum": null, - "inferrable": true - }, - { - "name": "tenant_id", - "type": "string", - "required": true, - "description": "The tenant/organization identifier", - "enum": null, - "inferrable": true - }, - { - "name": "source", - "type": "string", - "required": false, - "description": "Source system or application creating the user", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Test2.CreateUser", - "parameters": { - "body": { - "value": "{\"email\": \"johndoe@example.com\", \"profile\": {\"contact_method\": \"email\", \"address\": {\"street\": \"123 Main St\", \"city\": \"Anytown\", \"state\": \"CA\", \"zip\": \"12345\"}, \"preferences\": {\"newsletter\": true, \"notifications\": false}}}", - "type": "string", - "required": true - }, - "tenant_id": { - "value": "tenant_12345", - "type": "string", - "required": true - }, - "source": { - "value": "web_app", - "type": "string", - "required": false - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "GetBodyScheme", - "qualifiedName": "Test2.GetBodyScheme", - "fullyQualifiedName": "Test2.GetBodyScheme@0.1.0", - "description": "Get the OpenAPI schema for a tool's request body.\n\nThis function returns the complete OpenAPI specification for the body parameter\nof the specified tool, with all references resolved. Use this to understand\nthe exact structure required before calling any of the API tools.\n\nAvailable tools:\n- create_user\n- create_order\n- create_event\n- update_document\n- submit_application", - "parameters": [ - { - "name": "tool_name", - "type": "string", - "required": true, - "description": "The name of the tool to get the body schema for", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Test2.GetBodyScheme", - "parameters": { - "tool_name": { - "value": "create_user", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "SubmitApplication", - "qualifiedName": "Test2.SubmitApplication", - "fullyQualifiedName": "Test2.SubmitApplication@0.1.0", - "description": "Submit an application (job, visa, loan, or university).\n\nNOTE: If you have not already called get_body_scheme('submit_application') or are unsure\nabout the body structure, call it first to see the exact schema required.\n\nThe body should include applicationType, personalInfo with identification\n(oneOf: passport/drivers_license/ssn), documents array, and additionalInfo\n(anyOf based on application type).", - "parameters": [ - { - "name": "body", - "type": "string", - "required": true, - "description": "JSON string containing application data. If unsure about structure, call get_body_scheme('submit_application') first.", - "enum": null, - "inferrable": true - }, - { - "name": "priority", - "type": "string", - "required": true, - "description": "Processing priority level", - "enum": null, - "inferrable": true - }, - { - "name": "notification_email", - "type": "string", - "required": true, - "description": "Email for status notifications", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Test2.SubmitApplication", - "parameters": { - "body": { - "value": "{\"applicationType\":\"loan\",\"personalInfo\":{\"identification\":{\"type\":\"passport\",\"number\":\"123456789\"}},\"documents\":[\"income_statement.pdf\",\"bank_statement.pdf\"],\"additionalInfo\":{\"loanAmount\":10000,\"loanPurpose\":\"home_improvement\"}}", - "type": "string", - "required": true - }, - "priority": { - "value": "high", - "type": "string", - "required": true - }, - "notification_email": { - "value": "applicant@example.com", - "type": "string", - "required": true - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - }, - { - "name": "UpdateDocument", - "qualifiedName": "Test2.UpdateDocument", - "fullyQualifiedName": "Test2.UpdateDocument@0.1.0", - "description": "Update a document with multiple operations.\n\nNOTE: If you have not already called get_body_scheme('update_document') or are unsure\nabout the body structure, call it first to see the exact schema required.\n\nThe body should include documentId and an array of operations\n(anyOf: insert/delete/replace/format). Each operation type has different required fields.", - "parameters": [ - { - "name": "body", - "type": "string", - "required": true, - "description": "JSON string containing document update operations. If unsure about structure, call get_body_scheme('update_document') first.", - "enum": null, - "inferrable": true - }, - { - "name": "workspace_id", - "type": "string", - "required": true, - "description": "The workspace containing the document", - "enum": null, - "inferrable": true - }, - { - "name": "user_id", - "type": "string", - "required": true, - "description": "User performing the update", - "enum": null, - "inferrable": true - }, - { - "name": "track_changes", - "type": "boolean", - "required": false, - "description": "Whether to track changes", - "enum": null, - "inferrable": true - } - ], - "auth": null, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "No description provided." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Test2.UpdateDocument", - "parameters": { - "body": { - "value": "{\"documentId\":\"12345\",\"operations\":[{\"insert\":{\"text\":\"Hello, World!\"}},{\"delete\":{\"text\":\"Old text\"}}]}", - "type": "string", - "required": true - }, - "workspace_id": { - "value": "ws-67890", - "type": "string", - "required": true - }, - "user_id": { - "value": "user-abc123", - "type": "string", - "required": true - }, - "track_changes": { - "value": true, - "type": "boolean", - "required": false - } - }, - "requiresAuth": false, - "tabLabel": "Call the Tool" - } - } - ], - "documentationChunks": [], - "customImports": [], - "subPages": [], - "generatedAt": "2026-01-26T17:43:54.426Z", - "summary": "Test2 provides a streamlined toolkit for developers to create and manage events, orders, users, applications, and documents through a set of versatile API tools. This toolkit simplifies the integration process, allowing for efficient handling of complex data structures without requiring authentication.\n\n**Capabilities**\n- Create and manage calendar events with detailed attendee and location information.\n- Handle orders with dynamic pricing and various payment methods.\n- Create user profiles with comprehensive data and preferences.\n- Submit applications across multiple domains with required and optional documentation.\n- Update existing documents with flexible operations.\n\nNo authentication or secrets are required to utilize this toolkit, enabling straightforward implementation in various applications." -} diff --git a/toolkit-docs-generator/scripts/check-toolkit-coverage.ts b/toolkit-docs-generator/scripts/check-toolkit-coverage.ts new file mode 100644 index 000000000..d9ca384d0 --- /dev/null +++ b/toolkit-docs-generator/scripts/check-toolkit-coverage.ts @@ -0,0 +1,371 @@ +#!/usr/bin/env node +/** + * Toolkit coverage check + * + * Warns when toolkits in the Arcade Engine API are missing from: + * 1. data/toolkits/.json + * 2. app/en/resources/integrations//_meta.tsx sidebar entries + * + * Also warns about JSON files that have no matching API toolkit (orphaned files). + * + * Never exits non-zero. Use for CI awareness, not CI gating. + * + * Usage: + * ENGINE_API_URL=https://api.arcade.dev ENGINE_API_KEY=arc_... \ + * npx tsx toolkit-docs-generator/scripts/check-toolkit-coverage.ts + * + * # Local only (skip API call): + * npx tsx toolkit-docs-generator/scripts/check-toolkit-coverage.ts --skip-api + */ + +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const PROJECT_ROOT = resolve(__dirname, "..", ".."); + +// ā”€ā”€ Types ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +export interface ToolkitSummaryEntry { + name: string; + version: string; + toolCount: number; +} + +export interface CoverageReport { + /** API toolkits with no matching JSON data file */ + missingJson: ToolkitSummaryEntry[]; + /** JSON data files with no matching API toolkit (orphaned) */ + missingFromApi: string[]; + /** API toolkits present in JSON but absent from any sidebar _meta.tsx */ + missingFromSidebar: ToolkitSummaryEntry[]; + /** + * index.json entries whose individual JSON file does not exist. + * Indicates index.json was not updated after a file was deleted. + */ + indexOrphans: string[]; + /** + * Individual JSON files with no entry in index.json. + * Indicates index.json was not updated after a file was generated. + */ + missingFromIndex: string[]; +} + +// ā”€ā”€ Name normalisation ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Convert any toolkit identifier to a canonical lowercase slug for comparison. + * + * Handles the three naming conventions seen across the codebase: + * - API names: "GoogleCalendar" ā†’ "googlecalendar" + * - File stems: "googlecalendar" ā†’ "googlecalendar" + * - Sidebar keys: "google-calendar" ā†’ "googlecalendar" + */ +export function normalizeId(id: string): string { + return id.toLowerCase().replace(/-/g, ""); +} + +// ā”€ā”€ Core logic (pure, testable) ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Cross-reference all four data sources and return a coverage report. + * + * Matching uses {@link normalizeId} so "GoogleCalendar", "googlecalendar", + * and "google-calendar" all resolve to the same canonical key. + * + * @param apiToolkits Toolkit list from the API summary endpoint + * @param jsonFileStems Basenames (without .json) of files in data/toolkits/ + * @param sidebarEntries All sidebar keys collected from every _meta.tsx file + * @param indexIds Toolkit IDs listed in index.json + */ +export function checkCoverage( + apiToolkits: ToolkitSummaryEntry[], + jsonFileStems: string[], + sidebarEntries: Set, + indexIds: string[] = [] +): CoverageReport { + const jsonSet = new Set(jsonFileStems.map(normalizeId)); + const sidebarSet = new Set([...sidebarEntries].map(normalizeId)); + const indexSet = new Set(indexIds.map(normalizeId)); + + const missingJson: ToolkitSummaryEntry[] = []; + const missingFromSidebar: ToolkitSummaryEntry[] = []; + + for (const toolkit of apiToolkits) { + const key = normalizeId(toolkit.name); + if (!jsonSet.has(key)) { + missingJson.push(toolkit); + } else if (!sidebarSet.has(key)) { + // Only flag sidebar-missing for toolkits that DO have a JSON file. + // Toolkits already in missingJson are not double-counted here. + missingFromSidebar.push(toolkit); + } + } + + const apiKeys = new Set(apiToolkits.map((t) => normalizeId(t.name))); + const missingFromApi = jsonFileStems.filter( + (stem) => !apiKeys.has(normalizeId(stem)) + ); + + // index.json drift: entries in index with no individual file + const indexOrphans = indexIds.filter((id) => !jsonSet.has(normalizeId(id))); + + // index.json drift: individual files with no index entry + const missingFromIndex = jsonFileStems.filter( + (stem) => !indexSet.has(normalizeId(stem)) + ); + + return { + missingJson, + missingFromApi, + missingFromSidebar, + indexOrphans, + missingFromIndex, + }; +} + +// ā”€ā”€ Sidebar parser ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Parse sidebar entry keys from a _meta.tsx file's raw text content. + * + * Supports two formats: + * + * Old (static object literal): + * github: { ... + * "google-calendar": { ... + * + * New (createCategoryMeta helper): + * createCategoryMeta([{ slug: "github", ... }]) + * + * Skips separator entries whose keys start with "--". + */ +export function parseSidebarEntries(content: string): string[] { + const entries: string[] = []; + + // Old format: direct object-literal keys + const keyPattern = /^\s+(?:"([^"]+)"|([A-Za-z_$][A-Za-z0-9_$-]*))\s*:/gm; + for (const match of content.matchAll(keyPattern)) { + const key = match[1] ?? match[2] ?? ""; + if (key && !key.startsWith("--")) { + entries.push(key); + } + } + + // New format: createCategoryMeta([{ slug: "toolkit-id", ... }]) + const slugPattern = /^\s+slug:\s+"([^"]+)"/gm; + for (const match of content.matchAll(slugPattern)) { + const slug = match[1]; + if (slug) entries.push(slug); + } + + return entries; +} + +// ā”€ā”€ File readers ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Return the basenames (without .json) of every toolkit data file, + * excluding the aggregate index.json. + */ +export function readJsonFileStems(dataDir: string): string[] { + if (!existsSync(dataDir)) return []; + return readdirSync(dataDir) + .filter((f) => f.endsWith(".json") && f !== "index.json") + .map((f) => f.slice(0, -5)); +} + +/** + * Walk every category directory under integrationsDir and collect all + * sidebar entry keys found in _meta.tsx files. + */ +export function readSidebarEntries(integrationsDir: string): Set { + const all = new Set(); + if (!existsSync(integrationsDir)) return all; + + for (const category of readdirSync(integrationsDir)) { + const metaPath = join(integrationsDir, category, "_meta.tsx"); + if (!existsSync(metaPath)) continue; + const content = readFileSync(metaPath, "utf-8"); + for (const key of parseSidebarEntries(content)) { + all.add(key); + } + } + return all; +} + +/** + * Read toolkit IDs from index.json. + * Returns an empty array if the file is absent or malformed. + */ +export function readIndexIds(dataDir: string): string[] { + const indexPath = join(dataDir, "index.json"); + if (!existsSync(indexPath)) return []; + try { + const raw = JSON.parse(readFileSync(indexPath, "utf-8")) as unknown; + // Supports both a bare array and the { toolkits: [...] } wrapper format + const items: unknown[] = Array.isArray(raw) + ? raw + : Array.isArray((raw as Record)?.toolkits) + ? ((raw as Record).toolkits as unknown[]) + : []; + return items + .filter( + (item): item is { id: string } => + typeof item === "object" && + item !== null && + typeof (item as Record).id === "string" + ) + .map((item) => item.id); + } catch { + return []; + } +} + +// ā”€ā”€ API fetch ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +type ApiSummaryResponse = { + toolkits?: Array<{ name: string; version: string; tool_count: number }>; +}; + +async function fetchApiToolkits( + baseUrl: string, + apiKey: string +): Promise { + const url = `${baseUrl.replace(/\/+$/, "")}/v1/tool_metadata_summary`; + const res = await fetch(url, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + if (!res.ok) { + throw new Error(`API returned ${res.status} for ${url}`); + } + const data = (await res.json()) as ApiSummaryResponse; + return (data.toolkits ?? []).map((t) => ({ + name: t.name, + version: t.version, + toolCount: t.tool_count, + })); +} + +// ā”€ā”€ Report printer ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +function printSection( + items: readonly string[], + header: string, + footer: string +): void { + if (items.length === 0) return; + console.warn(header); + for (const item of items) { + console.warn(` ${item}`); + } + console.warn(footer); +} + +export function printReport(report: CoverageReport): void { + const hasIssues = + report.missingJson.length > 0 || + report.missingFromApi.length > 0 || + report.missingFromSidebar.length > 0 || + report.indexOrphans.length > 0 || + report.missingFromIndex.length > 0; + + if (!hasIssues) { + console.log( + "āœ… Toolkit coverage OK ā€” all API toolkits have JSON + sidebar entries." + ); + return; + } + + console.warn(`\nāš ļø Toolkit coverage warnings\n${"ā”€".repeat(50)}`); + + printSection( + report.missingJson.map( + (t) => `${t.name}@${t.version} (${t.toolCount} tools)` + ), + `\nšŸ“ In API but missing JSON data file (${report.missingJson.length}):`, + " ā†’ Run the toolkit docs generator to create these files." + ); + + printSection( + report.missingFromApi.map((stem) => `${stem}.json`), + `\nšŸ—‘ļø JSON files with no matching API toolkit ā€” possibly stale (${report.missingFromApi.length}):`, + " ā†’ Delete these files or verify the toolkit is still published." + ); + + printSection( + report.missingFromSidebar.map((t) => `${t.name}@${t.version}`), + `\nšŸ—‚ļø In API + JSON but missing from sidebar (${report.missingFromSidebar.length}):`, + " ā†’ Run: npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts" + ); + + printSection( + report.indexOrphans, + `\nšŸ“‹ index.json entries with no matching JSON file ā€” index is stale (${report.indexOrphans.length}):`, + " ā†’ Remove these entries from index.json or restore their data files." + ); + + printSection( + report.missingFromIndex.map((stem) => `${stem}.json`), + `\nšŸ“‹ JSON files not listed in index.json ā€” index is incomplete (${report.missingFromIndex.length}):`, + " ā†’ Add these toolkits to index.json or regenerate it with the toolkit docs generator." + ); + + console.warn(`\n${"ā”€".repeat(50)}`); + console.warn( + "ā„¹ļø These are warnings only ā€” build is not blocked by coverage gaps." + ); +} + +// ā”€ā”€ Entry point ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +async function main(): Promise { + const skipApi = process.argv.includes("--skip-api"); + const dataDir = join(PROJECT_ROOT, "toolkit-docs-generator/data/toolkits"); + const integrationsDir = join(PROJECT_ROOT, "app/en/resources/integrations"); + + const jsonStems = readJsonFileStems(dataDir); + const sidebarEntries = readSidebarEntries(integrationsDir); + const indexIds = readIndexIds(dataDir); + + let apiToolkits: ToolkitSummaryEntry[] = []; + + if (skipApi) { + console.log("ā„¹ļø --skip-api: checking local files only."); + } else { + const engineUrl = process.env.ENGINE_API_URL; + const engineKey = process.env.ENGINE_API_KEY; + + if (engineUrl && engineKey) { + try { + apiToolkits = await fetchApiToolkits(engineUrl, engineKey); + console.log(`ā„¹ļø Fetched ${apiToolkits.length} toolkits from API.`); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.warn(`āš ļø Could not fetch API toolkits: ${msg}`); + console.warn(" Continuing with local file checks only."); + } + } else { + console.warn( + "āš ļø ENGINE_API_URL or ENGINE_API_KEY not set ā€” skipping API check.\n" + + " Set both env vars to enable full toolkit coverage checking." + ); + } + } + + const report = checkCoverage( + apiToolkits, + jsonStems, + sidebarEntries, + indexIds + ); + printReport(report); +} + +main().catch((err) => { + // Always exit 0 ā€” this is a warning tool, never a CI blocker + console.error("Unexpected error in coverage check:", err); + process.exit(0); +}); diff --git a/toolkit-docs-generator/tests/scripts/check-toolkit-coverage.test.ts b/toolkit-docs-generator/tests/scripts/check-toolkit-coverage.test.ts new file mode 100644 index 000000000..4e794ac36 --- /dev/null +++ b/toolkit-docs-generator/tests/scripts/check-toolkit-coverage.test.ts @@ -0,0 +1,320 @@ +/** + * Tests for check-toolkit-coverage.ts + * + * Run with: npx vitest run tests/scripts/check-toolkit-coverage.test.ts + */ + +import { describe, expect, it } from "vitest"; +import { + checkCoverage, + normalizeId, + parseSidebarEntries, + readIndexIds, + readJsonFileStems, + readSidebarEntries, + type ToolkitSummaryEntry, +} from "../../scripts/check-toolkit-coverage"; + +// ā”€ā”€ Helpers ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +const entry = (name: string, toolCount = 5): ToolkitSummaryEntry => ({ + name, + version: "1.0.0", + toolCount, +}); + +// ā”€ā”€ normalizeId ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("normalizeId", () => { + it("lowercases plain names", () => { + expect(normalizeId("Github")).toBe("github"); + }); + + it("strips hyphens (kebab-case sidebar keys)", () => { + expect(normalizeId("google-calendar")).toBe("googlecalendar"); + }); + + it("lowercases and strips hyphens together", () => { + expect(normalizeId("Google-Calendar")).toBe("googlecalendar"); + }); + + it("handles already-normalized names", () => { + expect(normalizeId("googlecalendar")).toBe("googlecalendar"); + }); +}); + +// ā”€ā”€ checkCoverage ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("checkCoverage", () => { + it("returns empty report when everything matches", () => { + const apiToolkits = [entry("Github"), entry("Slack")]; + const jsonFiles = ["github", "slack"]; + const sidebarEntries = new Set(["github", "slack"]); + + const report = checkCoverage(apiToolkits, jsonFiles, sidebarEntries); + + expect(report.missingJson).toHaveLength(0); + expect(report.missingFromApi).toHaveLength(0); + expect(report.missingFromSidebar).toHaveLength(0); + }); + + it("detects API toolkit with no JSON file", () => { + const apiToolkits = [entry("Github"), entry("Attio")]; + const jsonFiles = ["github"]; + const sidebarEntries = new Set(["github"]); + + const report = checkCoverage(apiToolkits, jsonFiles, sidebarEntries); + + expect(report.missingJson).toEqual([ + { name: "Attio", version: "1.0.0", toolCount: 5 }, + ]); + expect(report.missingFromSidebar).toHaveLength(0); + }); + + it("detects JSON file with no API entry (orphaned)", () => { + const apiToolkits = [entry("Github")]; + const jsonFiles = ["github", "complextools", "test2"]; + const sidebarEntries = new Set(["github", "complextools", "test2"]); + + const report = checkCoverage(apiToolkits, jsonFiles, sidebarEntries); + + expect(report.missingFromApi).toContain("complextools"); + expect(report.missingFromApi).toContain("test2"); + }); + + it("detects API toolkit with JSON but missing from sidebar", () => { + const apiToolkits = [entry("Github"), entry("Slack")]; + const jsonFiles = ["github", "slack"]; + const sidebarEntries = new Set(["github"]); // slack missing + + const report = checkCoverage(apiToolkits, jsonFiles, sidebarEntries); + + expect(report.missingFromSidebar).toEqual([ + { name: "Slack", version: "1.0.0", toolCount: 5 }, + ]); + expect(report.missingJson).toHaveLength(0); + }); + + it("does not double-count: toolkit missing JSON is not also flagged as missing sidebar", () => { + const apiToolkits = [entry("Attio")]; + const jsonFiles: string[] = []; + const sidebarEntries = new Set(); + + const report = checkCoverage(apiToolkits, jsonFiles, sidebarEntries); + + expect(report.missingJson).toHaveLength(1); + expect(report.missingFromSidebar).toHaveLength(0); + }); + + it("normalizes names case-insensitively for JSON matching", () => { + const apiToolkits = [entry("GoogleCalendar")]; + const jsonFiles = ["googlecalendar"]; // lowercase file on disk + const sidebarEntries = new Set(["google-calendar"]); // kebab in _meta.tsx + + const report = checkCoverage(apiToolkits, jsonFiles, sidebarEntries); + + expect(report.missingJson).toHaveLength(0); + expect(report.missingFromSidebar).toHaveLength(0); + }); + + it("handles empty API response gracefully", () => { + const report = checkCoverage([], ["github", "slack"], new Set(["github"])); + + expect(report.missingJson).toHaveLength(0); + expect(report.missingFromApi).toEqual(["github", "slack"]); + expect(report.missingFromSidebar).toHaveLength(0); + }); + + it("handles empty local files gracefully", () => { + const apiToolkits = [entry("Github")]; + const report = checkCoverage(apiToolkits, [], new Set()); + + expect(report.missingJson).toHaveLength(1); + expect(report.missingFromSidebar).toHaveLength(0); + expect(report.missingFromApi).toHaveLength(0); + }); + + it("handles all sources empty", () => { + const report = checkCoverage([], [], new Set()); + + expect(report.missingJson).toHaveLength(0); + expect(report.missingFromApi).toHaveLength(0); + expect(report.missingFromSidebar).toHaveLength(0); + }); + + it("correctly counts tool count in missing entries", () => { + const apiToolkits = [entry("Attio", 42)]; + const report = checkCoverage(apiToolkits, [], new Set()); + + expect(report.missingJson[0]?.toolCount).toBe(42); + }); +}); + +// ā”€ā”€ parseSidebarEntries ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("parseSidebarEntries", () => { + it("extracts bare identifier keys", () => { + const content = ` +const meta = { + github: { title: "GitHub", href: "/en/resources/integrations/development/github" }, + slack: { title: "Slack", href: "/en/resources/integrations/social/slack" }, +}; +export default meta; + `; + const entries = parseSidebarEntries(content); + expect(entries).toContain("github"); + expect(entries).toContain("slack"); + }); + + it("extracts quoted string keys (kebab-case)", () => { + const content = ` +const meta = { + "google-calendar": { title: "Google Calendar", href: "..." }, +}; +export default meta; + `; + const entries = parseSidebarEntries(content); + expect(entries).toContain("google-calendar"); + }); + + it("excludes separator entries starting with --", () => { + const content = ` +const meta = { + "-- Optimized": { type: "separator", title: "Optimized" }, + github: { title: "GitHub", href: "..." }, + "-- Starter": { type: "separator", title: "Starter" }, + "arcade-engine-api": { title: "Arcade Engine API", href: "..." }, +}; +export default meta; + `; + const entries = parseSidebarEntries(content); + expect(entries).not.toContain("-- Optimized"); + expect(entries).not.toContain("-- Starter"); + expect(entries).toContain("github"); + expect(entries).toContain("arcade-engine-api"); + }); + + it("returns empty array for empty content", () => { + expect(parseSidebarEntries("")).toHaveLength(0); + }); + + it("extracts slug values from createCategoryMeta format", () => { + const content = ` +import { createCategoryMeta } from "../create-category-meta"; + +export default createCategoryMeta([ + { + slug: "imgflip", + title: "Imgflip", + href: "/en/resources/integrations/entertainment/imgflip", + type: "arcade", + }, + { + slug: "spotify", + title: "Spotify", + href: "/en/resources/integrations/entertainment/spotify", + type: "arcade", + }, +]); + `; + const entries = parseSidebarEntries(content); + expect(entries).toContain("imgflip"); + expect(entries).toContain("spotify"); + }); + + it("handles type: separator format without crashes", () => { + const content = ` +const meta: MetaRecord = { + "-- Section": { type: "separator", title: "Section" }, + brightdata: { title: "Bright Data", href: "/..." }, +}; + `; + const entries = parseSidebarEntries(content); + expect(entries).toContain("brightdata"); + }); +}); + +// ā”€ā”€ readJsonFileStems ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("readJsonFileStems", () => { + it("returns empty array for a non-existent directory", () => { + const stems = readJsonFileStems("/this/path/does/not/exist"); + expect(stems).toEqual([]); + }); +}); + +// ā”€ā”€ readSidebarEntries ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("readSidebarEntries", () => { + it("returns empty set for a non-existent directory", () => { + const entries = readSidebarEntries("/this/path/does/not/exist"); + expect(entries.size).toBe(0); + }); +}); + +// ā”€ā”€ readIndexIds ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("readIndexIds", () => { + it("returns empty array for a non-existent directory", () => { + expect(readIndexIds("/this/path/does/not/exist")).toEqual([]); + }); +}); + +// ā”€ā”€ index.json drift via checkCoverage ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("checkCoverage ā€” index.json drift", () => { + it("is clean when index matches JSON files exactly", () => { + const report = checkCoverage([], ["github", "slack"], new Set(), [ + "Github", + "Slack", + ]); + expect(report.indexOrphans).toHaveLength(0); + expect(report.missingFromIndex).toHaveLength(0); + }); + + it("detects index entry whose JSON file was deleted", () => { + // github.json was deleted but index.json still has Github + const report = checkCoverage([], ["slack"], new Set(), ["Github", "Slack"]); + expect(report.indexOrphans).toContain("Github"); + expect(report.missingFromIndex).toHaveLength(0); + }); + + it("detects JSON file not listed in index.json", () => { + // slack.json was added but index.json was not updated + const report = checkCoverage([], ["github", "slack"], new Set(), [ + "Github", + ]); + expect(report.missingFromIndex).toContain("slack"); + expect(report.indexOrphans).toHaveLength(0); + }); + + it("normalizes casing when comparing index IDs to file stems", () => { + // index uses CamelCase; files are lowercase on disk + const report = checkCoverage([], ["googlecalendar"], new Set(), [ + "GoogleCalendar", + ]); + expect(report.indexOrphans).toHaveLength(0); + expect(report.missingFromIndex).toHaveLength(0); + }); + + it("works with empty index (backward compat if index.json is missing)", () => { + const report = checkCoverage([], ["github", "slack"], new Set(), []); + expect(report.missingFromIndex).toEqual(["github", "slack"]); + expect(report.indexOrphans).toHaveLength(0); + }); + + it("flags the exact toolkits that were stale in the original incident", () => { + // Simulates complextools / deepwiki / test2 being deleted from JSON + // but left dangling in index.json + const report = checkCoverage([], ["github"], new Set(["github"]), [ + "Github", + "ComplexTools", + "Deepwiki", + "Test2", + ]); + expect(report.indexOrphans).toEqual( + expect.arrayContaining(["ComplexTools", "Deepwiki", "Test2"]) + ); + expect(report.indexOrphans).not.toContain("Github"); + }); +}); From 8c169033c834d5c0b218397a3fba9c900e0ab1a6 Mon Sep 17 00:00:00 2001 From: jottakka Date: Tue, 3 Mar 2026 17:39:09 -0300 Subject: [PATCH 2/8] adding to exclusion list --- toolkit-docs-generator/excluded-toolkits.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/toolkit-docs-generator/excluded-toolkits.txt b/toolkit-docs-generator/excluded-toolkits.txt index a090177af..37f6033cb 100644 --- a/toolkit-docs-generator/excluded-toolkits.txt +++ b/toolkit-docs-generator/excluded-toolkits.txt @@ -1,3 +1,8 @@ # Toolkits to exclude from docs generation Google Microsoft +ComplexTools +Deepwiki +Test2 +MyServer +Mytest From 9844e337153b3ff2ced1b84e624b52e8eab1fe70 Mon Sep 17 00:00:00 2001 From: jottakka Date: Tue, 3 Mar 2026 17:50:21 -0300 Subject: [PATCH 3/8] simplifying action --- .github/workflows/check-toolkit-coverage.yml | 125 +------------------ 1 file changed, 1 insertion(+), 124 deletions(-) diff --git a/.github/workflows/check-toolkit-coverage.yml b/.github/workflows/check-toolkit-coverage.yml index d2d8a37d5..824ec6304 100644 --- a/.github/workflows/check-toolkit-coverage.yml +++ b/.github/workflows/check-toolkit-coverage.yml @@ -1,11 +1,4 @@ name: Check toolkit coverage -# Runs on every PR to main. Never fails ā€” surfaces gaps as GitHub warnings -# and a PR comment so authors are informed without being blocked. -# -# Checks: -# 1. API toolkits missing a JSON data file in toolkit-docs-generator/data/toolkits/ -# 2. JSON data files with no matching API toolkit (stale orphans) -# 3. API toolkits present in JSON but absent from any sidebar _meta.tsx on: workflow_dispatch: @@ -16,7 +9,6 @@ on: permissions: contents: read - pull-requests: write jobs: coverage: @@ -40,122 +32,7 @@ jobs: run: pnpm install - name: Run toolkit coverage check - id: coverage - run: | - pnpm run check:toolkit-coverage 2>&1 | tee coverage-output.txt - if grep -q "āš ļø Toolkit coverage warnings" coverage-output.txt; then - echo "has_warnings=true" >> "$GITHUB_OUTPUT" - else - echo "has_warnings=false" >> "$GITHUB_OUTPUT" - fi + run: pnpm run check:toolkit-coverage env: ENGINE_API_URL: ${{ secrets.ENGINE_API_URL }} ENGINE_API_KEY: ${{ secrets.ENGINE_API_KEY }} - - - name: Emit GitHub warning annotations - if: steps.coverage.outputs.has_warnings == 'true' - run: | - # Emit one ::warning:: annotation per gap category so GitHub surfaces - # a yellow warning badge on the check without marking it as failed. - if grep -q "šŸ“ In API but missing JSON" coverage-output.txt; then - missing=$(grep -A 100 "šŸ“ In API but missing JSON" coverage-output.txt \ - | grep "^ [A-Z]" | sed 's/^ //' | paste -sd ', ') - echo "::warning::Toolkits in API with no JSON data file: ${missing}" - fi - - if grep -q "šŸ—‘ļø JSON files with no matching" coverage-output.txt; then - stale=$(grep -A 100 "šŸ—‘ļø JSON files with no matching" coverage-output.txt \ - | grep "^ [a-z]" | sed 's/^ //' | paste -sd ', ') - echo "::warning::Stale JSON files (no matching API toolkit): ${stale}" - fi - - if grep -q "šŸ—‚ļø In API + JSON but missing from sidebar" coverage-output.txt; then - sidebar=$(grep -A 100 "šŸ—‚ļø In API + JSON but missing from sidebar" coverage-output.txt \ - | grep "^ [A-Z]" | sed 's/^ //' | paste -sd ', ') - echo "::warning::Toolkits missing from sidebar _meta.tsx: ${sidebar}" - fi - - - name: Write job summary - if: always() - run: | - if [ "$(cat coverage-output.txt | grep -c 'āš ļø Toolkit coverage warnings')" -gt 0 ]; then - echo "## āš ļø Toolkit coverage ā€” gaps found" >> "$GITHUB_STEP_SUMMARY" - else - echo "## āœ… Toolkit coverage ā€” all clear" >> "$GITHUB_STEP_SUMMARY" - fi - echo "" >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - cat coverage-output.txt >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - - - name: Post or update PR comment - if: always() - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const output = fs.readFileSync('coverage-output.txt', 'utf8'); - const hasWarnings = output.includes('āš ļø Toolkit coverage warnings'); - const marker = ''; - - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - - const existing = comments.find(c => - c.user.type === 'Bot' && c.body.includes(marker) - ); - - if (!hasWarnings) { - if (existing) { - await github.rest.issues.deleteComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - }); - } - return; - } - - const body = `${marker} -## āš ļø Toolkit coverage gaps detected - -The coverage check found gaps between the Arcade API and the local toolkit data files. These are **warnings only** ā€” this PR is not blocked from merging. - -
-šŸ“‹ Full report - -\`\`\` -${output} -\`\`\` - -
- -### What each warning means - -| Warning | Cause | Fix | -|---|---|---| -| **In API but missing JSON** | A toolkit is published in the API but has no docs data file | Run the toolkit docs generator for those toolkits | -| **Stale JSON files** | A JSON data file exists locally but has no matching toolkit in the API | Delete the file or verify the toolkit is still published | -| **Missing from sidebar** | A toolkit has JSON but isn't listed in any \`_meta.tsx\` | Run \`npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts\` | - ---- -*This comment updates automatically when the PR is pushed. It is removed when all gaps are resolved.*`; - - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } From a4249fd11721157778bd7d8cb7bd5a6f85f1102f Mon Sep 17 00:00:00 2001 From: jottakka Date: Tue, 3 Mar 2026 17:56:12 -0300 Subject: [PATCH 4/8] Renamed workflow to "Toolkit docs coverage" and updated file names. --- .github/workflows/check-toolkit-coverage.yml | 4 +- .../scripts/check-toolkit-coverage.ts | 56 +++++++-- .../scripts/check-toolkit-coverage.test.ts | 107 ++++++++++++++++++ 3 files changed, 158 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check-toolkit-coverage.yml b/.github/workflows/check-toolkit-coverage.yml index 824ec6304..7d4a3c3eb 100644 --- a/.github/workflows/check-toolkit-coverage.yml +++ b/.github/workflows/check-toolkit-coverage.yml @@ -1,4 +1,4 @@ -name: Check toolkit coverage +name: Toolkit docs coverage on: workflow_dispatch: @@ -12,7 +12,7 @@ permissions: jobs: coverage: - name: Toolkit coverage check + name: Toolkit docs coverage runs-on: ubuntu-latest steps: diff --git a/toolkit-docs-generator/scripts/check-toolkit-coverage.ts b/toolkit-docs-generator/scripts/check-toolkit-coverage.ts index d9ca384d0..06641a90c 100644 --- a/toolkit-docs-generator/scripts/check-toolkit-coverage.ts +++ b/toolkit-docs-generator/scripts/check-toolkit-coverage.ts @@ -79,22 +79,26 @@ export function normalizeId(id: string): string { * @param jsonFileStems Basenames (without .json) of files in data/toolkits/ * @param sidebarEntries All sidebar keys collected from every _meta.tsx file * @param indexIds Toolkit IDs listed in index.json + * @param skipIds Toolkit IDs from ignored-toolkits.txt + excluded-toolkits.txt */ export function checkCoverage( apiToolkits: ToolkitSummaryEntry[], jsonFileStems: string[], sidebarEntries: Set, - indexIds: string[] = [] + indexIds: string[] = [], + skipIds: Set = new Set() ): CoverageReport { const jsonSet = new Set(jsonFileStems.map(normalizeId)); const sidebarSet = new Set([...sidebarEntries].map(normalizeId)); const indexSet = new Set(indexIds.map(normalizeId)); + const skipSet = new Set([...skipIds].map(normalizeId)); const missingJson: ToolkitSummaryEntry[] = []; const missingFromSidebar: ToolkitSummaryEntry[] = []; for (const toolkit of apiToolkits) { const key = normalizeId(toolkit.name); + if (skipSet.has(key)) continue; // intentionally excluded or ignored if (!jsonSet.has(key)) { missingJson.push(toolkit); } else if (!sidebarSet.has(key)) { @@ -106,15 +110,19 @@ export function checkCoverage( const apiKeys = new Set(apiToolkits.map((t) => normalizeId(t.name))); const missingFromApi = jsonFileStems.filter( - (stem) => !apiKeys.has(normalizeId(stem)) + (stem) => + !(apiKeys.has(normalizeId(stem)) || skipSet.has(normalizeId(stem))) ); - // index.json drift: entries in index with no individual file - const indexOrphans = indexIds.filter((id) => !jsonSet.has(normalizeId(id))); + // index.json drift: entries in index with no individual file (skip intentional exclusions) + const indexOrphans = indexIds.filter( + (id) => !(jsonSet.has(normalizeId(id)) || skipSet.has(normalizeId(id))) + ); // index.json drift: individual files with no index entry const missingFromIndex = jsonFileStems.filter( - (stem) => !indexSet.has(normalizeId(stem)) + (stem) => + !(indexSet.has(normalizeId(stem)) || skipSet.has(normalizeId(stem))) ); return { @@ -224,6 +232,34 @@ export function readIndexIds(dataDir: string): string[] { } } +/** + * Parse a plain-text toolkit list file (ignored-toolkits.txt or excluded-toolkits.txt). + * Lines starting with "#" and blank lines are ignored. + */ +export function parseSkipFile(content: string): string[] { + return content + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length > 0 && !l.startsWith("#")); +} + +/** + * Read and merge both skip-list files into one set of normalized IDs. + * Missing files are silently ignored. + */ +export function readSkipIds(generatorDir: string): Set { + const files = ["excluded-toolkits.txt", "ignored-toolkits.txt"]; + const ids = new Set(); + for (const file of files) { + const filePath = join(generatorDir, file); + if (!existsSync(filePath)) continue; + for (const id of parseSkipFile(readFileSync(filePath, "utf-8"))) { + ids.add(normalizeId(id)); + } + } + return ids; +} + // ā”€ā”€ API fetch ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ type ApiSummaryResponse = { @@ -323,12 +359,17 @@ export function printReport(report: CoverageReport): void { async function main(): Promise { const skipApi = process.argv.includes("--skip-api"); - const dataDir = join(PROJECT_ROOT, "toolkit-docs-generator/data/toolkits"); + const generatorDir = join(PROJECT_ROOT, "toolkit-docs-generator"); + const dataDir = join(generatorDir, "data/toolkits"); const integrationsDir = join(PROJECT_ROOT, "app/en/resources/integrations"); const jsonStems = readJsonFileStems(dataDir); const sidebarEntries = readSidebarEntries(integrationsDir); const indexIds = readIndexIds(dataDir); + const skipIds = readSkipIds(generatorDir); + console.log( + `ā„¹ļø Skipping ${skipIds.size} toolkits from exclude/ignore lists.` + ); let apiToolkits: ToolkitSummaryEntry[] = []; @@ -359,7 +400,8 @@ async function main(): Promise { apiToolkits, jsonStems, sidebarEntries, - indexIds + indexIds, + skipIds ); printReport(report); } diff --git a/toolkit-docs-generator/tests/scripts/check-toolkit-coverage.test.ts b/toolkit-docs-generator/tests/scripts/check-toolkit-coverage.test.ts index 4e794ac36..09daf4bf2 100644 --- a/toolkit-docs-generator/tests/scripts/check-toolkit-coverage.test.ts +++ b/toolkit-docs-generator/tests/scripts/check-toolkit-coverage.test.ts @@ -9,9 +9,11 @@ import { checkCoverage, normalizeId, parseSidebarEntries, + parseSkipFile, readIndexIds, readJsonFileStems, readSidebarEntries, + readSkipIds, type ToolkitSummaryEntry, } from "../../scripts/check-toolkit-coverage"; @@ -318,3 +320,108 @@ describe("checkCoverage ā€” index.json drift", () => { expect(report.indexOrphans).not.toContain("Github"); }); }); + +// ā”€ā”€ parseSkipFile ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("parseSkipFile", () => { + it("parses plain toolkit IDs", () => { + const result = parseSkipFile("Google\nMicrosoft\n"); + expect(result).toEqual(["Google", "Microsoft"]); + }); + + it("strips comment lines", () => { + const result = parseSkipFile( + "# Toolkits to exclude\nGoogle\n# another comment\nMicrosoft" + ); + expect(result).toEqual(["Google", "Microsoft"]); + }); + + it("strips blank lines", () => { + const result = parseSkipFile("Google\n\n\nMicrosoft\n"); + expect(result).toEqual(["Google", "Microsoft"]); + }); + + it("returns empty array for empty content", () => { + expect(parseSkipFile("")).toEqual([]); + expect(parseSkipFile("# only comments\n")).toEqual([]); + }); +}); + +// ā”€ā”€ readSkipIds ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("readSkipIds", () => { + it("returns empty set for a non-existent directory", () => { + expect(readSkipIds("/this/path/does/not/exist").size).toBe(0); + }); +}); + +// ā”€ā”€ skip list filtering via checkCoverage ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +describe("checkCoverage ā€” skip list filtering", () => { + const skipIds = new Set([ + "complextools", + "deepwiki", + "test2", + "myserver", + "mytest", + ]); + + it("does not flag excluded toolkits as missing JSON", () => { + const apiToolkits = [ + entry("ComplexTools"), + entry("Github"), + entry("Deepwiki"), + ]; + const report = checkCoverage( + apiToolkits, + ["github"], + new Set(["github"]), + [], + skipIds + ); + expect(report.missingJson).toHaveLength(0); + }); + + it("does not flag excluded toolkits as missing from sidebar", () => { + const apiToolkits = [entry("ComplexTools"), entry("Github")]; + const report = checkCoverage( + apiToolkits, + ["github", "complextools"], + new Set(["github"]), // complextools not in sidebar + [], + skipIds + ); + expect(report.missingFromSidebar).toHaveLength(0); + }); + + it("does not flag orphaned index entries for excluded toolkits", () => { + // ComplexTools is in index.json but its JSON file was deleted ā€” expected, it's excluded + const report = checkCoverage( + [], + ["github"], + new Set(), + ["Github", "ComplexTools"], + skipIds + ); + expect(report.indexOrphans).toHaveLength(0); + }); + + it("does not flag missing index entries for excluded toolkit files", () => { + // If somehow an excluded file exists on disk, don't warn about index missing + const report = checkCoverage( + [], + ["github", "complextools"], + new Set(), + ["Github"], + skipIds + ); + expect(report.missingFromIndex).not.toContain("complextools"); + }); + + it("still reports non-excluded toolkits normally", () => { + const apiToolkits = [entry("Attio"), entry("ComplexTools")]; + const report = checkCoverage(apiToolkits, [], new Set(), [], skipIds); + expect(report.missingJson).toHaveLength(1); + expect(report.missingJson[0]?.name).toBe("Attio"); + }); +}); From 3a05b540c0f2dfd821c3b30db2b305431867792e Mon Sep 17 00:00:00 2001 From: jottakka Date: Wed, 4 Mar 2026 17:08:07 +0000 Subject: [PATCH 5/8] Update excluded-toolkits.txt --- toolkit-docs-generator/excluded-toolkits.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/toolkit-docs-generator/excluded-toolkits.txt b/toolkit-docs-generator/excluded-toolkits.txt index 37f6033cb..56d33bc83 100644 --- a/toolkit-docs-generator/excluded-toolkits.txt +++ b/toolkit-docs-generator/excluded-toolkits.txt @@ -1,8 +1,4 @@ # Toolkits to exclude from docs generation Google Microsoft -ComplexTools -Deepwiki -Test2 -MyServer -Mytest + From 165a5d769a9d3ebffadec1b9ec77f6c64cf0155c Mon Sep 17 00:00:00 2001 From: jottakka Date: Wed, 4 Mar 2026 16:09:11 -0300 Subject: [PATCH 6/8] only warning when link broken --- .github/workflows/check-external-urls.yml | 35 +++++++++++++++++++++++ tests/external-url-check.test.ts | 27 +++++++++++------ vitest.config.ts | 10 ++++++- 3 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/check-external-urls.yml diff --git a/.github/workflows/check-external-urls.yml b/.github/workflows/check-external-urls.yml new file mode 100644 index 000000000..700636540 --- /dev/null +++ b/.github/workflows/check-external-urls.yml @@ -0,0 +1,35 @@ +name: External URL check + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + +permissions: + contents: read + +jobs: + check: + name: External URL check + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: pnpm + + - name: Install dependencies + run: pnpm install + + - name: Check external URLs + run: pnpm vitest run tests/external-url-check.test.ts diff --git a/tests/external-url-check.test.ts b/tests/external-url-check.test.ts index 635ed75e3..19bf22c62 100644 --- a/tests/external-url-check.test.ts +++ b/tests/external-url-check.test.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; import fg from "fast-glob"; -import { expect, test } from "vitest"; +import { test } from "vitest"; const TIMEOUT = 120_000; const CONCURRENCY = 10; @@ -223,16 +223,25 @@ test( await Promise.all(checks); - for (const failure of failures) { - console.error( - `Broken external URL: ${failure.url}` + - (failure.status ? ` (HTTP ${failure.status})` : "") + - (failure.error ? ` (${failure.error})` : "") + - ` ā€” found in: ${failure.files.join(", ")}` + if (failures.length > 0) { + console.warn( + `\nāš ļø External URL warnings (${failures.length} unreachable)\n${"ā”€".repeat(50)}` ); + for (const failure of failures) { + console.warn( + ` ${failure.url}` + + (failure.status ? ` ā†’ HTTP ${failure.status}` : "") + + (failure.error ? ` ā†’ ${failure.error}` : "") + + `\n in: ${failure.files.join(", ")}` + ); + } + console.warn("ā”€".repeat(50)); + console.warn( + "ā„¹ļø These are warnings only ā€” third-party URLs may be temporarily unreachable." + ); + } else { + console.log("āœ… All external URLs are reachable."); } - - expect(failures).toEqual([]); }, TIMEOUT ); diff --git a/vitest.config.ts b/vitest.config.ts index 8fb6f2dcf..70efbf8bf 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,3 +1,11 @@ import { defineConfig } from "vitest/config"; -export default defineConfig({}); +export default defineConfig({ + test: { + exclude: [ + // Runs in its own workflow ā€” third-party URLs are flaky in the main test run + "tests/external-url-check.test.ts", + "**/node_modules/**", + ], + }, +}); From 9f80a76ac0c8348804c3e365df53e2feaca11479 Mon Sep 17 00:00:00 2001 From: jottakka Date: Wed, 4 Mar 2026 16:18:50 -0300 Subject: [PATCH 7/8] fixing workflow --- .github/workflows/check-external-urls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-external-urls.yml b/.github/workflows/check-external-urls.yml index 700636540..f60519a50 100644 --- a/.github/workflows/check-external-urls.yml +++ b/.github/workflows/check-external-urls.yml @@ -32,4 +32,4 @@ jobs: run: pnpm install - name: Check external URLs - run: pnpm vitest run tests/external-url-check.test.ts + run: pnpm vitest run tests/external-url-check.test.ts --exclude "**/node_modules/**" From 48198a3d7804fdd58c2666e36e4eaed23a7db8df Mon Sep 17 00:00:00 2001 From: jottakka Date: Wed, 4 Mar 2026 16:26:06 -0300 Subject: [PATCH 8/8] fixing again --- .github/workflows/check-external-urls.yml | 2 +- vitest.external-urls.config.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 vitest.external-urls.config.ts diff --git a/.github/workflows/check-external-urls.yml b/.github/workflows/check-external-urls.yml index f60519a50..77264c8ea 100644 --- a/.github/workflows/check-external-urls.yml +++ b/.github/workflows/check-external-urls.yml @@ -32,4 +32,4 @@ jobs: run: pnpm install - name: Check external URLs - run: pnpm vitest run tests/external-url-check.test.ts --exclude "**/node_modules/**" + run: pnpm vitest run --config vitest.external-urls.config.ts diff --git a/vitest.external-urls.config.ts b/vitest.external-urls.config.ts new file mode 100644 index 000000000..1b71c5478 --- /dev/null +++ b/vitest.external-urls.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["tests/external-url-check.test.ts"], + }, +});