diff --git a/.github/workflows/agent-automations.yaml b/.github/workflows/agent-automations.yaml index e199c7be..8c1f7b14 100644 --- a/.github/workflows/agent-automations.yaml +++ b/.github/workflows/agent-automations.yaml @@ -2,124 +2,23 @@ name: Agent Automation on: schedule: - - cron: "0 6 * * *" + - cron: "0 6 * * *" # nightly UTC workflow_dispatch: + pull_request: + branches: + - main permissions: contents: read -concurrency: - group: provider-gap-audit-${{ github.ref }} - cancel-in-progress: false - jobs: - audit-sdk-coverage: - name: Audit SDK Coverage - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 - with: - app-id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} - private-key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} - owner: braintrustdata - repositories: | - braintrust-sdk-javascript - permission-contents: read - permission-issues: write - - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - - name: Run Claude provider gap audit - uses: anthropics/claude-code-action@df37d2f0760a4b5683a6e617c9325bc1a36443f6 # v1.0.75 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - # Pass our own app token so the Claude action does not try to mint its - # own GitHub token via OIDC. That OIDC path currently fails for this - # workflow shape upstream, and this app token also lets us tightly scope - # Claude's remote permissions to repo contents read + issues write only. - github_token: ${{ steps.app-token.outputs.token }} - prompt: | - # Goal - - Find important instrumentation gaps in this repository's AI-facing integrations. - - A gap means the upstream SDK or framework clearly supports something that this repository does not yet instrument, or instruments with materially less detail. - - # Scope - - - Infer the relevant surfaces from the checked-out repository itself. - - Focus on repo-owned AI instrumentation surfaces with meaningful upstream API, docs, and release surfaces. - - Ignore generic runtime or infrastructure integrations that do not have an upstream AI API surface to compare against. - - Do not open parity issues for deprecated or no-op surfaces unless you find real deprecation drift or docs drift worth reporting. - - # Process - - 1. Inspect local code, tests, docs, examples, and e2e scenarios to understand what is already instrumented. - 2. For each relevant surface, independently discover the current official upstream docs and recent official releases or changelogs. - 3. Compare current upstream capabilities to current Braintrust instrumentation in this repo. - 4. Always check the latest Braintrust docs at https://www.braintrust.dev/docs before deciding how to describe a gap. - 5. Search existing GitHub issues for duplicates before creating anything. - 6. Only act on high-confidence, concrete gaps tied to missing APIs, unsupported call patterns, or missing instrumentation detail. - - # Examples - - ## Good - - - The upstream SDK now has a stable `responses.stream()` helper, but this repo does not instrument it at all, or instruments it without final result metadata that other similar APIs already capture here. - - This repo instruments a provider's basic text generation API, but not its newer tool-calling or agent tracing API even though that API is now official and documented. - - ## Bad - - - A vague suspicion that "something in streaming may be missing" without a concrete upstream API and a concrete repo gap. - - Opening a separate issue for every release note bullet when they all describe the same missing instrumentation area. - - # If You Find Actionable Non-Duplicate Gaps - - - Create at most 5 issues in this run. - - Create one issue per distinct gap. - - Keep each issue concise, concrete, and source-backed. - - When creating an issue, always apply the `bot-automation` label. - - When creating an issue, always set the GitHub issue type to exactly one of `Bug`, `Feature`, or `Task`. - - Choose `Bug` when this repo appears to support the surface already but the instrumentation is incorrect, incomplete, or materially lower fidelity than expected. - - Choose `Feature` when the gap is a missing net-new instrumentation surface for an official upstream capability. - - Choose `Task` when the gap is concrete follow-up work that is neither a user-facing instrumentation bug nor a net-new instrumentation feature. - - Include a hidden marker comment near the top of the issue body in this exact form: - - ```html - - ``` - - Each issue should clearly include: - - - what instrumentation is missing - - whether Braintrust docs suggest the capability is `supported`, `unclear`, or `not_found` - - exact upstream sources - - exact Braintrust docs source or sources - - exact local repo files you inspected - - # Duplicate Handling - - - Do not create an issue if an open issue already covers the same gap. - - Treat a matching hidden marker comment or a clearly equivalent open issue as a duplicate. - - If duplicate checking is inconclusive, do not create the issue. - - # Constraints - - - Discover source URLs yourself. Do not rely on a preset list. - - Prefer official docs and official release sources. - - Do not create comments. - - Do not update, close, or label existing issues. - - Do not create pull requests. - - If there are no high-confidence non-duplicate gaps, do nothing. - # The Claude action includes base GitHub tools by default, and --allowedTools adds to that set - # rather than replacing it. Keep the deny-list so Claude cannot use other remote write tools. - claude_args: | - --model claude-opus-4-6 - --max-turns 20 - --allowedTools "Read,Glob,Grep,LS,WebSearch,WebFetch,mcp__github__get_issue,mcp__github__get_issue_comments,mcp__github__search_issues,mcp__github__list_issues,mcp__github__create_issue" - --disallowedTools "Bash,Edit,MultiEdit,Write,Replace,NotebookEditCell,mcp__github__create_issue_comment,mcp__github__update_issue,mcp__github__create_pr,mcp__github__create_or_update_file,mcp__github__delete_file,mcp__github_file_ops__commit_files,mcp__github_file_ops__delete_files" + provider-gap-audit: + uses: braintrustdata/sdk-actions/.github/workflows/agent-automation-reusable.yml@eeb44cc4c6ea61507d77c54c0bc430b96e7b1ea0 + with: + target_owner: ${{ github.repository_owner }} + target_repo: ${{ github.event.repository.name }} + max_issues: 3 + secrets: + braintrust_bot_app_id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} + braintrust_bot_private_key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}