From 641ecb62cc18c60b84306b3e68a8c93e951f667b Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Fri, 19 Jun 2026 02:45:09 +0200 Subject: [PATCH 1/2] docs: add alerting investigation skill reference --- packages/cli/e2e/__tests__/skills.spec.ts | 14 +- .../ai-context/__tests__/references.spec.ts | 15 ++ packages/cli/src/ai-context/context.ts | 4 + .../references/investigate-alerting.md | 188 ++++++++++++++++++ 4 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/ai-context/references/investigate-alerting.md diff --git a/packages/cli/e2e/__tests__/skills.spec.ts b/packages/cli/e2e/__tests__/skills.spec.ts index 26d225d09..ca911b315 100644 --- a/packages/cli/e2e/__tests__/skills.spec.ts +++ b/packages/cli/e2e/__tests__/skills.spec.ts @@ -1,12 +1,13 @@ import { ExecaError } from 'execa' import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { ACTIONS, REFERENCES, SKILL } from '../../src/ai-context/context' +import { ACTIONS, INVESTIGATE_REFERENCES, REFERENCES, SKILL } from '../../src/ai-context/context' import { FixtureSandbox } from '../../src/testing/fixture-sandbox' import { runCheckly } from '../run-checkly' const actionIds = ACTIONS.map(a => a.id) const referenceShortIds = REFERENCES.map(r => r.id.replace('configure-', '')) +const investigateReferenceShortIds = INVESTIGATE_REFERENCES.map(r => r.id.replace('investigate-', '')) describe('checkly skills', () => { let fixt: FixtureSandbox @@ -29,6 +30,9 @@ describe('checkly skills', () => { for (const ref of REFERENCES) { expect(stdout).toContain(ref.description) } + for (const ref of INVESTIGATE_REFERENCES) { + expect(stdout).toContain(ref.description) + } }) it('should show example commands', async () => { @@ -37,6 +41,7 @@ describe('checkly skills', () => { expect(stdout).toContain(`$ checkly skills ${id}`) } expect(stdout).toContain(`$ checkly skills configure ${referenceShortIds[0]}`) + expect(stdout).toContain(`$ checkly skills investigate ${investigateReferenceShortIds[0]}`) }) it('should show the initialize action content', async () => { @@ -54,6 +59,13 @@ describe('checkly skills', () => { expect(stdout).toBeTruthy() }) + it('should show the alerting investigation reference', async () => { + const { stdout } = await runCheckly(fixt, ['skills', 'investigate', 'alerting']) + expect(stdout).toContain('Investigating Alerting Behavior') + expect(stdout).toContain('checkly checks list --output json --limit 100') + expect(stdout).toContain('Effective Alerting Decision Tree') + }) + it('should fail for an invalid action', async () => { try { await runCheckly(fixt, ['skills', 'nonexistent']) diff --git a/packages/cli/src/ai-context/__tests__/references.spec.ts b/packages/cli/src/ai-context/__tests__/references.spec.ts index 066733aa3..6e3f9275e 100644 --- a/packages/cli/src/ai-context/__tests__/references.spec.ts +++ b/packages/cli/src/ai-context/__tests__/references.spec.ts @@ -14,6 +14,21 @@ function normalizeLineEndings (content: string): string { } describe('AI context investigation references', () => { + it('documents read-only alerting investigation without claiming unavailable globals', async () => { + const content = normalizeLineEndings(await readReference('investigate-alerting.md')) + + expect(content).toContain('checkly checks list --output json --limit 100') + expect(content).toContain('checkly checks get --output json') + expect(content).toContain('checkly alert-channels list --output json --limit 100') + expect(content).toContain('checkly alert-channels get --output json') + expect(content).toContain('checkly api /v1/check-groups') + expect(content).toContain('checkly api /v1/checks/') + expect(content).toContain('Table output is insufficient for this scenario.') + expect(content).toContain('If `checks list --output json --limit 100` returns no checks with `groupId`, say\nthat group override analysis is unavailable from the current evidence') + expect(content).toContain('Treat account/global settings as known only when CLI or API output exposes') + expect(content).toContain('Do not deploy, update, delete, trigger checks,\nstart test runs, run RCA, mutate incidents, or change alert channels.') + }) + it('points test-session asset retrieval at the assets commands', async () => { const content = normalizeLineEndings(await readReference('investigate-test-sessions.md')) diff --git a/packages/cli/src/ai-context/context.ts b/packages/cli/src/ai-context/context.ts index 9452423fb..e6e563056 100644 --- a/packages/cli/src/ai-context/context.ts +++ b/packages/cli/src/ai-context/context.ts @@ -62,6 +62,10 @@ export const INVESTIGATE_REFERENCES = [ id: 'investigate-checks', description: 'Inspect checks (`checks list`, `checks get`), retry attempts, result assets, and trigger on-demand runs', }, + { + id: 'investigate-alerting', + description: 'Inspect effective alerting behavior for a check across check-local, group, global, and alert-channel evidence', + }, { id: 'investigate-test-sessions', description: 'Run and inspect recorded test sessions, drill into test-session error groups, run RCA, and list/download result assets', diff --git a/packages/cli/src/ai-context/references/investigate-alerting.md b/packages/cli/src/ai-context/references/investigate-alerting.md new file mode 100644 index 000000000..cb8af391c --- /dev/null +++ b/packages/cli/src/ai-context/references/investigate-alerting.md @@ -0,0 +1,188 @@ +# Investigating Alerting Behavior + +Use this read-only flow when a user asks which alert settings actually apply to +a deployed check, especially when check-local, group, global, and alert-channel +settings may interact. + +Table output is insufficient for this scenario. Use JSON or raw API evidence +whenever available because human tables can omit `alertSettings`, +`useGlobalAlertSettings`, `alertChannelSubscriptions`, `groupId`, `muted`, +`activated`, `retryStrategy`, and `doubleCheck`. + +Only use read-only commands. Do not deploy, update, delete, trigger checks, +start test runs, run RCA, mutate incidents, or change alert channels. When using +`checkly api`, only use `GET` requests; do not pass write methods, `-F` fields, +or request bodies. + +## Recommended Read-Only Command Sequence + +Start by identifying one concrete check. If the user gave a name instead of an +ID, search and inspect the JSON list output before choosing: + +```bash +checkly checks list --output json --limit 100 +checkly checks list --output json --limit 100 --search "" +``` + +Then inspect the selected check with structured output: + +```bash +checkly checks get --output json +checkly api /v1/checks/ +``` + +Use the CLI detail output for context only after you have captured JSON/API +evidence: + +```bash +checkly checks get +``` + +Inspect alert channels as structured evidence. The list helps identify candidate +channel IDs and subscription scopes; `get` provides the complete channel +payload: + +```bash +checkly alert-channels list --output json --limit 100 +checkly alert-channels get --output json +``` + +If the check has a `groupId`, fetch groups and locate the matching group. Use +the raw API because group override fields may not have a dedicated CLI command: + +```bash +checkly api /v1/check-groups +``` + +If `checks list --output json --limit 100` returns no checks with `groupId`, say +that group override analysis is unavailable from the current evidence and +continue with check, global-selection, and alert-channel evidence. Do not invent +group behavior. + +## Fields To Inspect + +For the selected check, capture and explain: + +- `activated` - whether the check is scheduled to run. +- `muted` - whether check-level alert notifications are suppressed. +- `groupId` - whether group settings may affect the check. +- `alertSettings` - check-local escalation policy evidence. +- `useGlobalAlertSettings` - whether the check selects account/global alert + settings instead of a local policy. +- `alertChannelSubscriptions` - check-local or group-scoped channel bindings, + including each subscription's `activated`, `checkId`, `groupId`, and channel + ID. +- `retryStrategy` and `doubleCheck` - retry or double-check behavior that can + delay or change when a failure becomes alert-worthy. + +For a matching group, capture and explain: + +- `activated` and `muted` - whether the group can stop runs or suppress alerts. +- `alertSettings` and `useGlobalAlertSettings` - whether the group forces a + custom or global escalation policy for grouped checks. +- `alertChannelSubscriptions`, or alert-channel subscriptions whose `groupId` + matches the group - which group-scoped channels are active. +- `retryStrategy` and `doubleCheck` - group-level retry behavior when present. + +For alert channels, capture and explain: + +- channel `id`, `type`, name/config label, and whether the channel itself is + active if the API exposes that field. +- subscription scope: check-local (`checkId` matches the check and `groupId` is + empty), group-scoped (`groupId` matches the check's group), or unrelated. +- subscription `activated` status. Inactive subscriptions are evidence of + configuration, not active alert delivery. + +## Effective Alerting Decision Tree + +Apply this tree from confirmed evidence only: + +1. Check run and notification gates: + - If the check has `activated: false`, report that it is inactive and should + not produce scheduled alerts unless other output proves otherwise. + - If the check has `muted: true`, report that check-level notifications are + suppressed. + - If a fetched group for this check has `activated: false`, report that the + group stops grouped checks from running when the API output confirms it. + - If a fetched group has `muted: true`, report that group-level muting + suppresses alerts for grouped checks. +2. Group override status: + - If the check has no `groupId`, group overrides do not apply to this check. + - If no grouped checks are present in the inspected list output, group + override analysis is unavailable from current evidence. + - If the check has a `groupId`, inspect the matching group before deciding + effective escalation or group channel behavior. + - If group `useGlobalAlertSettings` is `false` and group `alertSettings` is + present, report the group escalation policy as the effective policy for + the grouped check. + - If group `useGlobalAlertSettings` is `true`, report that the group selects + account/global escalation. Do not describe the global policy unless another + output shows its values. + - If group `useGlobalAlertSettings` is missing, `null`, or `undefined`, and + no group `alertSettings` is present, do not claim a group escalation + override. Continue to check-local evidence. + - If group-scoped alert channel subscriptions are present and active, report + them as group-scoped channels. If group channels exist but group policy + fields are unavailable, say whether channel applicability is unavailable + rather than assuming they apply. +3. Check-local escalation: + - If check `useGlobalAlertSettings` is `false` and check `alertSettings` is + present, report the check-local escalation policy as effective unless a + confirmed group override above supersedes it. + - If check `useGlobalAlertSettings` is `true`, report that the check selects + account/global escalation unless a confirmed group override above supersedes + it. + - If check `alertSettings` and `useGlobalAlertSettings` are absent or + contradictory, report the escalation source as unavailable instead of + inferring defaults. +4. Account/global settings: + - Treat account/global settings as known only when CLI or API output exposes + their values. + - If output only shows `useGlobalAlertSettings: true`, say "global alert + settings are selected, but their policy details were not available in the + inspected CLI/API output." +5. Retry and double-check behavior: + - If `retryStrategy` is present, explain how it affects when alerts can fire + based on the returned fields. + - If `doubleCheck` is present and `retryStrategy` is absent, explain the + double-check behavior as returned. + - If neither field is available, say retry or double-check behavior was not + available in the inspected output. + +## Response Template + +Use this structure when reporting back: + +```markdown +## Alerting evidence for () + +### Confirmed evidence +- Check state: activated=, muted=, groupId= +- Check escalation fields: useGlobalAlertSettings=, alertSettings= +- Retry fields: retryStrategy=, doubleCheck= +- Sources: + +### Effective escalation policy + + +### Alert channels +- Check-local channels: +- Group-scoped channels: +- Unrelated channels ignored: + +### Group override status + + +### Unavailable evidence +- Account/global alert policy details: +- Missing fields/endpoints: + +### Recommended next read-only checks +- ` or + `checkly alert-channels get --output json`.> +``` From d37f8e1a4431bd9995bef10088fa75f2494dc14f Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Fri, 19 Jun 2026 05:03:48 +0200 Subject: [PATCH 2/2] Tighten alerting investigation skill guidance --- .../ai-context/__tests__/references.spec.ts | 6 ++- .../references/investigate-alerting.md | 44 ++++++++++++++----- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/ai-context/__tests__/references.spec.ts b/packages/cli/src/ai-context/__tests__/references.spec.ts index 6e3f9275e..fb2717f5a 100644 --- a/packages/cli/src/ai-context/__tests__/references.spec.ts +++ b/packages/cli/src/ai-context/__tests__/references.spec.ts @@ -24,8 +24,12 @@ describe('AI context investigation references', () => { expect(content).toContain('checkly api /v1/check-groups') expect(content).toContain('checkly api /v1/checks/') expect(content).toContain('Table output is insufficient for this scenario.') - expect(content).toContain('If `checks list --output json --limit 100` returns no checks with `groupId`, say\nthat group override analysis is unavailable from the current evidence') + expect(content).toContain('Do not guess multiple possible account/global alerting\nendpoints.') + expect(content).toContain('Only call `get` for channel IDs referenced by the\nselected check or matching group subscriptions') + expect(content).toContain('If the selected check has no `groupId`, or if\n`checks list --output json --limit 100` returns no checks with `groupId`, stop') + expect(content).toContain('stop\ngroup override investigation') expect(content).toContain('Treat account/global settings as known only when CLI or API output exposes') + expect(content).toContain('Do not probe guessed endpoints such as\naccount-settings, account/settings, check-alerts, or alert-notifications') expect(content).toContain('Do not deploy, update, delete, trigger checks,\nstart test runs, run RCA, mutate incidents, or change alert channels.') }) diff --git a/packages/cli/src/ai-context/references/investigate-alerting.md b/packages/cli/src/ai-context/references/investigate-alerting.md index cb8af391c..acfd4b62f 100644 --- a/packages/cli/src/ai-context/references/investigate-alerting.md +++ b/packages/cli/src/ai-context/references/investigate-alerting.md @@ -14,6 +14,12 @@ start test runs, run RCA, mutate incidents, or change alert channels. When using `checkly api`, only use `GET` requests; do not pass write methods, `-F` fields, or request bodies. +Keep the investigation bounded. Use the commands and endpoints listed in this +reference, plus exact endpoints returned by those commands if they are needed to +follow a concrete ID. Do not guess multiple possible account/global alerting +endpoints. If a listed command or endpoint does not expose a field, report that +the field was unavailable in the inspected output instead of probing variants. + ## Recommended Read-Only Command Sequence Start by identifying one concrete check. If the user gave a name instead of an @@ -38,26 +44,38 @@ evidence: checkly checks get ``` -Inspect alert channels as structured evidence. The list helps identify candidate -channel IDs and subscription scopes; `get` provides the complete channel -payload: +Inspect alert channels as structured evidence. The list helps identify channel +IDs and subscription scopes. Only call `get` for channel IDs referenced by the +selected check or matching group subscriptions, or for a user-named channel you +must verify. Do not fetch every channel when there are no relevant +subscriptions: ```bash checkly alert-channels list --output json --limit 100 checkly alert-channels get --output json ``` -If the check has a `groupId`, fetch groups and locate the matching group. Use -the raw API because group override fields may not have a dedicated CLI command: +If the check has a `groupId`, fetch groups once and locate the matching group. +Use the raw API because group override fields may not have a dedicated CLI +command: ```bash checkly api /v1/check-groups ``` -If `checks list --output json --limit 100` returns no checks with `groupId`, say -that group override analysis is unavailable from the current evidence and -continue with check, global-selection, and alert-channel evidence. Do not invent -group behavior. +If the selected check has no `groupId`, or if +`checks list --output json --limit 100` returns no checks with `groupId`, stop +group override investigation. Say that group override analysis is unavailable +from the current evidence and continue with check, global-selection, and +alert-channel evidence. Do not invent group behavior and do not keep looking for +alternative group endpoints. + +This reference does not define a dedicated account/global alert-settings +endpoint. When `useGlobalAlertSettings: true` is the only account/global signal, +say that global alert settings are selected but their policy details were not +available in the inspected output. Do not probe guessed endpoints such as +account-settings, account/settings, check-alerts, or alert-notifications just to +fill that gap. ## Fields To Inspect @@ -138,6 +156,10 @@ Apply this tree from confirmed evidence only: 4. Account/global settings: - Treat account/global settings as known only when CLI or API output exposes their values. + - Do not try multiple guessed account/global alerting endpoints. Use a + documented endpoint only when this reference, CLI output, or the API + reference gives the exact path. If that one documented request fails or does + not include the needed fields, report the values as unavailable. - If output only shows `useGlobalAlertSettings: true`, say "global alert settings are selected, but their policy details were not available in the inspected CLI/API output." @@ -184,5 +206,7 @@ or affect the check based only on confirmed fields.> ### Recommended next read-only checks - ` or - `checkly alert-channels get --output json`.> + `checkly alert-channels get --output json` for a channel + ID referenced by a relevant subscription. Do not recommend guessed + account/global alerting endpoints.> ```