diff --git a/.github/skills/issue-triage-report/SKILL.md b/.github/skills/issue-triage-report/SKILL.md index f88c19ed89..ffd6cfc5b2 100644 --- a/.github/skills/issue-triage-report/SKILL.md +++ b/.github/skills/issue-triage-report/SKILL.md @@ -51,8 +51,7 @@ The area contacts file maps feature areas to team members. This file is **requir { "areaContacts": { "area-FeatureName": { - "primary": "Primary Contact Name", - "secondary": "Secondary Contact or null", + "contact": "Contact Name", "notes": "Optional notes" } }, @@ -180,15 +179,25 @@ Retrieve or update the area-to-contact mapping configuration. Issues are scored (0-100) based on multiple factors. See [scoring-algorithm.md](./references/scoring-algorithm.md) for complete details. +Each score includes a confidence value `[confidence:XX]` indicating data reliability (grep-friendly format). + ### Quick Reference: Score Factors | Factor | Weight | Description | |--------|--------|-------------| | **Reactions** | 30 | Total reactions (👍, ❤️, 🚀, etc.) indicate community interest | -| **Age** | 25 | Older untriaged issues get higher priority | -| **Comments** | 20 | Active discussion indicates importance | -| **Severity** | 15 | Labels like `bug`, `regression`, `crash` increase score | -| **Blockers** | 10 | Issues blocking other work get prioritized | +| **Age** | 30 | Older untriaged issues get higher priority | +| **Comments** | 30 | Active discussion indicates importance | +| **Severity** | 10 | Labels like `bug`, `regression`, `crash`, `P0`-`P3` increase score | + +### Severity Label Tiers + +| Tier | Labels | Score | +|------|--------|-------| +| Critical | `regression`, `crash`, `hang`, `data-loss`, `P0` | 100% | +| High | `bug`, `P1` | 80% | +| Medium | `performance`, `feature proposal`, `P2` | 50% | +| Low | `documentation`, `enhancement`, `P3` | 20% | ### Highlight Labels (Output) @@ -196,12 +205,11 @@ The report adds reason labels to highlighted issues: | Label | Meaning | |-------|---------| -| `🔥 Hot` | High reaction count (community demand) | +| `🌟 Popular` | High reaction count (≥5 reactions) | | `⏰ Aging` | Open > 90 days without triage | | `🐛 Regression` | Marked as regression or recent breakage | | `🚧 Blocker` | Blocking other issues or teams | -| `📈 Trending` | High comment activity recently | -| ` Popular` | Feature proposal with significant support | +| `📈 Trending` | High comment activity (≥10 comments) | ## Report Output Format @@ -210,8 +218,8 @@ The report adds reason labels to highlighted issues: ```markdown | Feature Area | Area Contact | Open | Triage | Proposals | Closed | Highlights | |--------------|--------------|------|--------|-----------|--------|------------| -| area-Notification | Notifications Owner | 34 | 8 | 11 | 0 | 🔥 [#2894](link) Hot, ⏰ [#3001](link) Aging | -| area-Widgets | Widgets Owner | 21 | 10 | 4 | 0 | 📈 [#3958](link) Trending | +| area-Notification | Contact Name | 34 | 8 | 11 | 0 | 🌟 [#2894](link) [confidence:85], ⏰ [#3001](link) [confidence:72] | +| area-Widgets | Contact Name | 21 | 10 | 4 | 0 | 📈 [#3958](link) [confidence:68] | ``` ### Special Status Indicators @@ -226,7 +234,15 @@ The report adds reason labels to highlighted issues: ### Area Contacts -Contact mappings are stored in [area-contacts.md](./references/area-contacts.md). Update this file when team assignments change. +Contact mappings are stored in [area-contacts.json](./references/area-contacts.json). Update this file when team assignments change. + +```json +{ + "areaContacts": { + "area-FeatureName": { "contact": "Contact Name", "notes": "Optional notes" } + } +} +``` ### Custom Scoring Weights @@ -236,15 +252,22 @@ Modify scoring weights in `./scripts/ScoringConfig.json`: { "weights": { "reactions": 30, - "age": 25, - "comments": 20, - "severity": 15, - "blockers": 10 + "age": 30, + "comments": 30, + "severity": 10, + "blockers": 0 }, "thresholds": { - "hot_reactions": 10, "aging_days": 90, - "trending_comments": 5 + "trending_comments": 10, + "trending_days": 14, + "popular_reactions": 5 + }, + "severityLabels": { + "critical": ["regression", "crash", "hang", "data-loss", "P0"], + "high": ["bug", "P1"], + "medium": ["performance", "feature proposal", "feature-proposal", "P2"], + "low": ["documentation", "enhancement", "P3"] } } ``` @@ -262,8 +285,8 @@ Modify scoring weights in `./scripts/ScoringConfig.json`: ## Common Commands Reference ```powershell -# List all area labels -gh label list --repo microsoft/WindowsAppSDK --search "area-" --json name +# List all area labels (uses Get-RepositoryLabels.ps1 as the single source of truth) +./.github/skills/triage-meeting-prep/scripts/Get-RepositoryLabels.ps1 -Filter "area-*" -OutputFormat table # Get issue details with reactions gh issue view 4651 --repo microsoft/WindowsAppSDK --json number,title,labels,reactionGroups,createdAt,comments,author diff --git a/.github/skills/issue-triage-report/references/area-contacts.json b/.github/skills/issue-triage-report/references/area-contacts.json index 6a9c4f50d6..f548539287 100644 --- a/.github/skills/issue-triage-report/references/area-contacts.json +++ b/.github/skills/issue-triage-report/references/area-contacts.json @@ -2,9 +2,9 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$comment": "Area contacts configuration. Create your own copy at .user/issue-triage-report/area-contacts.json with actual team contacts.", "areaContacts": { - "area-ExampleFeature": { "primary": "Primary Contact Name", "secondary": null }, - "area-AnotherFeature": { "primary": "Primary Contact", "secondary": "Secondary Contact" }, - "area-FeatureWithNotes": { "primary": "Contact Name", "secondary": null, "notes": "Special handling notes" } + "area-ExampleFeature": { "contact": "Contact Name" }, + "area-AnotherFeature": { "contact": "Contact Name", "notes": "Optional notes about this area" }, + "area-FeatureWithNotes": { "contact": "Contact Name", "notes": "Special handling notes" } }, "specialAreas": { "triageOnly": ["area-TriageOnlyExample"], diff --git a/.github/skills/issue-triage-report/references/scoring-algorithm.md b/.github/skills/issue-triage-report/references/scoring-algorithm.md index 963ca6afe3..f7a6f58b67 100644 --- a/.github/skills/issue-triage-report/references/scoring-algorithm.md +++ b/.github/skills/issue-triage-report/references/scoring-algorithm.md @@ -1,244 +1,455 @@ -# Issue Highlight Scoring Algorithm +# ScoringConfig.json – Field-by-Field Guide -This document describes the scoring algorithm used to identify high-priority issues that should be highlighted in the Feature Area Status report. +This document explains every field in [`ScoringConfig.json`](../scripts/ScoringConfig.json) so you can tune issue highlight scoring without reading the PowerShell source. The config is loaded (and strictly validated) by `Get-ScoringConfig` in [`ReportLib.ps1`](../scripts/ReportLib.ps1). -## Overview +> **All fields are required.** The loader throws if any section or field is missing — there are no hidden defaults. -Each open issue receives a score from 0-100 based on weighted factors. Issues with the highest scores per feature area are marked as highlights with descriptive labels indicating why they're important. - -## Scoring Factors +--- -### 1. Reactions (Weight: 25 points max) +## File Location -Community engagement measured by GitHub reactions (👍, ❤️, 🚀, 👀, 🎉, 😕, 😄). +``` +.github/skills/issue-triage-report/scripts/ScoringConfig.json +``` -| Reactions | Score | -|-----------|-------| -| 0 | 0 | -| 1-4 | 5 | -| 5-9 | 10 | -| 10-19 | 15 | -| 20-49 | 20 | -| 50+ | 25 | +--- -**Rationale**: High reaction counts indicate community demand and widespread impact. +## Top-Level Structure -**Highlight Label**: `🔥 Hot` when reactions ≥ 10 +```jsonc +{ + "weights": { ... }, // How much each scoring factor matters (points) + "thresholds": { ... }, // Numeric cutoffs for highlight labels + "labelPriority": [ ... ], // Order for assigning highlight labels + "maxLabelsPerIssue": 2, // Cap on highlight labels per issue + "severityLabels": { ... } // Maps GitHub labels → severity tiers +} +``` --- -### 2. Age (Weight: 20 points max) - -Time since issue was created, with higher scores for older untriaged issues. - -| Days Open | Score | -|-----------|-------| -| 0-30 | 0 | -| 31-60 | 5 | -| 61-90 | 10 | -| 91-180 | 15 | -| 181+ | 20 | +## `weights` – Scoring Factor Points -**Rationale**: Older issues without resolution deserve attention to prevent backlog growth. +Controls how many points each factor can contribute to the 0–100 composite score. The values **must** sum to 100 (or less, if you intentionally disable a factor by setting it to 0). -**Highlight Label**: `⏰ Aging` when days > 90 AND has `needs-triage` label +| Field | Type | Description | +|-------|------|-------------| +| `reactions` | int | Max points from GitHub reaction count (👍 ❤️ 🚀 👀 🎉 😕 😄). Higher reaction counts award a larger fraction of this value. | +| `age` | int | Max points from issue age (days since creation). Older untriaged issues score higher. | +| `comments` | int | Max points from comment count. More discussion → higher score. | +| `severity` | int | Max points from severity label matching (see [`severityLabels`](#severitylabels--github-label-→-severity-tier)). | +| `blockers` | int | Max points for issues with blocking labels (`block`, `blocker`, `blocking`). Set to `0` to disable. | ---- +### How points are awarded within each factor -### 3. Comments (Weight: 15 points max) +The scoring engine divides each factor's weight into brackets. For example, with `"reactions": 30`: -Discussion activity measured by comment count. +| Raw Reactions | % of Weight | Points Awarded | +|---------------|-------------|----------------| +| 0 | 0% | 0 | +| 1–4 | 20% | 6 | +| 5–9 | 40% | 12 | +| 10–19 | 60% | 18 | +| 20–49 | 80% | 24 | +| 50+ | 100% | 30 | -| Comments | Score | -|----------|-------| -| 0 | 0 | -| 1-2 | 3 | -| 3-5 | 6 | -| 6-10 | 10 | -| 11+ | 15 | +The same 20/25/40/50/67/75/80/100% bracket pattern applies to `age` and `comments`. Changing the weight value scales all brackets proportionally. -**Recent Activity Bonus**: +5 if commented in last 14 days +**Severity** uses tier percentages instead (see below). -**Rationale**: Active discussions indicate ongoing relevance and potential blockers. +### Tuning tips -**Highlight Label**: `📈 Trending` when comments ≥ 5 AND recent activity (shows the issue is heating up NOW) +- **Emphasize community signal**: Increase `reactions` and `comments`, decrease `age`. +- **Prioritize backlog hygiene**: Increase `age`. +- **Re-enable blocker tracking**: Set `blockers` to a non-zero value (reduce other weights to keep the sum ≤ 100). --- -### 4. Severity Labels (Weight: 15 points max) +## `thresholds` – Highlight Label Cutoffs + +These values control when an issue earns a highlight label (🌟 Popular, ⏰ Aging, 📈 Trending). They do **not** affect the numeric score — only whether a label badge appears. -Priority based on issue labels indicating severity or type. +```json +"thresholds": { + "aging_days": 90, + "trending_comments": 10, + "trending_days": 14, + "popular_reactions": 5 +} +``` -| Label | Score | -|-------|-------| -| `regression` | 15 | -| `crash`, `hang`, `data-loss` | 12 | -| `bug` | 8 | -| `performance` | 6 | -| `documentation` | 2 | -| None of above | 0 | +| Field | Type | Used By | Description | +|-------|------|---------|-------------| +| `aging_days` | int | ⏰ Aging | Issue must be open longer than this **and** carry the `needs-triage` label to get the Aging badge. | +| `trending_comments` | int | 📈 Trending | Minimum comment count required for the Trending badge. | +| `trending_days` | int | 📈 Trending | Issue must have been updated within this many days **and** meet `trending_comments` to qualify. Ensures "trending" means trending *now*, not historically. | +| `popular_reactions` | int | 🌟 Popular | Minimum total reactions to earn the Popular badge. | -**Rationale**: Regressions and crashes have direct user impact. +### Tuning tips -**Highlight Labels**: -- `🐛 Regression` when has `regression` label -- (Bug severity shown in score, not separate label) +- **Tighter Trending window**: Lower `trending_days` to 7 to surface only very recent activity. +- **Broader Popular threshold**: Lower `popular_reactions` to 3 if your repo gets fewer reactions. +- **Stricter Aging**: Raise `aging_days` to 180 if a 90-day backlog is normal. --- -### 6. Blocker Status (Weight: 10 points max) +## `labelPriority` – Highlight Label Order -Issues that block other work or teams. +Determines which highlight labels are assigned first when an issue qualifies for more than `maxLabelsPerIssue` labels. Labels earlier in the array win. -| Condition | Score | -|-----------|-------| -| Has `blocking` or `blocker` label | 10 | -| Linked as blocking another issue | 8 | -| Mentioned in blocking context | 5 | -| No blocker indicators | 0 | +```json +"labelPriority": [ + "regression", + "blocker", + "popular", + "aging", + "trending" +] +``` + +| Position | Internal Name | Display Label | Condition | +|----------|---------------|---------------|-----------| +| 1 | `regression` | 🐛 Regression | Issue has a `regression` GitHub label | +| 2 | `blocker` | 🚧 Blocker | Issue has a label matching `block\|blocker\|blocking` | +| 3 | `popular` | 🌟 Popular | Reactions ≥ `thresholds.popular_reactions` | +| 4 | `aging` | ⏰ Aging | Days open > `thresholds.aging_days` AND has `needs-triage` label | +| 5 | `trending` | 📈 Trending | Comments ≥ `thresholds.trending_comments` AND updated within `thresholds.trending_days` | -**Rationale**: Blockers have multiplicative impact on productivity. +### Example -**Highlight Label**: `🚧 Blocker` when has blocking indicators +An issue with a `regression` label, 20 reactions, and 15 comments qualifies for Regression, Popular, and Trending. With `maxLabelsPerIssue: 2`, it receives **🐛 Regression** and **🌟 Popular** (positions 1 and 3 beat position 5). --- -## Composite Score Calculation +## `maxLabelsPerIssue` – Label Cap +```json +"maxLabelsPerIssue": 2 ``` -Total Score = Σ(factor_score × factor_weight / max_factor_weight) -Normalized Score = (Total Score / 100) × 100 +Maximum number of highlight labels attached to any single issue. After the engine evaluates all conditions (in `labelPriority` order), it keeps only the first N labels. Increase this if you want richer badge detail; decrease to keep reports cleaner. + +--- + +## `severityLabels` – GitHub Label → Severity Tier + +Maps GitHub issue label strings to four severity tiers. The scoring engine walks tiers top-to-bottom and awards points based on the **first matching label** it finds. + +```json +"severityLabels": { + "critical": ["regression", "crash", "hang", "data-loss", "P0"], + "high": ["bug", "P1"], + "medium": ["performance", "feature proposal", "feature-proposal", "P2"], + "low": ["documentation", "enhancement", "P3"] +} ``` -### Example Calculation +| Tier | % of `weights.severity` | Points (when severity=10) | When to use | +|------|-------------------------|---------------------------|-------------| +| `critical` | 100% | 10 | Regressions, crashes, data loss | +| `high` | 80% | 8 | Confirmed bugs | +| `medium` | 50% | 5 | Performance issues, feature proposals | +| `low` | 20% | 2 | Docs, enhancements | -Issue #2894: -- Reactions: 25 (score: 24) -- Age: 120 days (score: 18) -- Comments: 8 (score: 13) -- Labels: `bug` (score: 8) -- Blocker: No (score: 0) +### Customization -**Total**: 24 + 18 + 13 + 8 + 0 = **63/100** +- **Add a label**: Append it to the appropriate tier array (e.g., add `"memory-leak"` to `critical`). +- **Reclassify a label**: Move it between tier arrays. +- **Support custom P-labels**: The arrays already include `P0`–`P3`; add `P4`, `P5`, etc. to `low` if needed. +- **Order within a tier** doesn't matter — the engine matches the first label found on the issue against all tiers, checking from `critical` down to `low`. --- -## Highlight Label Assignment +## Composite Score Formula -After scoring, assign labels based on the highest-scoring factors: +``` +Total = reactions_pts + age_pts + comments_pts + severity_pts + blockers_pts + = weights.reactions + weights.age + weights.comments + weights.severity + weights.blockers + = 100 max (with current defaults) +``` -| Priority | Label | Condition | -|----------|-------|-----------| -| 1 | `🐛 Regression` | Has `regression` label | -| 2 | `🚧 Blocker` | Has blocking indicators | -| 3 | `🔥 Hot` | Reactions ≥ 10 (all-time popularity) | -| 4 | `⏰ Aging` | Days > 90 + needs-triage | -| 5 | `📈 Trending` | Comments ≥ 5 + recent activity (heating up NOW) | -| 6 | ` Popular` | Feature proposal + reactions ≥ 5 | +### Worked Example -**Rule**: Each issue gets **at most 2 labels** (most relevant based on score contribution). +Issue #2894 with 25 reactions, open 120 days, 8 comments, labeled `bug`: + +| Factor | Raw | Bracket | Points | +|--------|-----|---------|--------| +| Reactions | 25 | 20–49 → 80% | 24/30 | +| Age | 120 days | 91–180 → 75% | 22/30 | +| Comments | 8 | 6–10 → 67% | 20/30 | +| Severity | `bug` | high → 80% | 8/10 | +| Blockers | — | disabled | 0/0 | +| **Total** | | | **74/100** | --- -## Special Cases +## Confidence Scoring + +Each score carries a `[confidence:XX]` value (0–100) indicating data reliability. This is computed at runtime by `Get-ScoreConfidence` and is **not** configurable in `ScoringConfig.json`. -### Zero Issues (Celebration) +| Factor | Points | +|--------|--------| +| Has reactions data | +15 | +| Has comments data | +15 | +| Has labels | +15 | +| Has created date | +15 | +| Score ≥ 60 (clear priority) | +25 | +| Score 40–59 | +15 | +| Score 20–39 | +10 | +| 3+ factors contributing | +15 | +| 2 factors contributing | +10 | -When a feature area has zero open bugs: -- Display: `0️⃣🐛🥳` -- Meaning: Celebrate zero bugs! +```powershell +# Find high-confidence highlights (80+) +Select-String -Path report.md -Pattern '\[confidence:[89][0-9]\]' -### New Area +# Find low-confidence items (below 50) +Select-String -Path report.md -Pattern '\[confidence:[0-4][0-9]\]' +``` + +--- -When a feature area was recently created or has no historical data: -- Display: `🆕` -- Skip historical comparisons +## Validation Rules -### Redirect Area (area-External) +`Get-ScoringConfig` in `ReportLib.ps1` enforces these constraints at load time — if any fail, the script throws immediately: -For `area-External` label (issues to redirect to other teams): -- Display: `N/A` for highlights -- Note: "Current backlog contains issues for redirection to other teams" +| Rule | Error if violated | +|------|-------------------| +| File must be named `ScoringConfig.json` | `ConfigPath must point to ScoringConfig.json` | +| Top-level sections `weights`, `thresholds`, `maxLabelsPerIssue`, `severityLabels` must all exist | `missing required section: ''` | +| `weights` must contain: `reactions`, `age`, `comments`, `severity`, `blockers` | `'weights' missing required field: ''` | +| `thresholds` must contain: `aging_days`, `trending_comments`, `trending_days`, `popular_reactions` | `'thresholds' missing required field: ''` | +| `severityLabels` must contain: `critical`, `high`, `medium`, `low` | `'severityLabels' missing required level: ''` | + +`labelPriority` is the only optional field — if omitted, highlight labels are still assigned by the hardcoded priority in `Get-HighlightLabels`. --- -## Thresholds Configuration +## Script Architecture & Function Reference -These thresholds can be adjusted in `ScoringConfig.json`: +Five scripts implement the scoring system. All scoring-related scripts dot-source `ReportLib.ps1` and load `ScoringConfig.json` through `Get-ScoringConfig`. -```json -{ - "weights": { - "reactions": 30, - "age": 25, - "comments": 20, - "severity": 15, - "blockers": 10 - }, - "thresholds": { - "hot_reactions": 10, - "aging_days": 90, - "trending_comments": 5, - "trending_days": 14, - "popular_reactions": 5 - }, - "labelPriority": [ - "regression", - "blocker", - "hot", - "aging", - "trending", - "popular" - ], - "maxLabelsPerIssue": 2 +``` +ScoringConfig.json + │ + ▼ + Get-ScoringConfig() ◄── ReportLib.ps1 (shared library) + │ + ├──► Get-IssueScore() scores one issue (canonical math) + ├──► Get-HighlightLabels() assigns badge labels + └──► Get-ScoreConfidence() rates data reliability + │ + ┌──────┴──────────────────────────────────┐ + ▼ ▼ + Get-HighlightScore.ps1 Generate-FeatureAreaReport.ps1 + (single-issue CLI tool) (full report pipeline) + │ │ + ▼ ▼ + Get-DetailedIssueScore() Get-HighlightedIssues() + Format-ScoreBreakdown() Format-ReportMarkdown() + + Validate-FeatureAreaReport.ps1 Get-AreaContacts.ps1 + (live issue counts — no scoring) (contact lookup — no scoring) +``` + +--- + +### `ReportLib.ps1` — Shared Library + +Location: `./scripts/ReportLib.ps1` + +All scoring math lives here. Other scripts call these functions — they never duplicate the logic. + +#### Core Functions + +| Function | Purpose | Config Fields Read | +|----------|---------|-------------------| +| `Get-ScoringConfig` | Loads and validates `ScoringConfig.json`. Returns a hashtable consumed by every other function. | All fields (validation only) | +| `Get-IssueScore` | **Canonical scorer.** Takes one issue + config, returns a hashtable with per-factor scores (`Reactions`, `Age`, `Comments`, `Severity`, `Blockers`), raw values, and `Total`. | `weights.*`, `severityLabels.*` | +| `Get-HighlightLabels` | Evaluates label conditions and returns up to `maxLabelsPerIssue` badge strings (e.g., `🌟 Popular`). Checks in `labelPriority` order. | `thresholds.*`, `maxLabelsPerIssue` | +| `Get-ScoreConfidence` | Rates data reliability (0–100) based on data completeness and score magnitude. | None (runtime only) | + +#### Helper Functions + +| Function | Purpose | +|----------|---------| +| `Get-TotalReactions` | Sums `users.totalCount` across all `reactionGroups` on an issue. | +| `Get-IssueAgeInDays` | Calculates `(UTC now − createdAt)` in whole days. | +| `Test-HasLabel` | Exact-match check for a single label name on an issue. | +| `Test-HasLabelMatching` | Regex-match check for a label pattern (e.g., `block\|blocker`). | +| `Format-IssueLink` | Returns `[#NNN](url)` markdown link. | +| `Format-Confidence` | Returns `[confidence:XX]` string. | +| `Get-AreaSuggestionConfidence` | Rates how likely a suggested area label is correct (used outside scoring). | + +#### `Get-IssueScore` Return Shape + +```powershell +@{ + Reactions = [int] # Points awarded (0 – weights.reactions) + Age = [int] # Points awarded (0 – weights.age) + Comments = [int] # Points awarded (0 – weights.comments) + Severity = [int] # Points awarded (0 – weights.severity) + Blockers = [int] # Points awarded (0 – weights.blockers) + Total = [int] # Sum of above + RawReactions = [int] # Actual reaction count + RawAge = [int] # Days since creation + RawComments = [int] # Actual comment count + RawUpdateAgeDays = [int] # Days since last update + SeverityLabel = [string] # First matching severity label (e.g., "bug") + IsBlocker = [bool] # True if any label matches block|blocker|blocking } ``` --- -## Usage in Reports +### `Get-HighlightScore.ps1` — Single-Issue Scorer + +Location: `./scripts/Get-HighlightScore.ps1` + +CLI tool that fetches one issue from GitHub and displays its score breakdown. + +#### Functions + +| Function | Purpose | +|----------|---------| +| `Get-DetailedIssueScore` | Wraps `Get-IssueScore` (ReportLib) and adds presentation metadata — reason strings (e.g., `"🌟 Popular (community interest)"`) and `MaxScore` per factor. Does **not** duplicate scoring math. | +| `Format-ScoreBreakdown` | Renders the visual bar-chart output (`█░`) with factor-by-factor breakdown, total score, confidence, and a priority recommendation (HIGH / MEDIUM / NORMAL / LOW). | -When generating reports: +#### Usage -1. Fetch all open issues per area -2. Calculate scores for each issue -3. Sort by score descending -4. Take top N issues (default: 3) -5. Assign highlight labels -6. Format for display +```powershell +# Quick score +./scripts/Get-HighlightScore.ps1 -IssueNumber 4651 -### Output Example +# Detailed breakdown with visual bars +./scripts/Get-HighlightScore.ps1 -IssueNumber 4651 -Verbose -```markdown -| Feature Area | ... | Highlights | -|--------------|-----|------------| -| area-Notification | ... | 🔥 [#2894](link) Hot, ⏰ [#3001](link) Aging | +# Custom config path +./scripts/Get-HighlightScore.ps1 -IssueNumber 4651 -ConfigPath .\custom-config.json +``` + +#### Output (Verbose) + +``` +═══════════════════════════════════════════════════════════ + Issue #4651: Example issue title +═══════════════════════════════════════════════════════════ + + SCORE BREAKDOWN: + ─────────────────────────────────────────────────────── + Reactions [████░░░░░░] 12/30 (raw: 7) + └─ 🌟 Popular (community interest) + Age [██████████] 30/30 (raw: 200) + └─ ⏰ Aging (needs triage for 200 days) + Comments [████████░░] 20/30 (raw: 8) + Severity [████████░░] 8/10 (raw: bug) + └─ 🟠 High: bug + ─────────────────────────────────────────────────────── + TOTAL SCORE: 70 / 100 [confidence:85] + + Recommendation: 🔴 HIGH PRIORITY - Should be highlighted in triage report ``` --- -## API Data Requirements +### `Generate-FeatureAreaReport.ps1` — Report Pipeline -To calculate scores, fetch these fields per issue: +Location: `./scripts/Generate-FeatureAreaReport.ps1` -```json -{ - "number": 2894, - "title": "...", - "createdAt": "2024-01-15T...", - "labels": [{"name": "area-Notification"}, {"name": "bug"}], - "reactionGroups": [ - {"content": "THUMBS_UP", "users": {"totalCount": 12}}, - {"content": "HEART", "users": {"totalCount": 5}} - ], - "comments": {"totalCount": 8}, - "author": {"login": "contributor"}, - "linkedIssues": [...] -} +Generates the full Feature Area Status report by iterating every `area-*` label. + +#### Data Flow + +``` +1. Get-AllAreaLabels Fetch all area-* labels from the repo + │ +2. Get-IssuesForArea For each area, fetch open (and optionally closed) issues + │ +3. Get-IssueStats Count totals, needs-triage, proposals, bugs per area + │ +4. Get-HighlightedIssues Score every issue via Get-IssueScore (ReportLib), + │ assign labels via Get-HighlightLabels (ReportLib), + │ sort descending, take top N + │ +5. Format-HighlightsMarkdown Format highlights as "🌟 [#NNN](url) [confidence:XX]" + │ +6. Format-ReportMarkdown Assemble the final markdown table with all areas ``` -GitHub CLI command: -```bash -gh issue list --repo microsoft/WindowsAppSDK --label "area-Notification" --state open --json number,title,createdAt,labels,reactionGroups,comments,author +#### Functions + +| Function | Purpose | Calls from ReportLib | +|----------|---------|---------------------| +| `Get-AllAreaLabels` | Discovers `area-*` labels via `Get-RepositoryLabels.ps1` (from the triage-meeting-prep skill). | — | +| `Get-IssuesForArea` | Fetches issues for one area label using `gh issue list`. | — | +| `Get-IssueStats` | Counts open, needs-triage, proposals, and bugs from label names. | — | +| `Get-HighlightedIssues` | Orchestrator: scores all issues, assigns labels, computes confidence, sorts, and returns top N. | `Get-IssueScore`, `Get-HighlightLabels`, `Get-ScoreConfidence` | +| `Format-HighlightsMarkdown` | Joins highlight entries into a comma-separated markdown string. | — | +| `Format-ReportMarkdown` | Builds the full markdown table with header, rows, and special-case handling (zero bugs, external area). | — | + +#### Usage + +```powershell +# Default markdown report +./scripts/Generate-FeatureAreaReport.ps1 + +# JSON output with 5 highlights per area +./scripts/Generate-FeatureAreaReport.ps1 -OutputFormat json -HighlightCount 5 + +# Include recently closed issues +./scripts/Generate-FeatureAreaReport.ps1 -IncludeClosed ``` + +--- + +### `Validate-FeatureAreaReport.ps1` — Live Counts + +Location: `./scripts/Validate-FeatureAreaReport.ps1` + +Queries live issue counts per area and prints a formatted console table. **Does not use `ScoringConfig.json` or any scoring functions** — it only counts open, needs-triage, proposal, and closed (30-day) issues. + +Use this script to sanity-check raw numbers before interpreting scored highlights. + +--- + +### `Get-AreaContacts.ps1` — Contact Lookup + +Location: `./scripts/Get-AreaContacts.ps1` + +Manages the area → contact mapping. **Does not use `ScoringConfig.json`** — it reads from `area-contacts.json` (user copy at `.user/issue-triage-report/area-contacts.json`, template at `references/area-contacts.json`). + +--- + +## Config → Function Cross-Reference + +Quick lookup: which function reads which `ScoringConfig.json` field. + +| Config Field | Read By | Effect | +|---|---|---| +| `weights.reactions` | `Get-IssueScore` | Max points for reaction count | +| `weights.age` | `Get-IssueScore` | Max points for issue age | +| `weights.comments` | `Get-IssueScore` | Max points for comment count | +| `weights.severity` | `Get-IssueScore` | Max points for severity label match | +| `weights.blockers` | `Get-IssueScore` | Max points for blocker labels (0 = disabled) | +| `thresholds.aging_days` | `Get-HighlightLabels` | Minimum days to qualify for ⏰ Aging badge | +| `thresholds.trending_comments` | `Get-HighlightLabels` | Minimum comments to qualify for 📈 Trending badge | +| `thresholds.trending_days` | `Get-HighlightLabels` | Maximum days since update to qualify for 📈 Trending badge | +| `thresholds.popular_reactions` | `Get-HighlightLabels` | Minimum reactions to qualify for 🌟 Popular badge | +| `labelPriority` | `Get-HighlightLabels` | Order of label assignment when issue qualifies for multiple | +| `maxLabelsPerIssue` | `Get-HighlightLabels` | Cap on badges per issue | +| `severityLabels.critical` | `Get-IssueScore` | Labels that earn 100% of `weights.severity` | +| `severityLabels.high` | `Get-IssueScore` | Labels that earn 80% of `weights.severity` | +| `severityLabels.medium` | `Get-IssueScore` | Labels that earn 50% of `weights.severity` | +| `severityLabels.low` | `Get-IssueScore` | Labels that earn 20% of `weights.severity` | + +--- + +## Special Report Indicators + +These are not configured in `ScoringConfig.json` but appear in report output: + +| Indicator | Meaning | +|-----------|---------| +| `0️⃣🐛🥳` | Feature area has zero open bugs | +| `🆕` | New area with no historical data | +| `N/A` | `area-External` — issues pending redirection to other teams | diff --git a/.github/skills/issue-triage-report/scripts/Generate-FeatureAreaReport.ps1 b/.github/skills/issue-triage-report/scripts/Generate-FeatureAreaReport.ps1 index 052587c43b..e0030b9f73 100644 --- a/.github/skills/issue-triage-report/scripts/Generate-FeatureAreaReport.ps1 +++ b/.github/skills/issue-triage-report/scripts/Generate-FeatureAreaReport.ps1 @@ -85,13 +85,22 @@ $AreaContacts = Get-AreaContacts -ContactsPath $ContactsPath function Get-AllAreaLabels { <# .SYNOPSIS - Fetches all labels starting with "area-" from the repository. + Fetches all labels starting with "area-" from the repository + using Get-RepositoryLabels.ps1 as the single source of truth. #> param([string]$Repository) - Write-Verbose "Fetching area labels from $Repository..." - $labels = gh label list --repo $Repository --search "area-" --json name --limit 100 | ConvertFrom-Json - return $labels | ForEach-Object { $_.name } | Sort-Object + Write-Verbose "Fetching area labels from $Repository via Get-RepositoryLabels.ps1..." + $SkillsRoot = Split-Path $SkillDir -Parent + $LabelsScript = Join-Path $SkillsRoot "triage-meeting-prep\scripts\Get-RepositoryLabels.ps1" + + if (-not (Test-Path $LabelsScript)) { + Write-Error "Get-RepositoryLabels.ps1 not found at: $LabelsScript" + exit 1 + } + + $labels = & $LabelsScript -Repository $Repository -Filter "area-*" -OutputFormat json | ConvertFrom-Json + return @($labels | ForEach-Object { $_.name } | Sort-Object) } function Get-IssuesForArea { @@ -170,11 +179,13 @@ function Get-HighlightedIssues { foreach ($issue in $Issues) { $score = Get-IssueScore -Issue $issue -Config $Config $labels = Get-HighlightLabels -Issue $issue -Score $score -Config $Config + $confidence = Get-ScoreConfidence -Issue $issue -Score $score $scoredIssues += @{ Number = $issue.number Title = $issue.title Score = $score.Total + Confidence = $confidence Labels = $labels ScoreBreakdown = $score } @@ -186,13 +197,13 @@ function Get-HighlightedIssues { return $highlights } -# Note: Get-IssueScore and Get-HighlightLabels are now defined in ReportLib.ps1 +# Note: Get-IssueScore, Get-HighlightLabels, and Get-ScoreConfidence are now defined in ReportLib.ps1 # to provide a single source of truth for scoring logic across the skill. function Format-HighlightsMarkdown { <# .SYNOPSIS - Formats highlighted issues as markdown links with labels. + Formats highlighted issues as markdown links with labels and confidence. #> param( [array]$Highlights, @@ -208,10 +219,11 @@ function Format-HighlightsMarkdown { $labelArray = @($h.Labels) $label = if ($labelArray.Count -gt 0) { $labelArray[0] } else { "" } $link = "[#$($h.Number)](https://github.com/$Repository/issues/$($h.Number))" + $confStr = "[confidence:$($h.Confidence)]" if ($label) { - $parts += "$label $link" + $parts += "$label $link $confStr" } else { - $parts += $link + $parts += "$link $confStr" } } @@ -317,9 +329,9 @@ try { $closedIssues = Get-IssuesForArea -Repository $Repo -AreaLabel $areaLabel -GetClosed # Filter to last 30 days $thirtyDaysAgo = (Get-Date).AddDays(-30) - $recentlyClosed = $closedIssues | Where-Object { + $recentlyClosed = @($closedIssues | Where-Object { [datetime]$_.updatedAt -gt $thirtyDaysAgo - } + }) $closedCount = $recentlyClosed.Count } @@ -330,10 +342,10 @@ try { $highlights = Get-HighlightedIssues -Issues $openIssues -Config $Config -MaxHighlights $HighlightCount $highlightsFormatted = Format-HighlightsMarkdown -Highlights $highlights -Repository $Repo - # Get contact + # Get contact - use new schema (single contact field) $contact = if ($AreaContacts[$areaLabel]) { $c = $AreaContacts[$areaLabel] - if ($c.secondary) { "$($c.primary), $($c.secondary)" } else { $c.primary } + if ($c.contact) { $c.contact } else { "TBD" } } else { "TBD" } diff --git a/.github/skills/issue-triage-report/scripts/Get-HighlightScore.ps1 b/.github/skills/issue-triage-report/scripts/Get-HighlightScore.ps1 index 91edc3e541..632f6fe86c 100644 --- a/.github/skills/issue-triage-report/scripts/Get-HighlightScore.ps1 +++ b/.github/skills/issue-triage-report/scripts/Get-HighlightScore.ps1 @@ -55,121 +55,70 @@ $Config = Get-ScoringConfig -ConfigPath $ConfigPath function Get-DetailedIssueScore { <# .SYNOPSIS - Calculates detailed score breakdown for an issue. + Wraps Get-IssueScore with a detailed breakdown for diagnostic display. + + .DESCRIPTION + Delegates all scoring math to Get-IssueScore (ReportLib.ps1) and adds + presentation metadata (Reason strings, MaxScore) for the formatted output. #> param( [object]$Issue, [hashtable]$Config ) + $score = Get-IssueScore -Issue $issue -Config $Config $weights = $Config.weights $thresholds = $Config.thresholds + # Build breakdown with presentation metadata on top of canonical scores $breakdown = @{ - Reactions = @{ Raw = 0; Score = 0; MaxScore = $weights.reactions; Reason = "" } - Age = @{ Raw = 0; Score = 0; MaxScore = $weights.age; Reason = "" } - Comments = @{ Raw = 0; Score = 0; MaxScore = $weights.comments; Reason = "" } - Severity = @{ Raw = ""; Score = 0; MaxScore = $weights.severity; Reason = "" } - Blockers = @{ Raw = $false; Score = 0; MaxScore = $weights.blockers; Reason = "" } - } - - # 1. Reactions - $totalReactions = Get-TotalReactions -ReactionGroups $Issue.reactionGroups - $breakdown.Reactions.Raw = $totalReactions - - $breakdown.Reactions.Score = switch ($totalReactions) { - { $_ -ge 50 } { $weights.reactions; break } - { $_ -ge 20 } { [math]::Floor($weights.reactions * 0.8); break } - { $_ -ge 10 } { [math]::Floor($weights.reactions * 0.6); break } - { $_ -ge 5 } { [math]::Floor($weights.reactions * 0.4); break } - { $_ -ge 1 } { [math]::Floor($weights.reactions * 0.2); break } - default { 0 } - } - - if ($totalReactions -ge $thresholds.hot_reactions) { - $breakdown.Reactions.Reason = "🔥 Hot (high community interest)" - } - elseif ($totalReactions -ge 5) { - $breakdown.Reactions.Reason = "Notable community interest" + Reactions = @{ Raw = $score.RawReactions; Score = $score.Reactions; MaxScore = $weights.reactions; Reason = "" } + Age = @{ Raw = $score.RawAge; Score = $score.Age; MaxScore = $weights.age; Reason = "" } + Comments = @{ Raw = $score.RawComments; Score = $score.Comments; MaxScore = $weights.comments; Reason = "" } + Severity = @{ Raw = $score.SeverityLabel; Score = $score.Severity; MaxScore = $weights.severity; Reason = "" } + Blockers = @{ Raw = $score.IsBlocker; Score = $score.Blockers; MaxScore = $weights.blockers; Reason = "" } } - # 2. Age - $ageInDays = Get-IssueAgeInDays -CreatedAt $Issue.createdAt - $breakdown.Age.Raw = $ageInDays - - $breakdown.Age.Score = switch ($ageInDays) { - { $_ -ge 181 } { $weights.age; break } - { $_ -ge 91 } { [math]::Floor($weights.age * 0.75); break } - { $_ -ge 61 } { [math]::Floor($weights.age * 0.5); break } - { $_ -ge 31 } { [math]::Floor($weights.age * 0.25); break } - default { 0 } + # Reason strings (presentation only — scoring math lives in Get-IssueScore) + if ($score.RawReactions -ge $thresholds.popular_reactions) { + $breakdown.Reactions.Reason = "🌟 Popular (community interest)" } $hasNeedsTriage = Test-HasLabel -Labels $Issue.labels -LabelName "needs-triage" - if ($ageInDays -gt $thresholds.aging_days -and $hasNeedsTriage) { - $breakdown.Age.Reason = "⏰ Aging (needs triage for $ageInDays days)" - } - elseif ($ageInDays -gt $thresholds.aging_days) { - $breakdown.Age.Reason = "Open for $ageInDays days" + if ($score.RawAge -gt $thresholds.aging_days -and $hasNeedsTriage) { + $breakdown.Age.Reason = "⏰ Aging (needs triage for $($score.RawAge) days)" } - - # 3. Comments - $commentCount = if ($Issue.comments) { $Issue.comments.Count } else { 0 } - $breakdown.Comments.Raw = $commentCount - - $breakdown.Comments.Score = switch ($commentCount) { - { $_ -ge 11 } { $weights.comments; break } - { $_ -ge 6 } { [math]::Floor($weights.comments * 0.67); break } - { $_ -ge 3 } { [math]::Floor($weights.comments * 0.4); break } - { $_ -ge 1 } { [math]::Floor($weights.comments * 0.2); break } - default { 0 } + elseif ($score.RawAge -gt $thresholds.aging_days) { + $breakdown.Age.Reason = "Open for $($score.RawAge) days" } - if ($commentCount -ge $thresholds.trending_comments) { - $breakdown.Comments.Reason = "📈 Trending ($commentCount comments recently)" + if ($score.RawComments -ge $thresholds.trending_comments -and $score.RawUpdateAgeDays -le $thresholds.trending_days) { + $breakdown.Comments.Reason = "📈 Trending ($($score.RawComments) comments, updated within $($thresholds.trending_days) days)" } - # 4. Severity - $severityLabel = "" - if (Test-HasLabel -Labels $Issue.labels -LabelName "regression") { - $severityLabel = "regression" - $breakdown.Severity.Score = $weights.severity - $breakdown.Severity.Reason = "🐛 Regression" - } - elseif (Test-HasLabelMatching -Labels $Issue.labels -Pattern "crash|hang|data-loss") { - $severityLabel = "crash/hang" - $breakdown.Severity.Score = [math]::Floor($weights.severity * 0.8) - $breakdown.Severity.Reason = "Critical: crash/hang/data-loss" - } - elseif (Test-HasLabel -Labels $Issue.labels -LabelName "bug") { - $severityLabel = "bug" - $breakdown.Severity.Score = [math]::Floor($weights.severity * 0.53) - $breakdown.Severity.Reason = "Bug report" - } - elseif (Test-HasLabel -Labels $Issue.labels -LabelName "performance") { - $severityLabel = "performance" - $breakdown.Severity.Score = [math]::Floor($weights.severity * 0.4) - $breakdown.Severity.Reason = "Performance issue" + if ($score.SeverityLabel) { + $severityLabels = $Config.severityLabels + if ($severityLabels.critical -contains $score.SeverityLabel) { + $breakdown.Severity.Reason = "🔴 Critical: $($score.SeverityLabel)" + } + elseif ($severityLabels.high -contains $score.SeverityLabel) { + $breakdown.Severity.Reason = "🟠 High: $($score.SeverityLabel)" + } + elseif ($severityLabels.medium -contains $score.SeverityLabel) { + $breakdown.Severity.Reason = "🟡 Medium: $($score.SeverityLabel)" + } + elseif ($severityLabels.low -contains $score.SeverityLabel) { + $breakdown.Severity.Reason = "🟢 Low: $($score.SeverityLabel)" + } } - $breakdown.Severity.Raw = $severityLabel - - # 5. Blockers - $isBlocker = Test-HasLabelMatching -Labels $Issue.labels -Pattern "block|blocker|blocking" - $breakdown.Blockers.Raw = $isBlocker - if ($isBlocker) { - $breakdown.Blockers.Score = $weights.blockers + if ($score.IsBlocker -and $weights.blockers -gt 0) { $breakdown.Blockers.Reason = "🚧 Blocker issue" } - # Calculate total - $totalScore = $breakdown.Reactions.Score + $breakdown.Age.Score + - $breakdown.Comments.Score + - $breakdown.Severity.Score + $breakdown.Blockers.Score - return @{ Breakdown = $breakdown - TotalScore = $totalScore + TotalScore = $score.Total MaxPossible = 100 } } @@ -181,7 +130,8 @@ function Format-ScoreBreakdown { #> param( [hashtable]$ScoreResult, - [object]$Issue + [object]$Issue, + [int]$Confidence = 0 ) $sb = [System.Text.StringBuilder]::new() @@ -211,6 +161,9 @@ function Format-ScoreBreakdown { $raw = $data.Raw $reason = $data.Reason + # Skip factors with 0 max score (disabled) + if ($max -eq 0) { continue } + $bar = "█" * [math]::Floor($score / $max * 10) $bar = $bar.PadRight(10, "░") @@ -223,7 +176,7 @@ function Format-ScoreBreakdown { } [void]$sb.AppendLine(" ───────────────────────────────────────────────────────") - [void]$sb.AppendLine(" TOTAL SCORE: $($ScoreResult.TotalScore) / $($ScoreResult.MaxPossible)") + [void]$sb.AppendLine(" TOTAL SCORE: $($ScoreResult.TotalScore) / $($ScoreResult.MaxPossible) [confidence:$Confidence]") [void]$sb.AppendLine("") # Highlight recommendation @@ -280,15 +233,24 @@ if ($issue.state -ne "OPEN") { # Calculate score $scoreResult = Get-DetailedIssueScore -Issue $issue -Config $Config +# Calculate confidence using shared function +$confidence = Get-ScoreConfidence -Issue $issue -Score @{ + Total = $scoreResult.TotalScore + Reactions = $scoreResult.Breakdown.Reactions.Score + Age = $scoreResult.Breakdown.Age.Score + Comments = $scoreResult.Breakdown.Comments.Score + Severity = $scoreResult.Breakdown.Severity.Score +} + # Output if ($VerbosePreference -eq "Continue" -or $PSBoundParameters.ContainsKey('Verbose')) { - $output = Format-ScoreBreakdown -ScoreResult $scoreResult -Issue $issue + $output = Format-ScoreBreakdown -ScoreResult $scoreResult -Issue $issue -Confidence $confidence Write-Output $output } else { # Simple output Write-Host "" - Write-Host "Issue #$($IssueNumber) Score: $($scoreResult.TotalScore)/100" -ForegroundColor Green + Write-Host "Issue #$($IssueNumber) Score: $($scoreResult.TotalScore)/100 [confidence:$confidence]" -ForegroundColor Green Write-Host "" # Show top contributing factors diff --git a/.github/skills/issue-triage-report/scripts/ReportLib.ps1 b/.github/skills/issue-triage-report/scripts/ReportLib.ps1 index 8cae2fe914..4d21331527 100644 --- a/.github/skills/issue-triage-report/scripts/ReportLib.ps1 +++ b/.github/skills/issue-triage-report/scripts/ReportLib.ps1 @@ -12,115 +12,137 @@ function Get-ScoringConfig { <# .SYNOPSIS - Loads the scoring configuration from JSON file or returns defaults. + Loads the scoring configuration from ScoringConfig.json (failfast, no defaults). + + .DESCRIPTION + All required fields must be present in the JSON file. Throws if any + required section or field is missing. #> param( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] [string]$ConfigPath ) - $defaultConfig = @{ + # Validate config file exists + if (-not (Test-Path -LiteralPath $ConfigPath -PathType Leaf)) { + throw "Scoring config file not found: $ConfigPath" + } + + # Validate JSON structure and correct filename + if ([System.IO.Path]::GetFileName($ConfigPath) -cne "ScoringConfig.json") { + throw "ConfigPath must point to ScoringConfig.json. Got: $ConfigPath" + } + + $loaded = Get-Content $ConfigPath -Raw | ConvertFrom-Json + + # --- Validate required top-level sections --- + foreach ($section in @("weights", "thresholds", "maxLabelsPerIssue", "severityLabels")) { + if ($null -eq $loaded.$section) { + throw "ScoringConfig.json missing required section: '$section'" + } + } + + # --- Validate required weight fields --- + foreach ($field in @("reactions", "age", "comments", "severity", "blockers")) { + if ($null -eq $loaded.weights.$field) { + throw "ScoringConfig.json 'weights' missing required field: '$field'" + } + } + + # --- Validate required threshold fields --- + foreach ($field in @("aging_days", "trending_comments", "trending_days", "popular_reactions")) { + if ($null -eq $loaded.thresholds.$field) { + throw "ScoringConfig.json 'thresholds' missing required field: '$field'" + } + } + + # --- Validate required severity label levels --- + foreach ($level in @("critical", "high", "medium", "low")) { + if ($null -eq $loaded.severityLabels.$level) { + throw "ScoringConfig.json 'severityLabels' missing required level: '$level'" + } + } + + # --- Build config hashtable entirely from loaded values (no defaults) --- + $config = @{ weights = @{ - reactions = 30 - age = 25 - comments = 20 - severity = 15 - blockers = 10 + reactions = [int]$loaded.weights.reactions + age = [int]$loaded.weights.age + comments = [int]$loaded.weights.comments + severity = [int]$loaded.weights.severity + blockers = [int]$loaded.weights.blockers } thresholds = @{ - hot_reactions = 10 - aging_days = 90 - trending_comments = 5 - trending_days = 14 - popular_reactions = 5 - } - labelPriority = @( - "regression" - "blocker" - "hot" - "aging" - "trending" - "popular" - ) - maxLabelsPerIssue = 2 - } - - if ($ConfigPath -and (Test-Path $ConfigPath)) { - try { - $loaded = Get-Content $ConfigPath -Raw | ConvertFrom-Json - # Merge weights - if ($loaded.weights) { - $defaultConfig.weights.reactions = [int]$loaded.weights.reactions - $defaultConfig.weights.age = [int]$loaded.weights.age - $defaultConfig.weights.comments = [int]$loaded.weights.comments - $defaultConfig.weights.severity = [int]$loaded.weights.severity - $defaultConfig.weights.blockers = [int]$loaded.weights.blockers - } - # Merge thresholds - if ($loaded.thresholds) { - $defaultConfig.thresholds.hot_reactions = [int]$loaded.thresholds.hot_reactions - $defaultConfig.thresholds.aging_days = [int]$loaded.thresholds.aging_days - $defaultConfig.thresholds.trending_comments = [int]$loaded.thresholds.trending_comments - $defaultConfig.thresholds.trending_days = [int]$loaded.thresholds.trending_days - $defaultConfig.thresholds.popular_reactions = [int]$loaded.thresholds.popular_reactions - } - if ($loaded.maxLabelsPerIssue) { - $defaultConfig.maxLabelsPerIssue = [int]$loaded.maxLabelsPerIssue - } + aging_days = [int]$loaded.thresholds.aging_days + trending_comments = [int]$loaded.thresholds.trending_comments + trending_days = [int]$loaded.thresholds.trending_days + popular_reactions = [int]$loaded.thresholds.popular_reactions } - catch { - Write-Warning "Failed to load config from $ConfigPath, using defaults: $_" + maxLabelsPerIssue = [int]$loaded.maxLabelsPerIssue + severityLabels = @{ + critical = @($loaded.severityLabels.critical) + high = @($loaded.severityLabels.high) + medium = @($loaded.severityLabels.medium) + low = @($loaded.severityLabels.low) } } - return $defaultConfig + if ($loaded.labelPriority) { + $config.labelPriority = @($loaded.labelPriority) + } + + return $config } + function Get-AreaContacts { <# .SYNOPSIS - Loads the area-to-contact mapping from JSON file. + Loads the area-to-contact mapping from JSON file (failfast, no defaults). .DESCRIPTION - Loads contacts from the specified path. If no contacts file is found, - returns an empty hashtable and writes a warning. + Loads contacts from the specified path. Throws if the file is missing, + malformed, or if any area entry lacks a required 'contact' field. Users should create their own area-contacts.json file at: /.user/issue-triage-report/area-contacts.json See the template at: .github/skills/issue-triage-report/references/area-contacts.json + + Schema uses a single "contact" field per area. #> param( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] [string]$ContactsPath ) - if ($ContactsPath -and (Test-Path $ContactsPath)) { - try { - $loaded = Get-Content $ContactsPath -Raw | ConvertFrom-Json - if ($loaded.areaContacts) { - # Convert PSObject to hashtable for PS 5.1 compatibility - # (ConvertFrom-Json -AsHashtable is only available in PS 6.0+) - $hashtable = @{} - foreach ($prop in $loaded.areaContacts.PSObject.Properties) { - $hashtable[$prop.Name] = @{ - primary = $prop.Value.primary - secondary = $prop.Value.secondary - } - } - return $hashtable - } - } - catch { - Write-Warning "Failed to load contacts from $ContactsPath`: $_" - } + # Validate contacts file exists + if (-not (Test-Path -LiteralPath $ContactsPath -PathType Leaf)) { + throw "Area contacts file not found: $ContactsPath. Please create your contacts file at: /.user/issue-triage-report/area-contacts.json. See template at: .github/skills/issue-triage-report/references/area-contacts.json" } - else { - Write-Warning "Area contacts file not found at: $ContactsPath" - Write-Warning "Please create your contacts file at: /.user/issue-triage-report/area-contacts.json" - Write-Warning "See template at: .github/skills/issue-triage-report/references/area-contacts.json" + + $loaded = Get-Content $ContactsPath -Raw | ConvertFrom-Json + + if (-not $loaded.areaContacts) { + throw "Area contacts file missing required 'areaContacts' section: $ContactsPath" } - return @{} + # Convert PSObject to hashtable for PS 5.1 compatibility + # (ConvertFrom-Json -AsHashtable is only available in PS 6.0+) + $hashtable = @{} + foreach ($prop in $loaded.areaContacts.PSObject.Properties) { + if (-not $prop.Value.contact) { + throw "Area '$($prop.Name)' missing required 'contact' field in: $ContactsPath" + } + $hashtable[$prop.Name] = @{ + contact = $prop.Value.contact + notes = $prop.Value.notes + } + } + return $hashtable } function Get-TotalReactions { @@ -263,7 +285,6 @@ function Get-IssueScore { ) $weights = $Config.weights - $thresholds = $Config.thresholds $score = @{ Reactions = 0 @@ -276,6 +297,7 @@ function Get-IssueScore { RawReactions = 0 RawAge = 0 RawComments = 0 + RawUpdateAgeDays = 0 SeverityLabel = "" IsBlocker = $false } @@ -297,6 +319,18 @@ function Get-IssueScore { $ageInDays = Get-IssueAgeInDays -CreatedAt $Issue.createdAt $score.RawAge = $ageInDays + # Days since last update (for trending recency check) + if ($Issue.updatedAt) { + try { + $updated = [datetime]$Issue.updatedAt + $score.RawUpdateAgeDays = ([datetime]::UtcNow - $updated).Days + } catch { + $score.RawUpdateAgeDays = [int]::MaxValue + } + } else { + $score.RawUpdateAgeDays = [int]::MaxValue + } + $score.Age = switch ($ageInDays) { { $_ -ge 181 } { $weights.age; break } { $_ -ge 91 } { [math]::Floor($weights.age * 0.75); break } @@ -324,36 +358,70 @@ function Get-IssueScore { default { 0 } } - # 4. Severity score + # 4. Severity score - use configurable severity labels $labelNames = @() if ($Issue.labels) { $labelNames = @($Issue.labels | ForEach-Object { $_.name }) } - if ($labelNames -contains "regression") { - $score.Severity = $weights.severity - $score.SeverityLabel = "regression" - } - else { - $hasCrash = @($labelNames | Where-Object { $_ -match "crash|hang|data-loss" }).Count -gt 0 - if ($hasCrash) { - $score.Severity = [math]::Floor($weights.severity * 0.8) - $score.SeverityLabel = "crash/hang" + # Severity labels must be provided by config (no defaults) + if (-not $Config.severityLabels) { + throw "Config missing required 'severityLabels'. Ensure ScoringConfig.json is loaded via Get-ScoringConfig." + } + $severityLabels = $Config.severityLabels + + # Check for critical severity labels (100% of severity weight) + $hasCritical = $false + foreach ($critLabel in $severityLabels.critical) { + if ($labelNames -contains $critLabel) { + $score.Severity = $weights.severity + $score.SeverityLabel = $critLabel + $hasCritical = $true + break } - elseif ($labelNames -contains "bug") { - $score.Severity = [math]::Floor($weights.severity * 0.53) - $score.SeverityLabel = "bug" + } + + if (-not $hasCritical) { + # Check for high severity labels (80% of severity weight) + $hasHigh = $false + foreach ($highLabel in $severityLabels.high) { + if ($labelNames -contains $highLabel) { + $score.Severity = [math]::Floor($weights.severity * 0.8) + $score.SeverityLabel = $highLabel + $hasHigh = $true + break + } } - elseif ($labelNames -contains "performance") { - $score.Severity = [math]::Floor($weights.severity * 0.4) - $score.SeverityLabel = "performance" + + if (-not $hasHigh) { + # Check for medium severity labels (50% of severity weight) + $hasMedium = $false + foreach ($medLabel in $severityLabels.medium) { + if ($labelNames -contains $medLabel) { + $score.Severity = [math]::Floor($weights.severity * 0.5) + $score.SeverityLabel = $medLabel + $hasMedium = $true + break + } + } + + if (-not $hasMedium) { + # Check for low severity labels (20% of severity weight) + foreach ($lowLabel in $severityLabels.low) { + if ($labelNames -contains $lowLabel) { + $score.Severity = [math]::Floor($weights.severity * 0.2) + $score.SeverityLabel = $lowLabel + break + } + } + } } } - # 5. Blocker score - $hasBlocker = @($labelNames | Where-Object { $_ -match "block|blocker|blocking" }).Count -gt 0 + # 5. Blocker score (only if weight > 0) + $hasBlocker = @($labelNames | Where-Object { $_ -match "block" }).Count -gt 0 $score.IsBlocker = $hasBlocker - if ($hasBlocker) { + if ($hasBlocker -and $weights.blockers -gt 0) { $score.Blockers = $weights.blockers } @@ -382,7 +450,7 @@ function Get-HighlightLabels { The scoring configuration hashtable. .OUTPUTS - [array] Array of highlight label strings (e.g., "🔥 Hot", "⏰ Aging"). + [array] Array of highlight label strings (e.g., "🌟 Popular", "⏰ Aging"). #> param( [object]$Issue, @@ -406,21 +474,170 @@ function Get-HighlightLabels { if ($Score.IsBlocker) { $labels += "🚧 Blocker" } - if ($Score.RawReactions -ge $thresholds.hot_reactions) { - $labels += "🔥 Hot" + # Consolidated Popular label (replaces both Hot and old Popular) + if ($Score.RawReactions -ge $thresholds.popular_reactions) { + $labels += "🌟 Popular" } if ($Score.RawAge -gt $thresholds.aging_days -and $labelNames -contains "needs-triage") { $labels += "⏰ Aging" } - if ($Score.RawComments -ge $thresholds.trending_comments) { + if ($Score.RawComments -ge $thresholds.trending_comments -and $Score.RawUpdateAgeDays -le $thresholds.trending_days) { $labels += "📈 Trending" } - $hasFeatureProposal = $labelNames -contains "feature proposal" -or $labelNames -contains "feature-proposal" - if ($hasFeatureProposal -and $Score.RawReactions -ge $thresholds.popular_reactions) { - $labels += "📢 Popular" - } - # Return only top N labels return $labels | Select-Object -First $maxLabels } + +function Get-ScoreConfidence { + <# + .SYNOPSIS + Calculates confidence level for issue scoring. + + .DESCRIPTION + Returns a numeric confidence value (0-100) based on data completeness + and scoring factor quality. Format: [confidence:XX] for grep-friendliness. + + .PARAMETER Issue + The GitHub issue object. + + .PARAMETER Score + The score hashtable from Get-IssueScore. + + .OUTPUTS + [int] Confidence value 0-100. + #> + param( + [object]$Issue, + [hashtable]$Score + ) + + $confidence = 0 + + # Data completeness factors (max 60 points) + # Has reactions data + if ($null -ne $Issue.reactionGroups) { + $confidence += 15 + } + # Has comments data + if ($null -ne $Issue.comments) { + $confidence += 15 + } + # Has labels + if ($Issue.labels -and $Issue.labels.Count -gt 0) { + $confidence += 15 + } + # Has created date (age certainty) + if ($Issue.createdAt) { + $confidence += 15 + } + + # Score quality factors (max 40 points) + # Higher scores are more confident (clear priority signals) + if ($Score.Total -ge 60) { + $confidence += 25 + } + elseif ($Score.Total -ge 40) { + $confidence += 15 + } + elseif ($Score.Total -ge 20) { + $confidence += 10 + } + + # Multiple scoring factors contributing (not just one dimension) + $factorsContributing = 0 + if ($Score.Reactions -gt 0) { $factorsContributing++ } + if ($Score.Age -gt 0) { $factorsContributing++ } + if ($Score.Comments -gt 0) { $factorsContributing++ } + if ($Score.Severity -gt 0) { $factorsContributing++ } + + if ($factorsContributing -ge 3) { + $confidence += 15 + } + elseif ($factorsContributing -ge 2) { + $confidence += 10 + } + + return [math]::Min($confidence, 100) +} + +function Format-Confidence { + <# + .SYNOPSIS + Formats confidence value as grep-friendly string. + + .PARAMETER Confidence + Numeric confidence value 0-100. + + .OUTPUTS + [string] Formatted string like "[confidence:85]". + #> + param( + [int]$Confidence + ) + + return "[confidence:$Confidence]" +} + +function Get-AreaSuggestionConfidence { + <# + .SYNOPSIS + Calculates confidence for area label suggestions. + + .DESCRIPTION + Returns confidence (0-100) for how likely a suggested area label is correct. + Based on keyword matching, code path identification, and similar issues. + + .PARAMETER Issue + The GitHub issue object. + + .PARAMETER SuggestedArea + The suggested area label string. + + .PARAMETER MatchFactors + Hashtable with match quality indicators: + - KeywordMatches: Number of relevant keywords found + - CodePathFound: Boolean if code path was identified + - SimilarIssueFound: Boolean if similar issue with same area exists + - MultipleAreaCandidates: Boolean if multiple areas are possible + + .OUTPUTS + [int] Confidence value 0-100. + #> + param( + [object]$Issue, + [string]$SuggestedArea, + [hashtable]$MatchFactors = @{} + ) + + $confidence = 25 # Base confidence + + # Keyword match strength (0-35 points) + $keywordMatches = if ($MatchFactors.KeywordMatches) { $MatchFactors.KeywordMatches } else { 0 } + if ($keywordMatches -ge 5) { + $confidence += 35 + } + elseif ($keywordMatches -ge 3) { + $confidence += 25 + } + elseif ($keywordMatches -ge 1) { + $confidence += 15 + } + + # Code path identified (0-25 points) + if ($MatchFactors.CodePathFound) { + $confidence += 25 + } + + # Similar issue with same area found (0-15 points) + if ($MatchFactors.SimilarIssueFound) { + $confidence += 15 + } + + # Single clear area vs multiple candidates (penalty) + if ($MatchFactors.MultipleAreaCandidates) { + $confidence -= 20 + } + + return [math]::Max(0, [math]::Min($confidence, 100)) +} diff --git a/.github/skills/issue-triage-report/scripts/ScoringConfig.json b/.github/skills/issue-triage-report/scripts/ScoringConfig.json index e4de7e9aac..05c026b2a5 100644 --- a/.github/skills/issue-triage-report/scripts/ScoringConfig.json +++ b/.github/skills/issue-triage-report/scripts/ScoringConfig.json @@ -1,13 +1,12 @@ { "weights": { "reactions": 30, - "age": 25, - "comments": 20, - "severity": 15, - "blockers": 10 + "age": 30, + "comments": 30, + "severity": 10, + "blockers": 0 }, "thresholds": { - "hot_reactions": 10, "aging_days": 90, "trending_comments": 10, "trending_days": 14, @@ -16,16 +15,15 @@ "labelPriority": [ "regression", "blocker", - "hot", + "popular", "aging", - "trending", - "popular" + "trending" ], "maxLabelsPerIssue": 2, "severityLabels": { - "critical": ["regression", "crash", "hang", "data-loss"], - "high": ["bug", "security"], - "medium": ["performance"], - "low": ["documentation", "enhancement"] + "critical": ["regression", "crash", "hang", "data-loss", "P0"], + "high": ["bug", "P1"], + "medium": ["performance", "feature proposal", "feature-proposal", "P2"], + "low": ["documentation", "enhancement", "P3"] } } diff --git a/.github/skills/issue-triage-report/scripts/Validate-FeatureAreaReport.ps1 b/.github/skills/issue-triage-report/scripts/Validate-FeatureAreaReport.ps1 new file mode 100644 index 0000000000..e682c3b2d7 --- /dev/null +++ b/.github/skills/issue-triage-report/scripts/Validate-FeatureAreaReport.ps1 @@ -0,0 +1,193 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Generates a live feature area report from GitHub data. + +.DESCRIPTION + Discovers all area labels in the microsoft/WindowsAppSDK repository via GitHub CLI, + queries live issue counts per area label, and reports open, needs-triage, + feature-proposal, and closed (30-day) counts for each area. + +.PARAMETER Repository + Repository in 'owner/repo' format. Defaults to 'microsoft/WindowsAppSDK'. + +.EXAMPLE + ./Validate-FeatureAreaReport.ps1 + +.NOTES + Requires GitHub CLI (gh) to be installed and authenticated. +#> + +[CmdletBinding()] +param( + [Parameter()] + [ValidatePattern('^[a-zA-Z0-9_-]+/[a-zA-Z0-9_.-]+$')] + [string]$Repository = "microsoft/WindowsAppSDK" +) + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = 'Stop' + +# ── Helpers ────────────────────────────────────────────────────────────────── + +function Assert-GhCli { + try { + $null = Get-Command gh -ErrorAction Stop + } + catch { + Write-Error "GitHub CLI (gh) is not installed. Install: winget install GitHub.cli" + exit 1 + } + $null = gh auth status 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Error "GitHub CLI is not authenticated. Run: gh auth login" + exit 1 + } +} + +function Get-AreaLabels { + <# + .SYNOPSIS + Discovers all area-* labels in the repository via Get-RepositoryLabels.ps1. + #> + param( + [string]$Repo + ) + + $ScriptDir = $PSScriptRoot + $SkillsRoot = Split-Path (Split-Path $ScriptDir -Parent) -Parent + $LabelsScript = Join-Path $SkillsRoot "triage-meeting-prep\scripts\Get-RepositoryLabels.ps1" + + if (-not (Test-Path $LabelsScript)) { + Write-Error "Get-RepositoryLabels.ps1 not found at: $LabelsScript" + exit 1 + } + + $labels = @(& $LabelsScript -Repository $Repo -Filter "area-*" -OutputFormat json | ConvertFrom-Json) + if ($null -eq $labels -or $labels.Count -eq 0) { + return @() + } + + return @($labels | Sort-Object -Property name | ForEach-Object { $_.name }) +} + +function Get-AreaIssueStats { + <# + .SYNOPSIS + Fetches and categorises issues for a given area label. + #> + param( + [string]$Repo, + [string]$AreaLabel + ) + + # Open issues + $openJson = gh issue list --repo $Repo --label $AreaLabel --state open --limit 500 ` + --json "number,labels" + $openIssues = @($openJson | ConvertFrom-Json) + # ConvertFrom-Json may return $null on empty set + if ($null -eq $openIssues -or ($openIssues.Count -eq 1 -and $null -eq $openIssues[0])) { + $openIssues = @() + } + + $needsTriage = @($openIssues | Where-Object { + @($_.labels | ForEach-Object { $_.name }) -contains 'needs-triage' + }) + + $proposals = @($openIssues | Where-Object { + $names = @($_.labels | ForEach-Object { $_.name }) + ($names -contains 'feature proposal') -or ($names -contains 'feature-proposal') + }) + + # Closed issues (recent 30 days based on updatedAt) + $closedJson = gh issue list --repo $Repo --label $AreaLabel --state closed --limit 500 ` + --json "number,updatedAt" + $closedIssues = @($closedJson | ConvertFrom-Json) + if ($null -eq $closedIssues -or ($closedIssues.Count -eq 1 -and $null -eq $closedIssues[0])) { + $closedIssues = @() + } + + $thirtyAgo = (Get-Date).AddDays(-30) + $recentClosed = @($closedIssues | Where-Object { + $null -ne $_ -and [datetime]$_.updatedAt -gt $thirtyAgo + }) + + return @{ + Open = $openIssues.Count + NeedsTriage = $needsTriage.Count + Proposals = $proposals.Count + Closed = $recentClosed.Count + } +} + +# ── Main ───────────────────────────────────────────────────────────────────── + +try { + Assert-GhCli + + # ── Discover area labels dynamically ───────────────────────────────────── + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " Feature Area Report (Live)" -ForegroundColor Cyan + Write-Host " Repository: $Repository" -ForegroundColor Gray + Write-Host " Generated: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')" -ForegroundColor Gray + Write-Host "════════════════════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host "" + + Write-Verbose "Discovering area labels..." + $areaLabels = Get-AreaLabels -Repo $Repository + if ($areaLabels.Count -eq 0) { + Write-Error "No area labels found in $Repository" + exit 1 + } + Write-Verbose "Found $($areaLabels.Count) area labels" + + $totalOpen = 0 + $totalTriage = 0 + $totalProposals = 0 + $totalClosed = 0 + $areasChecked = 0 + + # Header + Write-Host (" {0,-30} {1,6} {2,8} {3,8} {4,10}" -f ` + "Area", "Open", "Triage", "Proposals", "Closed(30d)") -ForegroundColor White + Write-Host (" " + ("-" * 70)) -ForegroundColor DarkGray + + foreach ($area in $areaLabels) { + Write-Verbose "Querying $area..." + $stats = Get-AreaIssueStats -Repo $Repository -AreaLabel $area + + $color = if ($stats.NeedsTriage -gt 5) { 'Yellow' } + elseif ($stats.Open -eq 0 -and $stats.Closed -eq 0) { 'DarkGray' } + else { 'White' } + + Write-Host (" {0,-30} {1,6} {2,8} {3,8} {4,10}" -f ` + $area, $stats.Open, $stats.NeedsTriage, $stats.Proposals, $stats.Closed) ` + -ForegroundColor $color + + $totalOpen += $stats.Open + $totalTriage += $stats.NeedsTriage + $totalProposals += $stats.Proposals + $totalClosed += $stats.Closed + $areasChecked++ + } + + # ── Summary ────────────────────────────────────────────────────────────── + Write-Host "" + Write-Host (" " + ("-" * 70)) -ForegroundColor DarkGray + Write-Host (" {0,-30} {1,6} {2,8} {3,8} {4,10}" -f ` + "TOTAL", $totalOpen, $totalTriage, $totalProposals, $totalClosed) -ForegroundColor Cyan + Write-Host "" + Write-Host " Areas queried: $areasChecked" -ForegroundColor White + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════" -ForegroundColor Cyan + + exit 0 +} +catch { + Write-Error "Validation failed: $_" + exit 1 +} diff --git a/.github/skills/triage-meeting-prep/SKILL.md b/.github/skills/triage-meeting-prep/SKILL.md index df4b333eba..31df617beb 100644 --- a/.github/skills/triage-meeting-prep/SKILL.md +++ b/.github/skills/triage-meeting-prep/SKILL.md @@ -36,10 +36,12 @@ The skill tracks **four critical categories** for each meeting: | Category | Description | Action Required | |----------|-------------|-----------------| -| 🔥 **Hot Issues** | ≥5 increased activity (comments + reactions) | May need priority attention | +| 🌟 **Popular Issues** | ≥5 reactions (community interest) | May need priority attention | | 🆕 **Created This Week** | Issues created since last weekly triage | Review + assign area | | ⏳ **Older Pending** | Older issues still without area labels | Follow up — why no action? | -| ✅ **Closed (need reply)** | Closed issues with customer follow-up | Draft reply with confidence level | +| ✅ **Closed (need reply)** | Closed issues with customer follow-up | Draft reply with `[confidence:XX]` | + +All suggestions include confidence scoring in `[confidence:XX]` format for easy filtering. ## When to Use This Skill @@ -87,9 +89,14 @@ gh auth login 3. **Categorize issues by creation date**: - 🆕 **Created This Week** — `createdAt` ≥ last weekly triage date - ⏳ **Older Pending** — `createdAt` < last weekly triage date -4. **For each no-area issue**, analyze using `Get-IssueDetails.ps1`: +4. **Fetch area label definitions** for classification: + ```powershell + ./.github/skills/triage-meeting-prep/scripts/Get-RepositoryLabels.ps1 -Filter "area-*" -OutputFormat json + ``` +5. **For each no-area issue**, analyze using `Get-IssueDetails.ps1`: - Run: `./Get-IssueDetails.ps1 -IssueNumber -OutputFormat summary` - Review the issue body, comments, and labels + - **Classify the area label** by reasoning about the issue content against the area label definitions (see workflow for confidence rubric) 5. **Generate summary report** with links to each issue's `overview.md` 6. **Save current state** for next diff comparison @@ -178,7 +185,7 @@ Individual issue reviews are saved to `Generated Files/issueReview/ **Key Output**: The `summary.md` includes the **Suggested Actions** section extracted from each issue's `overview.md`. +> **Key Output**: The `summary.md` includes the **Suggested Actions** section extracted from each issue's `overview.md`, with `[confidence:XX]` scores. ## Detailed Workflow @@ -188,7 +195,7 @@ See [workflow-triage-prep.md](./references/workflow-triage-prep.md) for the comp | Category | Definition | Action Needed | |----------|------------|---------------| -| 🔥 **Hot Issues** | ≥5 combined new comments + reactions since last triage | May need priority attention | +| 🌟 **Popular Issues** | ≥5 reactions (community interest) | May need priority attention | | 🆕 **Created This Week** | `createdAt` is after last weekly triage date | Use `Get-IssueDetails.ps1` + assign area | | ⏳ **Older Pending** | Created before this week, still no area label | Follow up — why not actioned? | | ✅ **Resolved** | In previous state, NOT in current | Acknowledge (got area label or closed) | @@ -200,6 +207,26 @@ See [workflow-triage-prep.md](./references/workflow-triage-prep.md) for the comp See [template-summary.md](./templates/template-summary.md) for the full template structure. +## Area Classification (LLM-Based) + +Issues without an `area-*` label are classified by the agent using LLM reasoning — **not** keyword matching. The agent: + +1. Fetches area label definitions via `Get-RepositoryLabels.ps1 -Filter "area-*"` +2. Reads the issue title, body, and comments from `Get-IssueDetails.ps1` +3. Reasons about the best area match and assigns a confidence score + +**Confidence scale:** + +| Score | Level | Meaning | +|-------|-------|---------| +| **80–100** | High | Issue explicitly names the component; maps to exactly one area | +| **60–79** | Medium-High | Strong signals (stack traces, API names) point to one area | +| **40–59** | Medium | Reasonable match, but 2+ areas are plausible | +| **20–39** | Low | Weak signal; best guess | +| **0–19** | Very Low | No clear signal; vague or multi-area | + +See [workflow-triage-prep.md](./references/workflow-triage-prep.md) for the full confidence rubric and adjusters. + ## Action Item Types | Action | When to Suggest | diff --git a/.github/skills/triage-meeting-prep/references/workflow-triage-prep.md b/.github/skills/triage-meeting-prep/references/workflow-triage-prep.md index 1165aac3dc..d63473e245 100644 --- a/.github/skills/triage-meeting-prep/references/workflow-triage-prep.md +++ b/.github/skills/triage-meeting-prep/references/workflow-triage-prep.md @@ -228,7 +228,18 @@ For each issue still pending: ## Phase 4: Generate Issue Reviews -### Step 4.1: Analyze All No-Area Issues +### Step 4.1: Fetch Area Label Definitions + +Before analyzing issues, fetch the current area labels and their descriptions from the repository: + +```powershell +# Get all area-* labels with descriptions +./.github/skills/triage-meeting-prep/scripts/Get-RepositoryLabels.ps1 -Filter "area-*" -OutputFormat json +``` + +This provides the complete list of valid area labels the agent can assign. **Keep this label list in context** — it is the source of truth for area classification throughout Phase 4. + +### Step 4.2: Analyze All No-Area Issues **Review ALL issues without area labels.** Each issue needs analysis for the summary. @@ -244,13 +255,15 @@ For each **no-area issue** (new or older): ```powershell ./Get-IssueDetails.ps1 -IssueNumber -OutputFormat summary ``` - -3. **Generate review files:** + +3. **Classify the area label using LLM reasoning** (see "Area Classification" below) + +4. **Generate review files:** - `Generated Files/issueReview//overview.md` - `Generated Files/issueReview//implementation-plan.md` -4. **Extract the "Suggested Actions" section from overview.md**, which includes: - - Label recommendations (Add/Remove) +5. **Extract the "Suggested Actions" section from overview.md**, which includes: + - Label recommendations (Add/Remove) — with area classification from step 3 - Clarifying questions (if Clarity < 50) - Similar issues found - Potential assignees @@ -258,6 +271,45 @@ For each **no-area issue** (new or older): **Optimization:** Run analysis in parallel for up to 5 issues at a time. +### Area Classification (LLM Reasoning) + +For each issue without an `area-*` label, classify it by reasoning about the issue content against the area label definitions fetched in Step 4.1. + +**Inputs for classification:** +- Issue title, body, and comments (from `Get-IssueDetails.ps1`) +- Area label names and descriptions (from `Get-RepositoryLabels.ps1 -Filter "area-*"`) +- Codebase context (use `grep_search`, `semantic_search` on `dev/` directories if needed) + +**Classification output format:** +``` +area- [confidence:XX] +``` + +**Confidence rubric for area classification:** + +| Score | Level | Criteria | +|-------|-------|----------| +| **80–100** | High | Issue explicitly names the component or API surface; maps to exactly one area with no ambiguity | +| **60–79** | Medium-High | Strong technical signals (stack traces, file paths, API names) point to one area; minor overlap possible | +| **40–59** | Medium | Reasonable match based on topic, but issue could plausibly belong to 2+ areas | +| **20–39** | Low | Weak signal; classification is a best guess based on general topic | +| **0–19** | Very Low | No clear technical signal; issue is vague, off-topic, or spans multiple areas equally | + +**Confidence adjusters:** +- Issue mentions specific API or class name from the area: **+20** +- Stack trace or file path points to `dev//`: **+20** +- Issue title directly references the area topic: **+15** +- Codebase search confirms the component: **+10** +- Issue could plausibly belong to 2+ areas: **−15** +- Issue is a general question with no technical detail: **−20** +- Issue is about tooling/build/infra rather than a specific SDK feature: **−10** + +**Rules:** +- Always suggest the **single best area**, even at low confidence +- If confidence < 40, note the ambiguity and list alternative candidate areas +- Format: `Suggested: area-Notifications [confidence:75]` +- If alternatives exist: `Alternatives: area-AppLifecycle [confidence:35], area-Packaging [confidence:30]` + ### Step 4.2: Extract Action Items for Summary From each NEW issue's `overview.md`, extract and collect: @@ -396,7 +448,7 @@ Now draft the reply using research findings: - Documentation links - Technical context from code analysis -4. **Assign confidence level (0-100)**: +4. **Assign confidence level (0-100)** using `[confidence:XX]` format: - **80-100**: Clear response backed by research - **60-79**: Good draft, research supports it - **40-59**: Limited research, needs team input @@ -417,6 +469,28 @@ Now draft the reply using research findings: - Requires team decision to reopen: -20 - Research inconclusive: -25 +**Confidence output format**: `[confidence:XX]` — grep-friendly for filtering. + +```bash +# Find high-confidence suggestions (80+) +grep "\[confidence:[89][0-9]\]" summary.md + +# Find low-confidence items needing review +grep "\[confidence:[0-3][0-9]\]" summary.md +``` + +```powershell +# Find high-confidence suggestions (80+) +Select-String -Path summary.md -Pattern '\[confidence:[89][0-9]\]' + +# Find low-confidence items needing review +Select-String -Path summary.md -Pattern '\[confidence:[0-3][0-9]\]' + +# List all items with confidence, sorted by value +Get-Content summary.md | Select-String '\[confidence:(\d+)\]' | + ForEach-Object { $_.Line } +``` + --- ## Phase 6: Report Generation diff --git a/.github/skills/triage-meeting-prep/scripts/Get-IssueDetails.ps1 b/.github/skills/triage-meeting-prep/scripts/Get-IssueDetails.ps1 index 31ded53ba3..eed4f8cf7b 100644 --- a/.github/skills/triage-meeting-prep/scripts/Get-IssueDetails.ps1 +++ b/.github/skills/triage-meeting-prep/scripts/Get-IssueDetails.ps1 @@ -145,6 +145,11 @@ $result.hasAreaLabel = $areaLabels.Count -gt 0 $result.areaLabels = @($areaLabels | ForEach-Object { $_.name }) $result.needsTriage = $triageLabels.Count -gt 0 +# Note: Area classification is performed by the agent using LLM reasoning. +# The agent uses Get-RepositoryLabels.ps1 -Filter "area-*" to fetch valid +# area labels with descriptions, then reasons about the best match based on +# the issue title, body, and comments. See SKILL.md for the confidence rubric. + # Output switch ($OutputFormat) { 'summary' { @@ -174,6 +179,11 @@ switch ($OutputFormat) { Write-Host " Comments: $($result.commentCount)" -ForegroundColor Gray Write-Host "" + if (-not $result.hasAreaLabel) { + Write-Host " 🏷️ Area Classification: Pending (agent will classify using LLM reasoning)" -ForegroundColor Yellow + Write-Host "" + } + if ($result.body) { Write-Host " 📝 Issue Body (first 500 chars):" -ForegroundColor White $preview = if ($result.body.Length -gt 500) { $result.body.Substring(0, 500) + '...' } else { $result.body }