From f84fbee7a6170b956348dd5e9a0df11821602a98 Mon Sep 17 00:00:00 2001 From: xyliugo Date: Wed, 15 Apr 2026 22:07:40 +0800 Subject: [PATCH 1/5] Fix review verification to check inline review comments, not just review body The task description asks agents to add keywords in inline review comments (code-level comments added via add_comment_to_pending_review), but the verification only checked the top-level review body from /pulls/{n}/reviews. Now also fetches /pulls/{n}/comments to check inline review comment content. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../issue_pr_commit_workflow/verify.py | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/tasks/github/standard/harmony/issue_pr_commit_workflow/verify.py b/tasks/github/standard/harmony/issue_pr_commit_workflow/verify.py index 349e74f2..58df7985 100644 --- a/tasks/github/standard/harmony/issue_pr_commit_workflow/verify.py +++ b/tasks/github/standard/harmony/issue_pr_commit_workflow/verify.py @@ -139,6 +139,18 @@ def _get_pr_reviews( return [] +def _get_pr_review_comments( + pr_number: int, headers: Dict[str, str], org: str, repo: str = "harmony" +) -> List[Dict]: + """Get all review comments (inline comments on code) for a PR.""" + success, comments = _get_github_api( + f"pulls/{pr_number}/comments", headers, org, repo + ) + if success and comments: + return comments + return [] + + def _check_issue_comment_references( comments: List[Dict], pr_number: int, keywords: List[str] ) -> bool: @@ -170,12 +182,21 @@ def _check_headings_and_content( return has_headings and has_keywords -def _check_pr_review_content(reviews: List[Dict], keywords: List[str]) -> bool: - """Check if PR has review comments containing required keywords.""" +def _check_pr_review_content( + reviews: List[Dict], keywords: List[str], review_comments: Optional[List[Dict]] = None +) -> bool: + """Check if PR has review bodies or inline review comments containing required keywords.""" + # Check review top-level bodies for review in reviews: body = review.get("body", "") if body and all(keyword.lower() in body.lower() for keyword in keywords): return True + # Check inline review comments (code-level comments added during review) + if review_comments: + for comment in review_comments: + body = comment.get("body", "") + if body and all(keyword.lower() in body.lower() for keyword in keywords): + return True return False @@ -334,7 +355,8 @@ def verify() -> bool: # 5. Check PR review comments print("5. Verifying PR review comments...") reviews = _get_pr_reviews(pr_number, headers, github_org) - if not _check_pr_review_content(reviews, REVIEW_KEYWORDS): + review_comments = _get_pr_review_comments(pr_number, headers, github_org) + if not _check_pr_review_content(reviews, REVIEW_KEYWORDS, review_comments): print( "Error: PR missing review comment with required technical keywords", file=sys.stderr, From 765339c6729cd084571c329b26f09e93a4942425 Mon Sep 17 00:00:00 2001 From: xyliugo Date: Thu, 16 Apr 2026 00:01:05 +0800 Subject: [PATCH 2/5] fix: correct UTC date in multi_branch_commit_aggregation verify script Commit 82b3afb9 has a GitHub API committer date of 2025-08-06T00:12:50Z, so the UTC date is 2025-08-06, not 2025-08-05. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../standard/harmony/multi_branch_commit_aggregation/verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/github/standard/harmony/multi_branch_commit_aggregation/verify.py b/tasks/github/standard/harmony/multi_branch_commit_aggregation/verify.py index 6d8646ab..80129a3c 100644 --- a/tasks/github/standard/harmony/multi_branch_commit_aggregation/verify.py +++ b/tasks/github/standard/harmony/multi_branch_commit_aggregation/verify.py @@ -219,7 +219,7 @@ def _check_merge_timeline(content: str) -> bool: "2025-08-06 | Merge pull request #29 from axion66/improve-readme-and-checks | 3efbf742533a375fc148d75513597e139329578b", "2025-08-06 | Merge pull request #30 from Yuan-ManX/harmony-format | 9d653a4c7382abc42d115014d195d9354e7ad357", "2025-08-06 | Merge pull request #28 from dkqjrm/fix-typo-format-md | 161e5fe2a57c63e9f8353c4c5b8faa3c3854bb5f", - "2025-08-05 | Merge pull request #26 from jordan-wu-97/jordan/fix-function-call-atomic-bool | 82b3afb9eb043343f322c937262cc50405e892c3", + "2025-08-06 | Merge pull request #26 from jordan-wu-97/jordan/fix-function-call-atomic-bool | 82b3afb9eb043343f322c937262cc50405e892c3", "2025-08-05 | Merge pull request #18 from openai/dev/scl/better-ci | b255cbeb6274adbea774f26fd9590922ce8874ed", "2025-08-05 | Merge pull request #21 from Tialo/main | 058ef3257c24fb099aac7960c10ce51c8e55d9fe", "2025-08-05 | Merge branch 'main' into dev/scl/better-ci | 6375a15ea1b0a486cbb1468964cf8f5800ff5a5c", From d17c31ada83d878f9935e0c2715b8e0eb86c97a5 Mon Sep 17 00:00:00 2001 From: xyliugo Date: Thu, 16 Apr 2026 00:14:27 +0800 Subject: [PATCH 3/5] fix: make deployment_status_workflow verify more flexible with markdown formatting Use regex-based matching instead of exact string matching for rollback plan elements, so the verifier accepts both markdown bold and plain text formatting from different GitHub Actions bot implementations. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../deployment_status_workflow/verify.py | 106 +++++++++++------- 1 file changed, 68 insertions(+), 38 deletions(-) diff --git a/tasks/github/standard/mcpmark-cicd/deployment_status_workflow/verify.py b/tasks/github/standard/mcpmark-cicd/deployment_status_workflow/verify.py index 02b9d397..a59ae82b 100644 --- a/tasks/github/standard/mcpmark-cicd/deployment_status_workflow/verify.py +++ b/tasks/github/standard/mcpmark-cicd/deployment_status_workflow/verify.py @@ -332,28 +332,69 @@ def _verify_deployment_issue( print(" ✅ Found rollback plan comment from GitHub Actions bot") # Check for required rollback plan elements - required_elements = [ - "**Previous Commit**:", - "**Current Commit**:", - "**Package Version**:", - "✅ Executable rollback script created", - "✅ Configuration backups saved", - "✅ Dependency verification script prepared", - "✅ Comprehensive rollback documentation generated", - "✅ Compressed rollback package created", - "**SHA256**:", - "**Artifact**:", - "Quick Rollback Commands", + # Use flexible matching: accept both markdown bold ("**Key**:") and plain text ("Key:") + import re as _re + + def _flex_match(comment: str, keyword: str) -> bool: + """Match 'keyword:' with optional markdown bold wrapping.""" + pattern = r"(\*\*\s*)?" + _re.escape(keyword) + r"(\s*\*\*)?\s*:" + return bool(_re.search(pattern, comment)) + + field_keywords = [ + "Previous Commit", + "Current Commit", + "Package Version", + "SHA256", + "Artifact", ] - for element in required_elements: - if element not in rollback_comment: - errors.append(f"Missing element in rollback plan: '{element}'") + for kw in field_keywords: + if _flex_match(rollback_comment, kw): + print(f" ✅ Found rollback plan field: '{kw}'") else: - print(f" ✅ Found rollback plan element: '{element}'") + errors.append(f"Missing field in rollback plan: '{kw}'") + + # Check for at least 5 checkmarks (✅) with rollback-related keywords + # Accept any reasonable wording as long as the semantic component is mentioned + rollback_component_keywords = [ + r"rollback\s+script", + r"configuration\s+backup|config.*backup", + r"dependency\s+verification|dependency.*check", + r"rollback\s+documentation|rollback.*doc", + r"rollback\s+package|compressed.*package", + ] + + checkmark_lines = [ + line.strip() + for line in rollback_comment.split("\n") + if "✅" in line + ] + + matched_components = 0 + for kw_pattern in rollback_component_keywords: + for line in checkmark_lines: + if _re.search(kw_pattern, line, _re.IGNORECASE): + matched_components += 1 + print(f" ✅ Found rollback component checkmark matching: '{kw_pattern}'") + break + + if matched_components < 5: + errors.append( + f"Expected at least 5 rollback component checkmarks (✅), found {matched_components}" + ) + else: + print(f" ✅ All 5 rollback component checkmarks found") + + # Check for Quick Rollback Commands section + if "Quick Rollback Command" in rollback_comment or "rollback command" in rollback_comment.lower(): + print(" ✅ Found rollback commands section") + else: + errors.append("Missing 'Quick Rollback Commands' section in rollback plan") # Verify commit SHAs in rollback comment - if f"**Current Commit**: {head_sha}" in rollback_comment: + # Accept both "**Current Commit**: sha" and "Current Commit: sha" + current_sha_pattern = r"(?:\*\*\s*)?Current\s+Commit(?:\s*\*\*)?\s*:\s*" + _re.escape(head_sha) + if _re.search(current_sha_pattern, rollback_comment): print(f" ✅ Current commit SHA verified: {head_sha}") else: errors.append( @@ -361,30 +402,19 @@ def _verify_deployment_issue( ) # Extract and verify previous commit SHA - if "**Previous Commit**:" in rollback_comment: - import re - - prev_sha_match = re.search( - r"\*\*Previous Commit\*\*:\s*([a-f0-9]{40})", rollback_comment - ) - if prev_sha_match: - prev_sha = prev_sha_match.group(1) - print(f" ✅ Previous commit SHA found: {prev_sha}") - - # Verify it's a valid 40-character SHA - if len(prev_sha) != 40: - errors.append( - f"Previous commit SHA has invalid length: {len(prev_sha)}" - ) - else: - errors.append( - "Previous commit SHA format not found in rollback comment" - ) + prev_sha_pattern = r"(?:\*\*\s*)?Previous\s+Commit(?:\s*\*\*)?\s*:\s*([a-f0-9]{40})" + prev_sha_match = _re.search(prev_sha_pattern, rollback_comment) + if prev_sha_match: + prev_sha = prev_sha_match.group(1) + print(f" ✅ Previous commit SHA found: {prev_sha}") else: - errors.append("Previous commit SHA not found in rollback comment") + errors.append( + "Previous commit SHA (40-char hex) not found in rollback comment" + ) # Verify SHA256 checksum is present - sha256_match = re.search(r"\*\*SHA256\*\*:\s*([a-f0-9]{64})", rollback_comment) + sha256_pattern = r"(?:\*\*\s*)?SHA256(?:\s*\*\*)?\s*:\s*([a-f0-9]{64})" + sha256_match = _re.search(sha256_pattern, rollback_comment) if sha256_match: sha256_value = sha256_match.group(1) print(f" ✅ SHA256 checksum found: {sha256_value[:16]}...") From df7201246169950cd584068ce6d316ce50bf7946 Mon Sep 17 00:00:00 2001 From: xyliugo Date: Tue, 5 May 2026 15:36:13 +0800 Subject: [PATCH 4/5] fix: accept both valid SHAs for CHANGELOG 1.0.65 in feature_commit_tracking verify The CHANGELOG 1.0.65 feature can be identified by two valid commits: - 94dcaca5: merge commit that brought 1.0.65 into pr/2466-QwertyJack-main - 5faa082d: the actual commit that first added 1.0.65 to CHANGELOG.md on main Both are reasonable interpretations of "when the changelog was updated to include version 1.0.65", so the verify script now accepts either answer. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../feature_commit_tracking/verify.py | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tasks/github/standard/claude-code/feature_commit_tracking/verify.py b/tasks/github/standard/claude-code/feature_commit_tracking/verify.py index 90c6ad47..18bed08f 100644 --- a/tasks/github/standard/claude-code/feature_commit_tracking/verify.py +++ b/tasks/github/standard/claude-code/feature_commit_tracking/verify.py @@ -129,16 +129,23 @@ def verify_task() -> bool: } # Expected feature commits based on exploration + # For CHANGELOG Version 1.0.65, two valid answers exist: + # - 94dcaca5: merge commit that brought 1.0.65 into pr/2466-QwertyJack-main branch + # - 5faa082d: the actual commit that first added 1.0.65 content to CHANGELOG.md on main expected_features = { - "Shell Completion Scripts": "8a0febdd09bda32f38c351c0881784460d69997d", - "CHANGELOG Version 1.0.65": "94dcaca5d71ad82644ae97f3a2b0c5eb8b63eae0", - "Rust Extraction Improvements": "50e58affdf1bfc7d875202bc040ebe0dcfb7d332", + "Shell Completion Scripts": ["8a0febdd09bda32f38c351c0881784460d69997d"], + "CHANGELOG Version 1.0.65": [ + "94dcaca5d71ad82644ae97f3a2b0c5eb8b63eae0", + "5faa082d6e4e5300485daafb94615fe133175055", + ], + "Rust Extraction Improvements": ["50e58affdf1bfc7d875202bc040ebe0dcfb7d332"], } # Expected authors for each commit expected_authors = { "8a0febdd09bda32f38c351c0881784460d69997d": "gitmpr", "94dcaca5d71ad82644ae97f3a2b0c5eb8b63eae0": "QwertyJack", + "5faa082d6e4e5300485daafb94615fe133175055": "actions-user", "50e58affdf1bfc7d875202bc040ebe0dcfb7d332": "alokdangre", } @@ -146,6 +153,7 @@ def verify_task() -> bool: expected_messages = { "8a0febdd09bda32f38c351c0881784460d69997d": "feat: add shell completions (bash, zsh, fish)", "94dcaca5d71ad82644ae97f3a2b0c5eb8b63eae0": "Merge branch 'anthropics:main' into main", + "5faa082d6e4e5300485daafb94615fe133175055": "chore: Update CHANGELOG.md", "50e58affdf1bfc7d875202bc040ebe0dcfb7d332": "Enhance Rust extraction and output handling in workflows", } @@ -153,6 +161,7 @@ def verify_task() -> bool: expected_dates = { "8a0febdd09bda32f38c351c0881784460d69997d": "2025-08-01", "94dcaca5d71ad82644ae97f3a2b0c5eb8b63eae0": "2025-08-02", + "5faa082d6e4e5300485daafb94615fe133175055": "2025-07-31", "50e58affdf1bfc7d875202bc040ebe0dcfb7d332": "2025-08-09", } @@ -197,7 +206,7 @@ def verify_task() -> bool: for feature in features: found_features[feature["name"]] = feature["sha"] - for feature_name, expected_sha in expected_features.items(): + for feature_name, expected_shas in expected_features.items(): if feature_name not in found_features: print( f"Error: Feature '{feature_name}' not found in table", file=sys.stderr @@ -205,9 +214,9 @@ def verify_task() -> bool: return False actual_sha = found_features[feature_name] - if actual_sha != expected_sha: + if actual_sha not in expected_shas: print( - f"Error: Wrong SHA for '{feature_name}'. Expected: {expected_sha}, Got: {actual_sha}", + f"Error: Wrong SHA for '{feature_name}'. Expected one of: {expected_shas}, Got: {actual_sha}", file=sys.stderr, ) return False @@ -216,8 +225,12 @@ def verify_task() -> bool: # 5. Verify each commit exists and has correct author print("5. Verifying commit details...") + all_expected_shas = set() + for shas in expected_features.values(): + all_expected_shas.update(shas) + for feature in features: - if feature["sha"] in expected_features.values(): + if feature["sha"] in all_expected_shas: success, commit_data = _verify_commit_exists( feature["sha"], headers, github_org ) From 096fc148052a2bf5831e0c7a25f64f8bfbbd03ab Mon Sep 17 00:00:00 2001 From: xyliugo Date: Wed, 13 May 2026 00:52:39 +0800 Subject: [PATCH 5/5] fix(github): improve task descriptions to match the verification logic --- .../description.md | 10 +++--- .../issue_management_workflow/description.md | 31 ++++++++++++++++--- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/tasks/github/standard/claude-code/claude_collaboration_analysis/description.md b/tasks/github/standard/claude-code/claude_collaboration_analysis/description.md index d42eebb0..31f66dfa 100644 --- a/tasks/github/standard/claude-code/claude_collaboration_analysis/description.md +++ b/tasks/github/standard/claude-code/claude_collaboration_analysis/description.md @@ -1,10 +1,10 @@ I need you to analyze the collaboration patterns between human developers and Claude (the AI assistant) in the repository by examining all available commit history, then create a comprehensive analysis report and submit it as a new file to the repository. **Step 1: Commit History Analysis** -Analyze ALL commits in the repository to identify: +Analyze all commits reachable from the default branch (`main`) to identify: -1. **Claude Co-Authored Commits**: Find all commits that were co-authored by Claude (look for "Co-Authored-By: Claude " in commit messages) -2. **Top Claude Collaborators**: Identify the top 3 human developers who most frequently collaborated with Claude +1. **Claude Co-Authored Commits**: Find all commits whose message contains a `Co-Authored-By: Claude ` trailer. Match case-insensitively (both `Co-Authored-By` and `Co-authored-by` count). Count each commit at most once. +2. **Top Claude Collaborators**: Identify the top 3 human developers who most frequently collaborated with Claude. **Step 2: Create Collaboration Analysis Report** Create a file called `CLAUDE_COLLABORATION_ANALYSIS.md` in the repository root with: @@ -23,7 +23,7 @@ Create a file called `CLAUDE_COLLABORATION_ANALYSIS.md` in the repository root w ``` Include the top 3 developers by number of Claude collaborations. -**Step 3: Commit Analysis to Repository** +**Step 3: Commit the Analysis to Repository** Commit the `CLAUDE_COLLABORATION_ANALYSIS.md` file to the main branch with: - Commit message: "Add Claude AI collaboration analysis report" -- Ensure all statistics are accurate based on actual commit data \ No newline at end of file +- Ensure all statistics are accurate based on actual commit data diff --git a/tasks/github/standard/mcpmark-cicd/issue_management_workflow/description.md b/tasks/github/standard/mcpmark-cicd/issue_management_workflow/description.md index 32480c8f..a2dfbda7 100644 --- a/tasks/github/standard/mcpmark-cicd/issue_management_workflow/description.md +++ b/tasks/github/standard/mcpmark-cicd/issue_management_workflow/description.md @@ -20,11 +20,26 @@ Create `.github/workflows/issue-automation.yml` that triggers on `issues` events - For issues with a title containing "Epic", create exactly 4 sub-issues with the pattern: "[SUBTASK] [Original Title] - Task N: [Task Name]" - Task names: 1. Requirements Analysis, 2. Design and Architecture, 3. Implementation, 4. Testing and Documentation - Links sub-issues to parent using "Related to #[parent-number]" in sub-issue body - - Updates parent issue body with "## Epic Tasks" checklist linking to sub-issue numbers + - Updates parent issue body with an "## Epic Tasks" checklist that links to sub-issue numbers. Each checklist line MUST contain the literal substring `- [ ] #` (a GitHub-style task-list reference) AND the corresponding task name. The recommended line format is: + + ``` + - [ ] # - Task : + ``` + + Example for sub-issues #5–#8: + + ``` + ## Epic Tasks + + - [ ] #5 - Task 1: Requirements Analysis + - [ ] #6 - Task 2: Design and Architecture + - [ ] #7 - Task 3: Implementation + - [ ] #8 - Task 4: Testing and Documentation + ``` - All sub-issues get `enhancement` and `needs-review` labels ### 3. **auto-response** job: - - Checks if the issue author is creating their first issue in this repository (not first on GitHub globally, but first in this specific repo) + - Checks if the issue author is creating their first issue in this repository (not first on GitHub globally, but first in this specific repo). For the purpose of this check, automation-created sub-issues authored by `github-actions[bot]` do not count as prior issues by the human author. - If first issue in repo: adds `first-time-contributor` label and posts welcome message - Posts different responses based on issue type: - `bug` issues: comment must contain "Bug Report Guidelines" @@ -82,18 +97,24 @@ Create a comprehensive pull request and merge it to main: - Merge the pull request to main branch **Step 5: Test the Workflow** -Create test issues to demonstrate the issue automation workflow: +Create test issues to demonstrate the issue automation workflow. Create them in the order listed below, and ensure that the **Bug issue is the very first issue you (the human author) open in this repository** so that the first-time-contributor logic fires on it. + +When writing the issue bodies, be careful about which priority keywords you include. Priority is matched against title OR body, with the highest match winning, so avoid adding higher-priority keywords (e.g. `critical`, `urgent`, `production`, `outage`) when you want a lower priority outcome. 1. **Bug Issue**: "Bug: Login form validation not working" - - Expected: `bug`, `priority-high`, `needs-triage`→`needs-review`, milestone "v1.0.0" + - This must be the first issue authored by you in the repo (so that `first-time-contributor` is applied). + - Title/body should include high-priority wording (e.g. `important`, `high`, `blocking`) but MUST NOT contain any critical keyword (`critical`, `urgent`, `production`, `outage`). + - Expected: `bug`, `priority-high`, `first-time-contributor`, `needs-triage`→`needs-review`, milestone "v1.0.0" - Auto-response comment must contain "Bug Report Guidelines" 2. **Epic Issue**: "Epic: Redesign user dashboard interface" + - Title/body should include high-priority wording (`important`, `high`, `blocking`) and MUST NOT contain any critical keyword. - Expected: `epic`, `priority-high`, `needs-triage`→`needs-review`, milestone "v1.0.0" - Must create 4 sub-issues with `enhancement` and `needs-review` labels - - Parent updated with "## Epic Tasks" checklist, sub-issues linked with "Related to #[parent-number]" + - Parent updated with "## Epic Tasks" checklist whose lines contain `- [ ] #` plus the task name (see the example format under the task-breakdown job above), sub-issues linked back with "Related to #[parent-number]" - Auto-response comment must contain "Feature Request Process" 3. **Maintenance Issue**: "Weekly maintenance cleanup and refactor" + - Title/body should use medium/normal wording and MUST NOT contain any high or critical keyword. - Expected: `maintenance`, `priority-medium`, `needs-triage`→`needs-review`, no milestone - Auto-response comment must contain "Maintenance Guidelines" \ No newline at end of file