diff --git a/.github/workflows/notify_failure.yml b/.github/workflows/notify_failure.yml index 76633cc..204546c 100644 --- a/.github/workflows/notify_failure.yml +++ b/.github/workflows/notify_failure.yml @@ -7,6 +7,7 @@ on: - "Test Failure Notification" - "Crowdin Multiple Translations Report" - "Zendesk Ticket Triage" + - "Zendesk Resolve Positive Reviews" types: - completed diff --git a/.github/workflows/zendesk_resolve_reviews.yml b/.github/workflows/zendesk_resolve_reviews.yml new file mode 100644 index 0000000..59653c3 --- /dev/null +++ b/.github/workflows/zendesk_resolve_reviews.yml @@ -0,0 +1,82 @@ +name: Zendesk Resolve Positive Reviews + +# Weekly counterpart to the daily triage: solves the 4-5★ AppFollow reviews that +# were never going to be actioned, so the unsolved backlog reflects real work. +# 4,812 of them were sitting in `new` when this was written, and ~420 arrive a week. +# +# No state file is needed — a solved ticket drops out of the query, so runs are +# idempotent and the first few drain the backlog before the schedule just keeps pace. +# +# ⚠️ This workflow writes to Zendesk. A scheduled run always applies; a manual run +# is a dry run unless you tick `apply`. Solving a ticket can fire triggers and +# automations (satisfaction surveys among them), so read the warning in +# zendesk_triage/resolve_reviews.py before the first applied run. +on: + schedule: + - cron: "0 5 * * 1" + workflow_dispatch: + inputs: + apply: + description: "Actually solve the tickets (unticked = dry run, changes nothing)" + type: boolean + default: false + max_tickets: + description: "Max tickets to solve this run (default 1000, Zendesk's search cap)" + required: false + +# A second run while one is mid-flight would re-fetch tickets the first has already +# queued for update, and bulk jobs are asynchronous. Queue rather than cancel, so a +# manual run never abandons a scheduled run's in-flight batches. +concurrency: + group: zendesk-resolve-reviews + cancel-in-progress: false + +# Everything it writes goes to Zendesk over its own credentials. +permissions: + contents: read + +jobs: + resolve: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -r zendesk_triage/requirements.txt + + # Inputs arrive via env and are stripped to digits, never interpolated into the + # command. A scheduled run applies; a manual one only if it asked to, so the + # dispatch button cannot solve tickets by accident. + - name: Resolve options + id: cfg + env: + INPUT_MAX: ${{ github.event.inputs.max_tickets }} + APPLY: ${{ github.event_name == 'schedule' || github.event.inputs.apply == 'true' }} + run: | + max=$(printf '%s' "${INPUT_MAX:-1000}" | tr -cd '0-9') + : "${max:=1000}" + flags="" + if [ "$APPLY" = "true" ]; then flags="--apply"; fi + echo "max=$max" >> "$GITHUB_OUTPUT" + echo "flags=$flags" >> "$GITHUB_OUTPUT" + if [ "$APPLY" = "true" ]; then + echo "Applying: up to $max review(s) at 4★ or better will be solved." + else + echo "Dry run: reporting only, nothing will be changed." + fi + + - name: Resolve positive reviews + env: + ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }} + ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }} + ZENDESK_API_TOKEN: ${{ secrets.ZENDESK_API_TOKEN }} + run: | + python zendesk_triage/resolve_reviews.py \ + ${{ steps.cfg.outputs.flags }} \ + --max-tickets "${{ steps.cfg.outputs.max }}" diff --git a/README.md b/README.md index f9efbae..e1e8d9a 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Each line leads with a severity marker, a category emoji and a platform icon, li ``` 🗂️ **Zendesk triage** — analyzed **16** of **46** tickets in the window (created in the past 2 days). Skipped **30** positive app-store review(s). -Backlog: **5,680** unsolved tickets in total (not triaged). +Backlog: **428** unsolved excluding app-store reviews (**5,252** more are reviews, not triaged). **9** worth looking into. ⭐ **6** · 🐛 **3** · ❓ **2** · 🔑 **1** · ⚖️ **1** · 🔒 **1** Likely duplicates: **push-notifications-not-delivered** ×5 (#27637, #27610, #27606, #27605) @@ -115,7 +115,7 @@ Likely duplicates: **push-notifications-not-delivered** ×5 (#27637, #27610, #27 | Category | The emoji from `CATEGORY_SPECS`, so it matches the tally line | | Platform | 🤖 Android · 🍎 iOS · 🖥️ desktop (all three) · 🌐 multiple · ❔ unknown | -The header accounts for the batch in full, so nothing is dropped silently. An abuse report also carries the reported Session ID on its line, since that is the actionable part and it saves opening the ticket. +The header accounts for the batch in full, so nothing is dropped silently. The backlog line deliberately **excludes app-store reviews**: 92% of unsolved tickets are AppFollow reviews, so the unqualified number reads as roughly 13× the queue that actually needs a human (5,680 against 428). Both counts come from Zendesk's count-only search endpoint, one request each and both best-effort — if the review-excluded count fails, the line falls back to the plain total rather than disappearing. An abuse report also carries the reported Session ID on its line, since that is the actionable part and it saves opening the ticket. **Plain message content, no embeds.** The lines carry their own structure, so an embed added a border and nothing else. The cost is the character budget: Discord caps message content at 2,000 against an embed description's 4,096, and a masked link on the id spends 54 characters that the reader never sees. A real 9-highlight day comes to ~2,400 characters, so it arrives as two messages. Lines are clipped (`SUMMARY_CHARS`, `ROOT_CAUSE_CHARS`) and chunked against 2,000, counting the newlines that join them; each message records which ticket ids it accounts for, which is what makes a partial post failure recoverable. @@ -259,6 +259,38 @@ python zendesk_triage/triage.py --dump-batch /tmp/batch.json --window-hours 48 python zendesk_triage/triage.py --findings /tmp/findings.json --dry-run ``` +## Zendesk Resolve Positive Reviews + +Weekly counterpart to the triage: it solves the 4-5★ AppFollow reviews that were never going to be actioned, so the unsolved backlog reflects work that actually exists. When this was written **5,253** reviews were unsolved — **4,812** of them still `new` — against **428** non-review unsolved tickets. Solving reviews was already being done by hand: **4,959** were already solved or closed. + +> ⚠️ **This workflow writes to Zendesk.** A scheduled run always applies. A manual run is a **dry run** unless you tick `apply`, so the dispatch button cannot solve tickets by accident. Read the warning at the top of [resolve_reviews.py](zendesk_triage/resolve_reviews.py) before the first applied run. + +### What it will and will not touch + +Deliberately narrow, because a mis-aimed bulk status change is not recoverable by re-running: + +- **App-store reviews only**, by the same detection the triage uses — `triage.is_store_review`, so the two can't drift apart. Every fetched ticket is re-checked locally, since the query can't express the rating. +- **Rated 4★ or better.** A fixed floor (`MIN_STARS`), not a flag — 3★ and below are what the triage reads as bug reports in disguise, so a lower floor would have this job close the reviews most worth looking at. A review whose stars can't be parsed from the subject is skipped, never solved. +- **`new` only** — untouched reviews. The other 441 unsolved reviews are `open`, and every one of a 100-ticket sample had an assignee, a group, and an `updated_at` past its `created_at`: something already acted on them, so a bulk status change has no business there. There is deliberately no flag to widen this. +- **`solved`, never `closed`.** Solved is reversible; closed is not. +- **Tagged** `auto-resolved-review`, so they stay identifiable and a trigger can exclude them, and annotated with a **private** note — a public comment would email the person who wrote the review. + +### Before the first applied run + +Solving a ticket fires triggers and automations, and an AppFollow requester may carry a real email address. **A satisfaction survey trigger would email thousands of app-store reviewers.** Check Admin Center → Objects and rules → Business rules first, then do the first applied run with `--max-tickets 5` so the effects are observable before they're bulk. + +### How it drains + +No state file: a solved ticket drops out of the query, so runs are idempotent. Zendesk's search API caps at 1,000 results, so a run can never see more than that — the first few runs work the backlog down and after that the weekly schedule comfortably clears the ~420 reviews a week that arrive. `update_many` takes [100 ids per request](https://developer.zendesk.com/api-reference/ticketing/tickets/tickets/#update-many-tickets) and is asynchronous, so each batch's job is polled to completion and per-ticket failures fail the run rather than being reported as success. + +### Required Secrets + +`ZENDESK_SUBDOMAIN`, `ZENDESK_EMAIL`, `ZENDESK_API_TOKEN` — the same three the triage uses. No Claude or Discord credentials: it posts nothing. + +### Schedule + +Mondays at 05:00 UTC. Failures are reported through the Discord failure-notification workflow, which watches this workflow by name — renaming `Zendesk Resolve Positive Reviews` means updating the `workflows:` list in [`notify_failure.yml`](.github/workflows/notify_failure.yml) too. + ## Workflow Failure Notificaiton If a workflow fails and is in the list of workflows monitored by the failure notificaiton workflow, the failure notificaiton workflow will send a message to a discord webhook. diff --git a/zendesk_triage/resolve_reviews.py b/zendesk_triage/resolve_reviews.py new file mode 100644 index 0000000..fec1a59 --- /dev/null +++ b/zendesk_triage/resolve_reviews.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +""" +Solve the app-store reviews that were never going to be actioned. + +59% of all tickets are 4-5★ AppFollow reviews with nothing to act on, and 4,812 of +them sit unsolved in `new`. The daily triage already counts them without spending +tokens (see partition_reviews in triage.py); this closes them out so the unsolved +backlog reflects work that actually exists. + + ⚠️ This writes to Zendesk. Nothing happens without --apply: by default the + script reports exactly what it would solve and exits. + + ⚠️ Solving a ticket can fire triggers and automations, including satisfaction + surveys, and an AppFollow requester may carry a real email address. Check + Admin Center → Objects and rules → Business rules before the first --apply, + and do that run with a small --max-tickets so the effects are observable. + Every ticket is tagged (--tag) so a trigger can exclude them. + +What it will and will not touch, deliberately narrow: + + * app-store reviews only, by the same detection the triage uses — the Zendesk + `via.channel`, or a leading ★ run in the subject + * with a parsed rating of MIN_STARS (4) or better — a fixed floor, not an option, + because 3★ and below are what the triage wants to see. A review whose stars + cannot be parsed is skipped, never solved + * in `new` only. The other 441 unsolved reviews are `open`, and every one of a + 100-ticket sample had an assignee, a group, and an updated_at past its + created_at — something already acted on them, which is exactly what a bulk + status change should keep its hands off + * `solved`, never `closed` — solved is reversible, closed is not + +No state file: solved tickets drop out of the query, so runs are idempotent and a +weekly schedule drains the backlog and then keeps pace with new reviews. + +Config (env vars, or flags for local runs): + ZENDESK_SUBDOMAIN e.g. "mycompany" -> https://mycompany.zendesk.com + ZENDESK_EMAIL agent email for API token auth + ZENDESK_API_TOKEN Zendesk API token + +Usage: + # report what would be solved, touch nothing (the default) + python resolve_reviews.py + + # actually solve them + python resolve_reviews.py --apply + + # first real run: small, observable + python resolve_reviews.py --apply --max-tickets 5 +""" +import argparse +import os +import sys +import time + +import requests + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import triage # noqa: E402 (needs the path insert above) + +# Reviews at or above this rating carry nothing to act on. Fixed rather than a flag: +# 3★ and below are what the triage treats as bug reports in disguise, so lowering the +# floor would have this job close the reviews most worth reading. Changing it is a +# deliberate edit here, not something a dispatch can do by accident. +MIN_STARS = 4 +# Zendesk's search API returns at most 1000 results, so a run can never see more +# than that anyway. At ~420 new reviews a week the first few runs drain the +# backlog and every run after that clears the week's intake. +DEFAULT_MAX_TICKETS = 1000 +# update_many takes at most 100 ids per request. +# https://developer.zendesk.com/api-reference/ticketing/tickets/tickets/#update-many-tickets +BATCH_SIZE = 100 +RESOLVED_TAG = "auto-resolved-review" +# Long enough for a 100-ticket batch, short enough that a wedged job fails the run +# rather than holding a scheduled job open. +JOB_TIMEOUT_SECONDS = 300 +JOB_POLL_SECONDS = 3 + + +def build_query(): + """Untouched app-store reviews, newest first. + + `via:any_channel` is what AppFollow imports arrive on, and it is the cheap half + of the filter — the star rating lives in the subject, which Zendesk's search + index will not match, so the rating is applied locally in select_resolvable. + + `status:new` rather than `status= 400: + sys.exit(f"update_many failed ({resp.status_code}): {resp.text[:300]}") + job = (resp.json() or {}).get("job_status") or {} + job_id = job.get("id") + if not job_id: + sys.exit(f"update_many returned no job id: {str(resp.text)[:300]}") + return job_id + + +def wait_for_job(session, subdomain, job_id, timeout=JOB_TIMEOUT_SECONDS): + """Poll a bulk-update job to completion; return (solved_count, failures). + + update_many is asynchronous, so a 200 on the PUT only means Zendesk queued the + work. Without this a run would report success for tickets that failed to update. + """ + url = f"https://{subdomain}.zendesk.com/api/v2/job_statuses/{job_id}.json" + deadline = time.monotonic() + timeout + while True: + resp = triage.request_with_retry(session, "GET", url) + if resp.status_code >= 400: + sys.exit(f"could not read job {job_id} ({resp.status_code}): {resp.text[:200]}") + job = (resp.json() or {}).get("job_status") or {} + status = job.get("status") + if status in ("completed", "failed", "killed"): + results = job.get("results") or [] + failures = [r for r in results if r.get("success") is False or r.get("error")] + solved = [r for r in results if r.get("success") is not False and not r.get("error")] + if status != "completed": + print(f"Job {job_id} ended as {status}.") + return len(solved), failures + if time.monotonic() >= deadline: + sys.exit(f"Job {job_id} still {status} after {timeout}s; " + f"check it in Zendesk before re-running.") + time.sleep(JOB_POLL_SECONDS) + + +def main(): + parser = argparse.ArgumentParser( + description="Solve non-actionable positive app-store reviews in Zendesk.") + parser.add_argument("--apply", action="store_true", + help="Actually solve the tickets. Without this the script " + "reports what it would do and changes nothing.") + parser.add_argument("--max-tickets", type=int, default=DEFAULT_MAX_TICKETS, metavar="N", + help=f"Runaway guard on tickets solved per run " + f"(default: {DEFAULT_MAX_TICKETS}, Zendesk's search cap).") + parser.add_argument("--tag", default=RESOLVED_TAG, + help=f"Tag added to every ticket solved, so they stay " + f"identifiable and a trigger can exclude them " + f"(default: {RESOLVED_TAG}).") + parser.add_argument("--no-note", action="store_true", + help="Skip the private note explaining the automated close.") + parser.add_argument("--subdomain", help="Zendesk subdomain (else ZENDESK_SUBDOMAIN).") + parser.add_argument("--email", help="Zendesk agent email (else ZENDESK_EMAIL).") + parser.add_argument("--api-token", help="Zendesk API token (else ZENDESK_API_TOKEN).") + args = parser.parse_args() + + subdomain = triage.get_env("ZENDESK_SUBDOMAIN", args.subdomain) + email = triage.get_env("ZENDESK_EMAIL", args.email) + api_token = triage.get_env("ZENDESK_API_TOKEN", args.api_token) + + session = triage.zendesk_session(email, api_token) + query = build_query() + tickets, total_matched = triage.fetch_tickets(session, subdomain, query, args.max_tickets) + matched = "?" if total_matched is None else total_matched + print(f"Fetched {len(tickets)} of {matched} matching tickets (query: {query!r}).") + + resolvable, skipped = select_resolvable(tickets, MIN_STARS) + if skipped: + reasons = {} + for _, reason in skipped: + reasons[reason] = reasons.get(reason, 0) + 1 + for reason, count in sorted(reasons.items(), key=lambda kv: -kv[1]): + print(f" skipped {count}: {reason}") + if not resolvable: + print("Nothing to solve.") + return + + print(f"{len(resolvable)} review(s) at {MIN_STARS}★ or better would be solved " + f"and tagged {args.tag!r}.") + if total_matched is not None and total_matched > len(tickets): + print(f"Note: {total_matched - len(tickets)} more match the query than this run " + f"looked at; the next run picks them up.") + + if not args.apply: + print("Dry run: nothing was changed. Re-run with --apply to solve them.") + return + + note = None if args.no_note else ( + f"Solved automatically: {MIN_STARS}★ or better app-store review with no " + f"actionable content. See zendesk_triage/resolve_reviews.py.") + + solved_total, failures = 0, [] + ids = [t["id"] for t in resolvable] + for number, batch in enumerate(batches(ids), start=1): + print(f" batch {number}: solving {len(batch)} ticket(s)…") + job_id = solve_batch(session, subdomain, batch, args.tag, note) + solved, batch_failures = wait_for_job(session, subdomain, job_id) + solved_total += solved + failures.extend(batch_failures) + + print(f"Solved {solved_total} of {len(ids)} ticket(s).") + if failures: + for failure in failures[:10]: + print(f" failed: id={failure.get('id')} {failure.get('error') or failure}") + sys.exit(f"{len(failures)} ticket(s) failed to update.") + + +if __name__ == "__main__": + main() diff --git a/zendesk_triage/test_resolve_reviews.py b/zendesk_triage/test_resolve_reviews.py new file mode 100644 index 0000000..07051e8 --- /dev/null +++ b/zendesk_triage/test_resolve_reviews.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Tests for the positive-review resolver. + +Stdlib unittest, same as test_triage.py. Everything is offline — the Zendesk calls +run against a stub session. Run from anywhere: + + python -m unittest discover -s zendesk_triage -v + +This script writes to Zendesk, so the tests lean on the guards rather than the happy +path: that a dry run cannot PUT, that only positive app-store reviews are selected, +and that an asynchronous job's failures are surfaced instead of swallowed. +""" +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import resolve_reviews # noqa: E402 +import triage # noqa: E402 +from test_triage import FakeResponse, FakeSession, NoSleep # noqa: E402 + + +def review(ticket_id, stars=5, channel="any_channel", subject=None): + """An AppFollow-shaped review ticket: the rating lives in the subject.""" + if subject is None: + subject = ("★" * stars) + ("☆" * (5 - stars)) + " Great app" + return { + "id": ticket_id, + "result_type": "ticket", + "subject": subject, + "description": "d", + "tags": ["appfollow"], + "status": "new", + "via": {"channel": channel}, + } + + +def human_ticket(ticket_id): + return { + "id": ticket_id, + "result_type": "ticket", + "subject": "Cannot log in after update", + "description": "d", + "tags": [], + "status": "new", + "via": {"channel": "email"}, + } + + +class TestQuery(unittest.TestCase): + def test_only_untouched_reviews_are_eligible(self): + """`new`, not `status= 400: print(f"Note: could not count the unsolved backlog ({resp.status_code}).") @@ -528,6 +528,9 @@ def save_state(path, state, reported, retention_days): # channel, not on tags. 4-5 star reviews were 59% of *all* tickets and are never # actionable, so counting them beats paying tokens to classify them. REVIEW_CHANNEL = "any_channel" +# The same backlog minus store reviews. 92% of unsolved tickets are AppFollow +# reviews, so the unqualified number reads as ~13x the queue that needs a human. +BACKLOG_NON_REVIEW_QUERY = f"{BACKLOG_QUERY} -via:{REVIEW_CHANNEL}" STAR_SUBJECT = re.compile(r"^\s*([★☆]{1,10})") DEFAULT_REVIEW_STAR_FLOOR = 3 @@ -883,7 +886,11 @@ def build_header(findings, highlights, stats=None): window += f" Skipped **{reviews}** positive app-store review(s)." lines = [window] - if backlog is not None: + non_review = stats.get("total_unsolved_non_review") + if backlog is not None and non_review is not None: + lines.append(f"Backlog: **{non_review:,}** unsolved excluding app-store reviews " + f"(**{backlog - non_review:,}** more are reviews, not triaged).") + elif backlog is not None: lines.append(f"Backlog: **{backlog:,}** unsolved tickets in total (not triaged).") serious = by_severity.get("crash", 0) + by_severity.get("data_loss", 0) @@ -1093,6 +1100,8 @@ def main(): stats["matched"] = total_matched stats["total_unsolved"] = fetch_total_unsolved(zd, subdomain) + stats["total_unsolved_non_review"] = fetch_total_unsolved( + zd, subdomain, BACKLOG_NON_REVIEW_QUERY) # Drop positive store reviews before anything expensive: they were 59% of all # tickets in the sample and never actionable.