diff --git a/services/hackbot-pulse-listener/Dockerfile b/services/hackbot-pulse-listener/Dockerfile new file mode 100644 index 0000000000..2376966800 --- /dev/null +++ b/services/hackbot-pulse-listener/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.14-slim AS builder + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV UV_PROJECT_ENVIRONMENT=/opt/venv + +WORKDIR /app + +# Install external deps without building workspace members. +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=VERSION,target=VERSION \ + uv sync --frozen --no-dev --no-install-workspace --package hackbot-pulse-listener + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,target=/app,rw \ + uv sync --locked --no-dev --no-editable --package hackbot-pulse-listener + +FROM python:3.14-slim AS base + +COPY --from=builder /opt/venv /opt/venv +WORKDIR /app + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PATH="/opt/venv/bin:$PATH" + +RUN useradd --create-home --shell /bin/bash app +USER app + +CMD ["python", "-m", "app"] diff --git a/services/hackbot-pulse-listener/README.md b/services/hackbot-pulse-listener/README.md new file mode 100644 index 0000000000..57984bd5e7 --- /dev/null +++ b/services/hackbot-pulse-listener/README.md @@ -0,0 +1,44 @@ +# Hackbot Pulse Listener + +Listens to Taskcluster build-failure pulse messages, and for failed **Firefox build +tasks** triggers the `build-repair` hackbot agent through the hackbot-api. When a run +finishes (minutes later) it emails the developer who pushed the change a link to the +hackbot UI and a summary of the analysis and fix. + +## How it works + +1. Consume `task-failed` messages from `pulse.mozilla.org`. +2. Keep only **build-kind** tasks (`tags.kind == "build"`) on a watched `project` + (`WATCHED_REPOS`, default `try`). Build tasks don't run tests, so a failure is a + compilation/link error. +3. Fetch the task definition to read `GECKO_HEAD_REV` (the revision is not in the message). +4. Dedupe by revision with an in-memory TTL cache, so only one agent run is triggered per + revision even when many build tasks fail for the same push. +5. `POST /agents/build-repair/runs`, then poll `GET /runs/{run_id}` until terminal and send + the report email. + +The dedupe cache and pending-run tracking are in-memory (reset on restart). + +## Run locally + +```bash +export PULSE_USER=... PULSE_PASSWORD=... # https://pulseguardian.mozilla.org +export HACKBOT_API_URL=https://hackbot-api.../ HACKBOT_API_KEY=... +export HACKBOT_UI_URL=https://hackbot-ui.../ +export WATCHED_REPOS=try +export DRY_RUN=true # log intended calls, don't POST +uv run --package hackbot-pulse-listener python -m app +``` + +Email is sent only when `SENDGRID_API_KEY` and `NOTIFICATION_SENDER` are set; otherwise it +is logged and skipped. + +## Test + +```bash +uv run --package hackbot-pulse-listener pytest services/hackbot-pulse-listener/tests +``` + +## Deploy + +Cloud Run worker pool (no HTTP). See `deploy.sh`. diff --git a/services/hackbot-pulse-listener/app/__init__.py b/services/hackbot-pulse-listener/app/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/hackbot-pulse-listener/app/__main__.py b/services/hackbot-pulse-listener/app/__main__.py new file mode 100644 index 0000000000..80c99d76e7 --- /dev/null +++ b/services/hackbot-pulse-listener/app/__main__.py @@ -0,0 +1,44 @@ +import logging +import signal +from concurrent.futures import ThreadPoolExecutor + +from app import consumer +from app.config import settings + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main() -> None: + if settings.sentry_dsn: + import sentry_sdk + + sentry_sdk.init(dsn=settings.sentry_dsn, environment=settings.environment) + + if not (settings.pulse_user and settings.pulse_password): + logger.warning("PULSE_USER/PULSE_PASSWORD not set; listener will not start") + return + + executor = ThreadPoolExecutor(max_workers=settings.poll_max_workers) + consumer_obj = consumer.build_consumer(executor) + + def shutdown(signum, _frame): + logger.info("Received signal %s; shutting down", signum) + consumer_obj.should_stop = True + + signal.signal(signal.SIGTERM, shutdown) + signal.signal(signal.SIGINT, shutdown) + + logger.info( + "Listening for build failures on %s; watched repos: %s", + ", ".join(consumer.EXCHANGES), + sorted(settings.watched_repos_set), + ) + try: + consumer_obj.run() + finally: + executor.shutdown(wait=False) + + +if __name__ == "__main__": + main() diff --git a/services/hackbot-pulse-listener/app/client.py b/services/hackbot-pulse-listener/app/client.py new file mode 100644 index 0000000000..a83e6ce974 --- /dev/null +++ b/services/hackbot-pulse-listener/app/client.py @@ -0,0 +1,32 @@ +import logging + +import httpx + +from app.config import settings + +logger = logging.getLogger(__name__) + +_TIMEOUT = httpx.Timeout(30.0) + + +def _headers() -> dict[str, str]: + return {"X-API-Key": settings.hackbot_api_key} + + +def trigger_run(inputs: dict) -> str | None: + """Create a build-repair run. Returns the run id, or None in dry-run mode.""" + if settings.dry_run: + logger.info("[dry-run] would trigger %s run: %s", settings.agent_name, inputs) + return None + + url = f"{settings.hackbot_api_url}/agents/{settings.agent_name}/runs" + resp = httpx.post(url, json=inputs, headers=_headers(), timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json()["run_id"] + + +def get_run(run_id: str) -> dict: + url = f"{settings.hackbot_api_url}/runs/{run_id}" + resp = httpx.get(url, headers=_headers(), timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json() diff --git a/services/hackbot-pulse-listener/app/config.py b/services/hackbot-pulse-listener/app/config.py new file mode 100644 index 0000000000..49666308c8 --- /dev/null +++ b/services/hackbot-pulse-listener/app/config.py @@ -0,0 +1,51 @@ +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + # Pulse (https://pulseguardian.mozilla.org) + pulse_user: str = "" + pulse_password: str = "" + taskcluster_root_url: str = "https://firefox-ci-tc.services.mozilla.com" + + # hackbot-api + hackbot_api_url: str = "" + hackbot_api_key: str = "" + hackbot_ui_url: str = "" + agent_name: str = "build-repair" + + # Failure filtering and agent inputs. + # ``watched_repos`` is a comma-separated list of Taskcluster ``project`` tags. + watched_repos: str = "try,autoland" + run_try_push: bool = False + model: str | None = None + max_turns: int | None = None + + # Dedupe (in-memory, by git revision) + dedupe_ttl_seconds: int = 6 * 60 * 60 + dedupe_max_size: int = 4096 + + # Polling the API for run completion + poll_interval_seconds: int = 60 + run_max_age_minutes: int = 12 * 60 + poll_max_workers: int = 8 + + # Email notifications (SendGrid) + sendgrid_api_key: str | None = None + notification_sender: str | None = None + + dry_run: bool = False + environment: str = "development" + sentry_dsn: str | None = None + + model_config = { + "env_file": ".env", + "env_file_encoding": "utf-8", + "extra": "ignore", + } + + @property + def watched_repos_set(self) -> set[str]: + return {r.strip() for r in self.watched_repos.split(",") if r.strip()} + + +settings = Settings() diff --git a/services/hackbot-pulse-listener/app/consumer.py b/services/hackbot-pulse-listener/app/consumer.py new file mode 100644 index 0000000000..a44599768f --- /dev/null +++ b/services/hackbot-pulse-listener/app/consumer.py @@ -0,0 +1,118 @@ +import logging +from concurrent.futures import Executor + +from cachetools import TTLCache +from kombu import Connection, Exchange, Queue +from kombu.mixins import ConsumerMixin + +from app import client, taskcluster, worker +from app.config import settings + +logger = logging.getLogger(__name__) + +CONNECTION_URL = "amqp://{}:{}@pulse.mozilla.org:5671/?ssl=1" + +EXCHANGES = ("exchange/taskcluster-queue/v1/task-failed",) + +# In-memory dedupe of git revisions already handed to the agent. Only the +# single consumer thread touches it, so no lock is needed. +_seen: TTLCache = TTLCache( + maxsize=settings.dedupe_max_size, ttl=settings.dedupe_ttl_seconds +) + + +def process(body: dict, executor: Executor) -> str | None: + """Handle one Taskcluster failure message. Returns the triggered run id.""" + tags = (body.get("task") or {}).get("tags") or {} + + if tags.get("kind") != "build": + return None + + project = tags.get("project") + if project not in settings.watched_repos_set: + return None + + task_id = body["status"]["taskId"] + task_name = tags.get("label") or task_id + developer_email = tags.get("createdForUser") + + revision = taskcluster.get_revision(task_id) + if not revision: + logger.warning("No GECKO_HEAD_REV for task %s; skipping", task_id) + return None + + if revision in _seen: + logger.info("Revision %s already processed; skipping", revision) + return None + _seen[revision] = True + + inputs: dict = { + "git_commit": revision, + "failure_tasks": {task_name: task_id}, + "run_try_push": settings.run_try_push, + } + if settings.model: + inputs["model"] = settings.model + if settings.max_turns is not None: + inputs["max_turns"] = settings.max_turns + + try: + run_id = client.trigger_run(inputs) + except Exception: + logger.exception("Failed to trigger build-repair run for %s", revision) + _seen.pop(revision, None) + return None + + logger.info("Triggered build-repair run %s for %s@%s", run_id, project, revision) + if run_id is not None: + executor.submit( + worker.poll_and_notify, run_id, revision, project, developer_email + ) + return run_id + + +def make_handler(executor: Executor): + def on_message(body, message): + try: + process(body, executor) + except Exception: + logger.exception("Error handling pulse message") + finally: + message.ack() + + return on_message + + +def _build_queues(user: str) -> list[Queue]: + queues = [] + for exchange in EXCHANGES: + suffix = exchange.rsplit("/", 1)[-1] + queues.append( + Queue( + name=f"queue/{user}/build-repair-{suffix}", + exchange=Exchange(exchange, type="topic", no_declare=True), + routing_key="#", + durable=True, + auto_delete=True, + ) + ) + return queues + + +class BuildFailureConsumer(ConsumerMixin): + def __init__(self, connection, queues, on_message): + self.connection = connection + self.queues = queues + self.on_message = on_message + + def get_consumers(self, Consumer, channel): + return [Consumer(queues=self.queues, callbacks=[self.on_message])] + + +def build_consumer(executor: Executor) -> BuildFailureConsumer: + connection = Connection( + CONNECTION_URL.format(settings.pulse_user, settings.pulse_password) + ) + return BuildFailureConsumer( + connection, _build_queues(settings.pulse_user), make_handler(executor) + ) diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py new file mode 100644 index 0000000000..459fae3d95 --- /dev/null +++ b/services/hackbot-pulse-listener/app/notify.py @@ -0,0 +1,74 @@ +import logging + +from app.config import settings + +logger = logging.getLogger(__name__) + + +def send_email( + developer_email: str | None, + revision: str, + repo: str, + run_id: str, + run_doc: dict, +) -> None: + """Email the developer the run outcome. No-op (logged) if not configured.""" + if not developer_email: + logger.info("No developer email for run %s; skipping notification", run_id) + return + if not (settings.sendgrid_api_key and settings.notification_sender): + logger.info("SendGrid not configured; skipping email for run %s", run_id) + return + + import sendgrid + from sendgrid.helpers.mail import Content, From, Mail, Subject, To + + status = run_doc.get("status", "unknown") + subject = f"[build-repair] {status} for {repo}@{revision[:12]}" + + sg = sendgrid.SendGridAPIClient(api_key=settings.sendgrid_api_key) + message = Mail( + From(settings.notification_sender), + To(developer_email), + Subject(subject), + Content("text/plain", _build_body(revision, repo, run_id, run_doc)), + ) + response = sg.send(message=message) + logger.info( + "Sent build-repair notification to %s (status %s)", + developer_email, + response.status_code, + ) + + +def _build_body(revision: str, repo: str, run_id: str, run_doc: dict) -> str: + status = run_doc.get("status", "unknown") + lines = [ + f"The build-repair agent finished with status: {status}.", + "", + f"Repository: {repo}", + f"Revision: {revision}", + ] + + if settings.hackbot_ui_url: + lines += [ + "", + f"Run details: {settings.hackbot_ui_url.rstrip('/')}/runs/{run_id}", + ] + + summary = run_doc.get("summary") or {} + findings = summary.get("findings") or {} + if findings.get("summary"): + lines += ["", "Summary:", findings["summary"]] + if findings.get("analysis"): + lines += ["", "Analysis:", findings["analysis"]] + if findings.get("local_build_verified") is not None: + lines += ["", f"Local build verified: {findings['local_build_verified']}"] + if findings.get("treeherder_url"): + lines += [f"Try push: {findings['treeherder_url']}"] + + error = run_doc.get("error") or summary.get("error") + if status != "succeeded" and error: + lines += ["", f"Error: {error}"] + + return "\n".join(lines) diff --git a/services/hackbot-pulse-listener/app/taskcluster.py b/services/hackbot-pulse-listener/app/taskcluster.py new file mode 100644 index 0000000000..13c5827962 --- /dev/null +++ b/services/hackbot-pulse-listener/app/taskcluster.py @@ -0,0 +1,26 @@ +import logging + +import taskcluster + +from app.config import settings + +logger = logging.getLogger(__name__) + +_queue: taskcluster.Queue | None = None + + +def _get_queue() -> taskcluster.Queue: + global _queue + if _queue is None: + _queue = taskcluster.Queue({"rootUrl": settings.taskcluster_root_url}) + return _queue + + +def get_revision(task_id: str) -> str | None: + """Return the GECKO_HEAD_REV (git SHA) for a task, or None if unavailable. + + The revision is not in the pulse message, so we fetch the full task + definition. Task definitions are public, so no credentials are needed. + """ + task = _get_queue().task(task_id) + return task.get("payload", {}).get("env", {}).get("GECKO_HEAD_REV") diff --git a/services/hackbot-pulse-listener/app/worker.py b/services/hackbot-pulse-listener/app/worker.py new file mode 100644 index 0000000000..4f733968b8 --- /dev/null +++ b/services/hackbot-pulse-listener/app/worker.py @@ -0,0 +1,47 @@ +import logging +import time + +from app import client, notify +from app.config import settings + +logger = logging.getLogger(__name__) + +TERMINAL_STATUSES = {"succeeded", "failed", "timed_out"} + + +def poll_and_notify( + run_id: str, revision: str, repo: str, developer_email: str | None +) -> None: + """Poll the run until terminal, then notify the developer. + + Runs on a background executor thread; never lets an exception escape. + """ + try: + run_doc = _poll_until_terminal(run_id) + except Exception: + logger.exception("Polling failed for run %s", run_id) + return + + if run_doc is None: + logger.warning( + "Run %s did not finish within %s minutes; giving up", + run_id, + settings.run_max_age_minutes, + ) + return + + try: + notify.send_email(developer_email, revision, repo, run_id, run_doc) + except Exception: + logger.exception("Failed to send notification for run %s", run_id) + + +def _poll_until_terminal(run_id: str) -> dict | None: + deadline = time.monotonic() + settings.run_max_age_minutes * 60 + while True: + run_doc = client.get_run(run_id) + if run_doc.get("status") in TERMINAL_STATUSES: + return run_doc + if time.monotonic() >= deadline: + return None + time.sleep(settings.poll_interval_seconds) diff --git a/services/hackbot-pulse-listener/deploy.sh b/services/hackbot-pulse-listener/deploy.sh new file mode 100755 index 0000000000..4ac3a237a7 --- /dev/null +++ b/services/hackbot-pulse-listener/deploy.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# +# Deploy the build-failure Pulse listener to Cloud Run as a worker pool. +# +# A worker pool runs an always-on, non-request workload (no HTTP port). This +# service consumes Taskcluster build-failure pulse messages and triggers the +# build-repair hackbot agent via the hackbot-api. +# +# Prereqs (one-time): +# gcloud auth login +# gcloud config set project +# gcloud services enable run.googleapis.com artifactregistry.googleapis.com \ +# cloudbuild.googleapis.com secretmanager.googleapis.com +# +# Secrets (one-time) — store in Secret Manager: +# printf '%s' '' | gcloud secrets create pulse-password --data-file=- +# printf '%s' '' | gcloud secrets create sendgrid-api-key --data-file=- +# # HACKBOT_API_KEY reuses the existing shared `external-api-key` secret. +# +# Usage: +# PROJECT=my-proj REGION=us-central1 \ +# HACKBOT_API_URL=https://hackbot-api-xxxx.run.app \ +# HACKBOT_UI_URL=https://hackbot-ui-xxxx.run.app \ +# PULSE_USER=my-pulse-user NOTIFICATION_SENDER=build-repair@mozilla.com \ +# ./deploy.sh +set -euo pipefail + +PROJECT="${PROJECT:?set PROJECT to your GCP project id}" +REGION="${REGION:-us-central1}" +SERVICE="${SERVICE:-hackbot-pulse-listener}" +REPO="${REPO:-hackbot}" +HACKBOT_API_URL="${HACKBOT_API_URL:?set HACKBOT_API_URL to the hackbot-api base URL}" +HACKBOT_UI_URL="${HACKBOT_UI_URL:?set HACKBOT_UI_URL to the hackbot-ui base URL}" +PULSE_USER="${PULSE_USER:?set PULSE_USER (https://pulseguardian.mozilla.org)}" +WATCHED_REPOS="${WATCHED_REPOS:-try}" +NOTIFICATION_SENDER="${NOTIFICATION_SENDER:?set NOTIFICATION_SENDER (from address)}" + +SA_NAME="${SA_NAME:-hackbot-pulse-listener-run}" +SA_EMAIL="${SA_EMAIL:-${SA_NAME}@${PROJECT}.iam.gserviceaccount.com}" + +PULSE_SECRET="${PULSE_SECRET:-pulse-password}" +API_KEY_SECRET="${API_KEY_SECRET:-external-api-key}" +SENDGRID_SECRET="${SENDGRID_SECRET:-sendgrid-api-key}" + +IMAGE="${REGION}-docker.pkg.dev/${PROJECT}/${REPO}/${SERVICE}:latest" +# Build context is the repo root (the Dockerfile needs the workspace lock files). +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +echo "==> Ensuring runtime service account '${SA_EMAIL}' exists" +gcloud iam service-accounts describe "${SA_EMAIL}" >/dev/null 2>&1 || \ + gcloud iam service-accounts create "${SA_NAME}" \ + --display-name="Hackbot Pulse Listener (Cloud Run runtime)" + +echo "==> Granting the SA read access to its secrets" +for s in "${PULSE_SECRET}" "${API_KEY_SECRET}" "${SENDGRID_SECRET}"; do + gcloud secrets add-iam-policy-binding "$s" \ + --member="serviceAccount:${SA_EMAIL}" \ + --role=roles/secretmanager.secretAccessor >/dev/null +done + +echo "==> Ensuring Artifact Registry repo '${REPO}' exists in ${REGION}" +gcloud artifacts repositories describe "${REPO}" --location="${REGION}" >/dev/null 2>&1 || \ + gcloud artifacts repositories create "${REPO}" \ + --repository-format=docker --location="${REGION}" \ + --description="Hackbot container images" + +echo "==> Building & pushing image with Cloud Build: ${IMAGE}" +gcloud builds submit "${ROOT_DIR}" \ + --config <(printf 'steps:\n- name: gcr.io/cloud-builders/docker\n args: ["build","-t","%s","-f","services/%s/Dockerfile","."]\nimages: ["%s"]\n' "${IMAGE}" "${SERVICE}" "${IMAGE}") + +echo "==> Deploying worker pool" +ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL},HACKBOT_UI_URL=${HACKBOT_UI_URL}" +ENV_VARS="${ENV_VARS},PULSE_USER=${PULSE_USER},WATCHED_REPOS=${WATCHED_REPOS}" +ENV_VARS="${ENV_VARS},NOTIFICATION_SENDER=${NOTIFICATION_SENDER}" + +gcloud run worker-pools deploy "${SERVICE}" \ + --image "${IMAGE}" \ + --region "${REGION}" \ + --min-instances 1 \ + --max-instances 1 \ + --service-account "${SA_EMAIL}" \ + --set-env-vars "${ENV_VARS}" \ + --set-secrets "PULSE_PASSWORD=${PULSE_SECRET}:latest,HACKBOT_API_KEY=${API_KEY_SECRET}:latest,SENDGRID_API_KEY=${SENDGRID_SECRET}:latest" + +echo "==> Deployed worker pool '${SERVICE}'" diff --git a/services/hackbot-pulse-listener/docker-compose.dev.yml b/services/hackbot-pulse-listener/docker-compose.dev.yml new file mode 100644 index 0000000000..441d03e7b1 --- /dev/null +++ b/services/hackbot-pulse-listener/docker-compose.dev.yml @@ -0,0 +1,11 @@ +services: + hackbot-pulse-listener: + build: + context: ../.. + dockerfile: services/hackbot-pulse-listener/Dockerfile + # Inject config into the container (relative to this compose file = repo root .env). + env_file: ../../.env + # Live-edit the source without rebuilding (cwd takes precedence on sys.path). + volumes: + - ./app:/app/app + command: python -m app diff --git a/services/hackbot-pulse-listener/pyproject.toml b/services/hackbot-pulse-listener/pyproject.toml new file mode 100644 index 0000000000..15b114e329 --- /dev/null +++ b/services/hackbot-pulse-listener/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "hackbot-pulse-listener" +version = "0.1.0" +description = "Listens to Taskcluster build failures and triggers the build-repair hackbot agent" +requires-python = ">=3.12" +dependencies = [ + "kombu>=5.6,<6", + "taskcluster>=97.1,<100.4", + "httpx>=0.26.0", + "pydantic-settings>=2.1.0", + "sendgrid>=6.12.5", + "cachetools>=5.3.0", + "sentry-sdk>=2.51.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["app"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/services/hackbot-pulse-listener/tests/__init__.py b/services/hackbot-pulse-listener/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json b/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json new file mode 100644 index 0000000000..3813b8025a --- /dev/null +++ b/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json @@ -0,0 +1,892 @@ +[ + { + "payload": { + "status": { + "taskId": "QKbYKE7STVy3_DM6J9NKXw", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1400-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1400-r8", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "DJziYijSR9qv-cGTDdS92w", + "priority": "very-low", + "deadline": "2026-06-26T21:22:49.913Z", + "expires": "2026-07-23T21:22:49.913Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-381", + "takenUntil": "2026-06-25T22:47:29.995Z", + "scheduled": "2026-06-25T21:36:27.697Z", + "started": "2026-06-25T21:36:28.598Z", + "resolved": "2026-06-25T22:35:05.200Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "web-platform-tests", + "label": "test-macosx1470-64/opt-web-platform-tests-webgpu-backlog-long-1", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "trust-domain": "gecko", + "test-platform": "macosx1470-64/opt", + "tests_grouped": "1", + "createdForUser": "aleiserson@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-381", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.QKbYKE7STVy3_DM6J9NKXw.0.mdc1.macmini-r8-381.releng-hardware.gecko-t-osx-1400-r8.gecko-level-1.DJziYijSR9qv-cGTDdS92w._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "TISFEWXiRD-Fg4ifDcLm5A", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1015-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1015-r8", + "schedulerId": "comm-level-1", + "projectId": "none", + "taskGroupId": "VuZl_WA9QKq3m4U7zIVcUA", + "priority": "low", + "deadline": "2026-06-26T13:31:23.330Z", + "expires": "2026-07-23T13:31:23.330Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-209", + "takenUntil": "2026-06-25T23:27:40.611Z", + "scheduled": "2026-06-25T13:39:15.145Z", + "started": "2026-06-25T23:07:40.613Z", + "resolved": "2026-06-25T23:17:25.349Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "test", + "label": "test-macosx1015-64-qr/debug-mochitest-thunderbird-2", + "project": "try-comm-central", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome-thunderbird", + "trust-domain": "comm", + "test-platform": "macosx1015-64-qr/debug", + "tests_grouped": "1", + "createdForUser": "h.w.forms@arcor.de", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-209", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.TISFEWXiRD-Fg4ifDcLm5A.0.mdc1.macmini-r8-209.releng-hardware.gecko-t-osx-1015-r8.comm-level-1.VuZl_WA9QKq3m4U7zIVcUA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "TWIXsqxDTame750yV3a_0A", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.935Z", + "expires": "2026-07-23T22:29:42.935Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-east1-c", + "workerId": "4411950482911664357", + "takenUntil": "2026-06-25T23:21:38.581Z", + "scheduled": "2026-06-25T22:43:40.477Z", + "started": "2026-06-25T22:44:38.441Z", + "resolved": "2026-06-25T23:17:28.610Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/opt-web-platform-tests-async-9", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/opt", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-east1-c", + "workerId": "4411950482911664357", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.TWIXsqxDTame750yV3a_0A.0.us-east1-c.4411950482911664357.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "CLwxQmXLQFunTzlzrzshOg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.160Z", + "expires": "2027-06-25T20:21:36.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma", + "takenUntil": "2026-06-25T23:30:14.189Z", + "scheduled": "2026-06-25T22:18:46.436Z", + "started": "2026-06-25T22:19:13.295Z", + "resolved": "2026-06-25T23:19:31.591Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-8", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.CLwxQmXLQFunTzlzrzshOg.0.southindia.vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "d4YZWZQXR-SdCVwlrVTTUA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.156Z", + "expires": "2027-06-25T20:21:36.156Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09", + "takenUntil": "2026-06-25T23:30:22.934Z", + "scheduled": "2026-06-25T22:18:46.462Z", + "started": "2026-06-25T22:19:22.411Z", + "resolved": "2026-06-25T23:19:37.633Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-2", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.d4YZWZQXR-SdCVwlrVTTUA.0.southindia.vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "EJXnn6xsQ4OeDMK3W1aksg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.160Z", + "expires": "2027-06-25T20:21:36.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "westus3", + "workerId": "vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf", + "takenUntil": "2026-06-25T23:30:32.889Z", + "scheduled": "2026-06-25T22:18:46.487Z", + "started": "2026-06-25T22:19:32.554Z", + "resolved": "2026-06-25T23:19:38.966Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-7", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "westus3", + "workerId": "vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.EJXnn6xsQ4OeDMK3W1aksg.0.westus3.vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "cnTkiN1YROGfMWecDxyMEQ", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.413Z", + "expires": "2026-07-23T22:29:42.413Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-west1-b", + "workerId": "525836151908471198", + "takenUntil": "2026-06-25T23:39:39.063Z", + "scheduled": "2026-06-25T22:44:50.986Z", + "started": "2026-06-25T22:45:38.942Z", + "resolved": "2026-06-25T23:19:39.842Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/debug-web-platform-tests-async-12", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/debug", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-west1-b", + "workerId": "525836151908471198", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.cnTkiN1YROGfMWecDxyMEQ.0.us-west1-b.525836151908471198.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "DPltHwhPTI29TinNU_XBbw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.472Z", + "expires": "2027-06-25T20:22:15.472Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "canadacentral", + "workerId": "vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6", + "takenUntil": "2026-06-25T23:31:17.350Z", + "scheduled": "2026-06-25T22:19:36.851Z", + "started": "2026-06-25T22:20:16.923Z", + "resolved": "2026-06-25T23:20:25.989Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-7", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "canadacentral", + "workerId": "vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.DPltHwhPTI29TinNU_XBbw.0.canadacentral.vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "DEt854d0QAOgI-OZsHoMmw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.468Z", + "expires": "2027-06-25T20:22:15.468Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg", + "takenUntil": "2026-06-25T23:31:13.812Z", + "scheduled": "2026-06-25T22:19:36.839Z", + "started": "2026-06-25T22:20:12.929Z", + "resolved": "2026-06-25T23:20:28.897Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-13", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.DEt854d0QAOgI-OZsHoMmw.0.southindia.vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "AlWS0HmFQiKPjBS1HcFAzw", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1015-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1015-r8", + "schedulerId": "comm-level-1", + "projectId": "none", + "taskGroupId": "VuZl_WA9QKq3m4U7zIVcUA", + "priority": "low", + "deadline": "2026-06-26T13:31:23.054Z", + "expires": "2026-07-23T13:31:23.054Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-77", + "takenUntil": "2026-06-25T23:28:29.702Z", + "scheduled": "2026-06-25T13:42:42.448Z", + "started": "2026-06-25T23:08:29.704Z", + "resolved": "2026-06-25T23:20:29.089Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "test", + "label": "test-macosx1015-64-qr/opt-xpcshell-4", + "project": "try-comm-central", + "retrigger": "true", + "test-suite": "xpcshell", + "trust-domain": "comm", + "test-platform": "macosx1015-64-qr/opt", + "tests_grouped": "1", + "createdForUser": "h.w.forms@arcor.de", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-77", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.AlWS0HmFQiKPjBS1HcFAzw.0.mdc1.macmini-r8-77.releng-hardware.gecko-t-osx-1015-r8.comm-level-1.VuZl_WA9QKq3m4U7zIVcUA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "YO-gvnzTTGS7INXYIN8Nbg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.156Z", + "expires": "2027-06-25T20:21:36.156Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "centralindia", + "workerId": "vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0", + "takenUntil": "2026-06-25T23:31:12.293Z", + "scheduled": "2026-06-25T22:18:46.606Z", + "started": "2026-06-25T22:20:11.674Z", + "resolved": "2026-06-25T23:20:29.185Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-15", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "centralindia", + "workerId": "vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.YO-gvnzTTGS7INXYIN8Nbg.0.centralindia.vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "YV2f3tnLSCyXUIG4EDq5Xw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.153Z", + "expires": "2027-06-25T20:21:36.153Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "centralindia", + "workerId": "vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm", + "takenUntil": "2026-06-25T23:31:12.727Z", + "scheduled": "2026-06-25T22:18:46.629Z", + "started": "2026-06-25T22:20:12.243Z", + "resolved": "2026-06-25T23:20:30.629Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-4", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "centralindia", + "workerId": "vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.YV2f3tnLSCyXUIG4EDq5Xw.0.centralindia.vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "dtOy4UGdTHm2s5XCVuNkqA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.473Z", + "expires": "2027-06-25T20:22:15.473Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "northcentralus", + "workerId": "vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h", + "takenUntil": "2026-06-25T23:31:33.280Z", + "scheduled": "2026-06-25T22:19:36.866Z", + "started": "2026-06-25T22:20:32.966Z", + "resolved": "2026-06-25T23:20:41.142Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-11", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "northcentralus", + "workerId": "vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.dtOy4UGdTHm2s5XCVuNkqA.0.northcentralus.vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "fRj5qgRVS4adFANUGOexwA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.474Z", + "expires": "2027-06-25T20:22:15.474Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "westus", + "workerId": "vm-slqhmhydqni6evb1vuglbaat8icl5saifla", + "takenUntil": "2026-06-25T23:31:38.392Z", + "scheduled": "2026-06-25T22:19:36.878Z", + "started": "2026-06-25T22:20:38.071Z", + "resolved": "2026-06-25T23:20:45.381Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-5", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "westus", + "workerId": "vm-slqhmhydqni6evb1vuglbaat8icl5saifla", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.fRj5qgRVS4adFANUGOexwA.0.westus.vm-slqhmhydqni6evb1vuglbaat8icl5saifla.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "ZFZw8IBSSWKrZmgZcdpUKA", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.408Z", + "expires": "2026-07-23T22:29:42.408Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-central1-b", + "workerId": "1690275899809730974", + "takenUntil": "2026-06-25T23:39:44.587Z", + "scheduled": "2026-06-25T22:44:51.205Z", + "started": "2026-06-25T22:45:44.383Z", + "resolved": "2026-06-25T23:21:01.122Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/debug-web-platform-tests-async-1", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/debug", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-central1-b", + "workerId": "1690275899809730974", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.ZFZw8IBSSWKrZmgZcdpUKA.0.us-central1-b.1690275899809730974.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "f_BW7zM6RCi8p_Ah5i57Qg", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-amd", + "taskQueueId": "gecko-t/t-linux-docker-amd", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "cXOzgHGNRt25Mv8Aq5TvLw", + "priority": "low", + "deadline": "2026-06-26T22:51:20.160Z", + "expires": "2027-06-25T22:51:20.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-central1-b", + "workerId": "9170129125611932268", + "takenUntil": "2026-06-25T23:36:36.096Z", + "scheduled": "2026-06-25T23:16:35.979Z", + "started": "2026-06-25T23:16:36.098Z", + "resolved": "2026-06-25T23:21:06.146Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "source-test", + "label": "source-test-node-newtab-unit-tests", + "project": "autoland", + "retrigger": "true", + "trust-domain": "gecko", + "createdForUser": "dwhisman@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-central1-b", + "workerId": "9170129125611932268", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.f_BW7zM6RCi8p_Ah5i57Qg.0.us-central1-b.9170129125611932268.gecko-t.t-linux-docker-amd.gecko-level-3.cXOzgHGNRt25Mv8Aq5TvLw._", + "redelivered": false, + "cc": [] + } +] diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py new file mode 100644 index 0000000000..6a68968ec9 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -0,0 +1,100 @@ +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from app import consumer + +FIXTURES = Path(__file__).parent / "fixtures" + + +def setup_function(): + consumer._seen.clear() + + +def _sample_bodies(): + data = json.loads((FIXTURES / "pulse_messages.json").read_text()) + # The inspector wraps the real AMQP body under "payload". + return [m["payload"] for m in data] + + +def _build_msg(task_id="ABC", project="try", label="build-linux64/opt"): + return { + "status": {"taskId": task_id}, + "runId": 0, + "task": { + "tags": { + "kind": "build", + "project": project, + "label": label, + "createdForUser": "dev@mozilla.com", + } + }, + } + + +def test_sample_messages_are_all_tests_and_skipped(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision") as get_rev, + patch.object(consumer.client, "trigger_run") as trigger, + ): + for body in _sample_bodies(): + assert consumer.process(body, executor) is None + get_rev.assert_not_called() + trigger.assert_not_called() + executor.submit.assert_not_called() + + +def test_build_failure_triggers_run_and_submits_poll(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, + ): + run_id = consumer.process(_build_msg(), executor) + + assert run_id == "run-1" + trigger.assert_called_once() + inputs = trigger.call_args.args[0] + assert inputs["git_commit"] == "deadbeef" + assert inputs["failure_tasks"] == {"build-linux64/opt": "ABC"} + executor.submit.assert_called_once() + + +def test_same_revision_triggers_once(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, + ): + consumer.process(_build_msg(task_id="T1"), executor) + consumer.process(_build_msg(task_id="T2"), executor) + + trigger.assert_called_once() + + +def test_unwatched_project_skipped_before_api_call(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision") as get_rev, + patch.object(consumer.client, "trigger_run") as trigger, + ): + assert consumer.process(_build_msg(project="mozilla-central"), executor) is None + + get_rev.assert_not_called() + trigger.assert_not_called() + + +def test_trigger_failure_releases_revision_for_retry(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object( + consumer.client, "trigger_run", side_effect=[RuntimeError("boom"), "run-2"] + ) as trigger, + ): + assert consumer.process(_build_msg(task_id="T1"), executor) is None + # Same revision can be retried because the failed claim was released. + assert consumer.process(_build_msg(task_id="T2"), executor) == "run-2" + + assert trigger.call_count == 2 diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py new file mode 100644 index 0000000000..32cf831011 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_notify.py @@ -0,0 +1,62 @@ +from unittest.mock import MagicMock, patch + +from app import notify + + +def test_skips_without_developer_email(): + # Must not raise even with no SendGrid config. + notify.send_email(None, "rev", "try", "run-1", {"status": "succeeded"}) + + +def test_skips_without_sendgrid_config(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", None) + monkeypatch.setattr(notify.settings, "notification_sender", None) + notify.send_email("dev@mozilla.com", "rev", "try", "run-1", {"status": "succeeded"}) + + +def test_body_contains_ui_link_and_summary(monkeypatch): + monkeypatch.setattr(notify.settings, "hackbot_ui_url", "https://ui.example/") + body = notify._build_body( + "deadbeefcafe1234", + "try", + "run-1", + { + "status": "succeeded", + "summary": { + "findings": { + "summary": "Fixed a missing include", + "analysis": "The commit removed a needed header", + "local_build_verified": True, + } + }, + }, + ) + assert "https://ui.example/runs/run-1" in body + assert "Fixed a missing include" in body + assert "The commit removed a needed header" in body + assert "Local build verified: True" in body + + +def test_failure_body_includes_error(): + body = notify._build_body( + "deadbeef", "try", "run-1", {"status": "failed", "error": "build still broken"} + ) + assert "build still broken" in body + + +def test_sends_email_when_configured(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with patch("sendgrid.SendGridAPIClient", return_value=fake_client): + notify.send_email( + "dev@mozilla.com", + "rev", + "try", + "run-1", + {"status": "succeeded", "summary": {}}, + ) + + fake_client.send.assert_called_once() diff --git a/services/hackbot-pulse-listener/tests/test_worker.py b/services/hackbot-pulse-listener/tests/test_worker.py new file mode 100644 index 0000000000..1ef39ae448 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_worker.py @@ -0,0 +1,31 @@ +from unittest.mock import patch + +from app import worker + + +def test_terminal_run_notifies_once(): + run_doc = {"status": "succeeded", "summary": {}} + with ( + patch.object(worker.client, "get_run", return_value=run_doc) as get_run, + patch.object(worker, "notify") as notify, + ): + worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") + + get_run.assert_called_once() + notify.send_email.assert_called_once() + args = notify.send_email.call_args.args + assert args == ("dev@mozilla.com", "rev", "try", "run-1", run_doc) + + +def test_gives_up_after_max_age(monkeypatch): + monkeypatch.setattr(worker.settings, "run_max_age_minutes", 0) + with ( + patch.object( + worker.client, "get_run", return_value={"status": "running"} + ) as get_run, + patch.object(worker, "notify") as notify, + ): + worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") + + get_run.assert_called_once() + notify.send_email.assert_not_called() diff --git a/uv.lock b/uv.lock index c920ae9e15..6a2185e338 100644 --- a/uv.lock +++ b/uv.lock @@ -26,6 +26,7 @@ members = [ "hackbot-agent-bug-fix", "hackbot-agent-build-repair", "hackbot-api", + "hackbot-pulse-listener", "hackbot-runtime", "reviewhelper-api", ] @@ -2524,6 +2525,38 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "hackbot-pulse-listener" +version = "0.1.0" +source = { editable = "services/hackbot-pulse-listener" } +dependencies = [ + { name = "cachetools" }, + { name = "httpx" }, + { name = "kombu" }, + { name = "pydantic-settings" }, + { name = "sendgrid" }, + { name = "sentry-sdk" }, + { name = "taskcluster" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "cachetools", specifier = ">=5.3.0" }, + { name = "httpx", specifier = ">=0.26.0" }, + { name = "kombu", specifier = ">=5.6,<6" }, + { name = "pydantic-settings", specifier = ">=2.1.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "sendgrid", specifier = ">=6.12.5" }, + { name = "sentry-sdk", specifier = ">=2.51.0" }, + { name = "taskcluster", specifier = ">=97.1,<100.4" }, +] +provides-extras = ["dev"] + [[package]] name = "hackbot-runtime" version = "0.1.0"