From 77c913e57020d97e4c5714bb9262989f3bc81a47 Mon Sep 17 00:00:00 2001 From: spalen0 Date: Wed, 17 Jun 2026 14:51:15 +0200 Subject: [PATCH] Dispatch emergency withdrawals via signed webhook --- deploy/emergency-dispatch-demo.md | 53 ++++++++++++++++++--------- protocols/infinifi/README.md | 8 ++-- tests/conftest.py | 6 ++- tests/test_utils.py | 61 ++++++++++++++++++++++++------- utils/dispatch.py | 50 ++++++++++++++++--------- 5 files changed, 123 insertions(+), 55 deletions(-) diff --git a/deploy/emergency-dispatch-demo.md b/deploy/emergency-dispatch-demo.md index 0d71d538..c686b672 100644 --- a/deploy/emergency-dispatch-demo.md +++ b/deploy/emergency-dispatch-demo.md @@ -5,8 +5,8 @@ ``` monitoring-scripts-py liquidity-monitoring +------------------------+ +---------------------------+ -| Protocol monitor | GitHub | emergency_withdraw.yml | -| detects issue | ----------> | receives dispatch event | +| Protocol monitor | Webhook | /webhook/emergency | +| detects issue | ----------> | receives signed payload | | (HIGH or CRITICAL) | API | | +------------------------+ +---------------------------+ | @@ -23,8 +23,8 @@ monitoring-scripts-py liquidity-monitoring 1. A protocol monitor in `monitoring-scripts-py` detects an issue 2. It fires `send_alert(Alert(AlertSeverity.HIGH/CRITICAL, message, protocol))` -3. The alert hook calls `dispatch_emergency_withdrawal()`, which sends a `repository_dispatch` event to `liquidity-monitoring` -4. The `emergency_withdraw.yml` workflow picks it up and: +3. The alert hook calls `dispatch_emergency_withdrawal()`, which sends a signed JSON webhook to `liquidity-monitoring` +4. The webhook handler picks it up and: - Looks up the protocol in `emergency_config.json` to find which vaults/markets to act on - Zeros the `forced_cap` and `forced_percentage` for those markets in `forced_caps.json` @@ -53,32 +53,49 @@ monitoring-scripts-py liquidity-monitoring - **60-minute cooldown** per protocol (prevents duplicate dispatches from repeated alerts) - **DEBUG mode** skips dispatch (same as Telegram) - **DISPATCHABLE_PROTOCOLS** whitelist (only configured protocols can trigger) -- **`PAT_DISPATCH`** fine-grained token required (scoped to liquidity-monitoring repo) +- **`LIQUIDITY_WEBHOOK_SECRET`** required for `X-Hub-Signature-256` HMAC verification + +## Webhook configuration + +By default, the monitoring dispatcher sends to: + +```text +http://127.0.0.1:8080/webhook/emergency +``` + +Set `LIQUIDITY_WEBHOOK_SECRET` to the shared webhook secret. The dispatcher serializes the JSON body once, signs those exact bytes with HMAC-SHA256, and sends: + +```text +X-Hub-Signature-256: sha256= +Content-Type: application/json +``` + +Override the endpoint with `LIQUIDITY_WEBHOOK_URL` only when the webhook is not running on the default local address. ## Manual trigger script -To manually trigger an emergency withdrawal dispatch: +To manually trigger an emergency withdrawal webhook: ### CRITICAL (direct commit + immediate reallocation) ```bash -gh api repos/tapired/liquidity-monitoring/dispatches \ - -X POST \ - -f event_type=emergency_withdrawal \ - -f 'client_payload[protocol]=usdai' \ - -f 'client_payload[severity]=CRITICAL' +body='{"event_type":"emergency_withdrawal","client_payload":{"protocol":"usdai","severity":"CRITICAL","message":"manual trigger"}}' +sig="$(printf '%s' "$body" | openssl dgst -sha256 -hmac "$LIQUIDITY_WEBHOOK_SECRET" -hex | awk '{print $2}')" +curl -sS -X POST http://127.0.0.1:8080/webhook/emergency \ + -H "Content-Type: application/json" \ + -H "X-Hub-Signature-256: sha256=$sig" \ + --data-binary "$body" ``` ### HIGH (opens PR for review) ```bash -gh api repos/tapired/liquidity-monitoring/dispatches \ - -X POST \ - -f event_type=emergency_withdrawal \ - -f 'client_payload[protocol]=usdai' \ - -f 'client_payload[severity]=HIGH' +body='{"event_type":"emergency_withdrawal","client_payload":{"protocol":"usdai","severity":"HIGH","message":"manual trigger"}}' +sig="$(printf '%s' "$body" | openssl dgst -sha256 -hmac "$LIQUIDITY_WEBHOOK_SECRET" -hex | awk '{print $2}')" +curl -sS -X POST http://127.0.0.1:8080/webhook/emergency \ + -H "Content-Type: application/json" \ + -H "X-Hub-Signature-256: sha256=$sig" \ + --data-binary "$body" ``` Replace `usdai` with any protocol from the list above. - -A **204 (no output)** response means the dispatch was sent successfully. Check the [Actions tab](https://github.com/tapired/liquidity-monitoring/actions) to see the workflow run. diff --git a/protocols/infinifi/README.md b/protocols/infinifi/README.md index c497c61f..778c1767 100644 --- a/protocols/infinifi/README.md +++ b/protocols/infinifi/README.md @@ -30,15 +30,17 @@ It compares cached `totalSupply` deltas and alerts when the increase is above: ### Emergency dispatch -HIGH and CRITICAL alerts automatically trigger a `repository_dispatch` to +HIGH and CRITICAL alerts automatically trigger a signed webhook to [liquidity-monitoring](https://github.com/tapired/liquidity-monitoring) to zero Morpho market caps for siUSD collateral: - **CRITICAL** — caps are zeroed and reallocation runs immediately - **HIGH** — a PR is opened with zeroed caps for team review; after merging, trigger reallocation manually -Dispatch is rate-limited to once per 60 minutes per protocol. See -`utils/dispatch.py` and `liquidity-monitoring/hooks.md` for details. +Dispatch is rate-limited to once per 60 minutes per protocol. The dispatcher +sends the exact JSON body to `http://127.0.0.1:8080/webhook/emergency` with +`X-Hub-Signature-256: sha256=` using `LIQUIDITY_WEBHOOK_SECRET`. See +`utils/dispatch.py` for details. ### Alerts disabled ⚠️ diff --git a/tests/conftest.py b/tests/conftest.py index fd896cf4..7febd910 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ def _isolate_from_live_apis(monkeypatch: pytest.MonkeyPatch) -> None: """Block accidental live API/RPC calls and reset cross-test singletons. Strips `ETHERSCAN_TOKEN`, every `PROVIDER_URL_*`, every `TELEGRAM_*` - credential, and `PAT_DISPATCH` so a missing mock short-circuits cheaply via + credential, and emergency webhook credentials so a missing mock short-circuits cheaply via the "no token / no provider / no credentials" code paths that already exist for production use. Forces `LOG_LEVEL=INFO` so a developer's `.env` `LOG_LEVEL=DEBUG` (which skips Telegram sends) can't change tested behavior. @@ -30,7 +30,9 @@ def _isolate_from_live_apis(monkeypatch: pytest.MonkeyPatch) -> None: one test can't leak into the next. """ for key in list(os.environ): - if key in {"ETHERSCAN_TOKEN", "PAT_DISPATCH"} or key.startswith(("PROVIDER_URL_", "TELEGRAM_")): + if key in {"ETHERSCAN_TOKEN", "LIQUIDITY_WEBHOOK_SECRET"} or key.startswith( + ("PROVIDER_URL_", "TELEGRAM_", "LIQUIDITY_WEBHOOK_") + ): monkeypatch.delenv(key, raising=False) monkeypatch.setenv("LOG_LEVEL", "INFO") try: diff --git a/tests/test_utils.py b/tests/test_utils.py index 116bab57..851a6db2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,9 @@ """Tests for utility functions.""" +import hashlib +import hmac import importlib +import json import os import sys import types @@ -590,12 +593,17 @@ def test_dispatch_sends_correct_payload(self, mock_cooldown, mock_record, mock_p alert = Alert(severity=AlertSeverity.HIGH, message="Reserves low", protocol="infinifi") - with patch.dict(os.environ, {"PAT_DISPATCH": "ghp_test_token", "LOG_LEVEL": "INFO"}): + with patch.dict(os.environ, {"LIQUIDITY_WEBHOOK_SECRET": "test_secret", "LOG_LEVEL": "INFO"}): dispatch_emergency_withdrawal(alert) mock_post.assert_called_once() + call_args = mock_post.call_args[0] call_kwargs = mock_post.call_args[1] - payload = call_kwargs["json"] + self.assertEqual(call_args[0], "http://127.0.0.1:8080/webhook/emergency") + body = call_kwargs["data"] + self.assertIsInstance(body, bytes) + self.assertNotIn("json", call_kwargs) + payload = json.loads(body.decode("utf-8")) self.assertEqual(payload["event_type"], "emergency_withdrawal") self.assertEqual(payload["client_payload"]["protocol"], "infinifi") self.assertEqual(payload["client_payload"]["severity"], "HIGH") @@ -603,12 +611,37 @@ def test_dispatch_sends_correct_payload(self, mock_cooldown, mock_record, mock_p # Payload should only contain protocol, severity, and message (no markets/vault/chain) self.assertEqual(set(payload["client_payload"].keys()), {"protocol", "severity", "message"}) - # Verify auth header headers = call_kwargs["headers"] - self.assertEqual(headers["Authorization"], "Bearer ghp_test_token") + expected_hmac = hmac.new(b"test_secret", body, hashlib.sha256).hexdigest() + self.assertEqual(headers["X-Hub-Signature-256"], f"sha256={expected_hmac}") + self.assertEqual(headers["Content-Type"], "application/json") mock_record.assert_called_once_with("infinifi") + @patch("utils.dispatch.requests.post") + @patch("utils.dispatch._record_dispatch") + @patch("utils.dispatch._is_on_cooldown", return_value=False) + def test_dispatch_uses_configured_webhook_url(self, mock_cooldown, mock_record, mock_post): + from utils.dispatch import dispatch_emergency_withdrawal + + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_post.return_value = mock_response + + alert = Alert(severity=AlertSeverity.HIGH, message="Reserves low", protocol="infinifi") + + with patch.dict( + os.environ, + { + "LIQUIDITY_WEBHOOK_SECRET": "test_secret", + "LIQUIDITY_WEBHOOK_URL": "http://localhost:9000/webhook/emergency", + "LOG_LEVEL": "INFO", + }, + ): + dispatch_emergency_withdrawal(alert) + + self.assertEqual(mock_post.call_args[0][0], "http://localhost:9000/webhook/emergency") + @patch("utils.dispatch.requests.post") def test_dispatch_skips_low_severity(self, mock_post): from utils.dispatch import dispatch_emergency_withdrawal @@ -632,7 +665,7 @@ def test_dispatch_skips_unknown_protocol(self, mock_cooldown, mock_post): alert = Alert(severity=AlertSeverity.HIGH, message="alert", protocol="unknown_protocol") - with patch.dict(os.environ, {"PAT_DISPATCH": "ghp_test_token"}): + with patch.dict(os.environ, {"LIQUIDITY_WEBHOOK_SECRET": "test_secret"}): dispatch_emergency_withdrawal(alert) mock_post.assert_not_called() @@ -644,14 +677,14 @@ def test_dispatch_skips_on_cooldown(self, mock_cooldown, mock_post): alert = Alert(severity=AlertSeverity.HIGH, message="alert", protocol="infinifi") - with patch.dict(os.environ, {"PAT_DISPATCH": "ghp_test_token"}): + with patch.dict(os.environ, {"LIQUIDITY_WEBHOOK_SECRET": "test_secret"}): dispatch_emergency_withdrawal(alert) mock_post.assert_not_called() @patch("utils.dispatch.requests.post") @patch("utils.dispatch._is_on_cooldown", return_value=False) - def test_dispatch_skips_missing_pat(self, mock_cooldown, mock_post): + def test_dispatch_skips_missing_webhook_secret(self, mock_cooldown, mock_post): from utils.dispatch import dispatch_emergency_withdrawal alert = Alert(severity=AlertSeverity.HIGH, message="alert", protocol="infinifi") @@ -673,10 +706,10 @@ def test_dispatch_critical_sends_critical_severity(self, mock_cooldown, mock_rec alert = Alert(severity=AlertSeverity.CRITICAL, message="total failure", protocol="infinifi") - with patch.dict(os.environ, {"PAT_DISPATCH": "ghp_test_token", "LOG_LEVEL": "INFO"}): + with patch.dict(os.environ, {"LIQUIDITY_WEBHOOK_SECRET": "test_secret", "LOG_LEVEL": "INFO"}): dispatch_emergency_withdrawal(alert) - payload = mock_post.call_args[1]["json"] + payload = json.loads(mock_post.call_args[1]["data"].decode("utf-8")) self.assertEqual(payload["client_payload"]["severity"], "CRITICAL") @patch("utils.dispatch.requests.post") @@ -689,7 +722,7 @@ def test_dispatch_handles_request_exception(self, mock_cooldown, mock_record, mo alert = Alert(severity=AlertSeverity.HIGH, message="alert", protocol="infinifi") - with patch.dict(os.environ, {"PAT_DISPATCH": "ghp_test_token", "LOG_LEVEL": "INFO"}): + with patch.dict(os.environ, {"LIQUIDITY_WEBHOOK_SECRET": "test_secret", "LOG_LEVEL": "INFO"}): # Should not raise dispatch_emergency_withdrawal(alert) @@ -708,10 +741,10 @@ def test_dispatch_uses_protocol_not_channel(self, mock_cooldown, mock_record, mo alert = Alert(severity=AlertSeverity.HIGH, message="redeem value dropped", protocol="origin", channel="pegs") - with patch.dict(os.environ, {"PAT_DISPATCH": "ghp_test_token", "LOG_LEVEL": "INFO"}): + with patch.dict(os.environ, {"LIQUIDITY_WEBHOOK_SECRET": "test_secret", "LOG_LEVEL": "INFO"}): dispatch_emergency_withdrawal(alert) - payload = mock_post.call_args[1]["json"] + payload = json.loads(mock_post.call_args[1]["data"].decode("utf-8")) self.assertEqual(payload["client_payload"]["protocol"], "origin") mock_record.assert_called_once_with("origin") @@ -723,7 +756,7 @@ def test_dispatch_skips_non_dispatchable_channel_protocol(self, mock_cooldown, m alert = Alert(severity=AlertSeverity.HIGH, message="peg alert", protocol="puffer", channel="pegs") - with patch.dict(os.environ, {"PAT_DISPATCH": "ghp_test_token"}): + with patch.dict(os.environ, {"LIQUIDITY_WEBHOOK_SECRET": "test_secret"}): dispatch_emergency_withdrawal(alert) mock_post.assert_not_called() @@ -734,7 +767,7 @@ def test_dispatch_skips_in_debug_mode(self, mock_post): alert = Alert(severity=AlertSeverity.HIGH, message="alert", protocol="infinifi") - with patch.dict(os.environ, {"PAT_DISPATCH": "ghp_test_token", "LOG_LEVEL": "DEBUG"}): + with patch.dict(os.environ, {"LIQUIDITY_WEBHOOK_SECRET": "test_secret", "LOG_LEVEL": "DEBUG"}): dispatch_emergency_withdrawal(alert) mock_post.assert_not_called() diff --git a/utils/dispatch.py b/utils/dispatch.py index bf414b3f..a7da80c1 100644 --- a/utils/dispatch.py +++ b/utils/dispatch.py @@ -1,14 +1,16 @@ -"""Dispatch emergency withdrawal requests to the liquidity-monitoring repo via GitHub API. +"""Dispatch emergency withdrawal requests to the liquidity-monitoring webhook. -When a HIGH or CRITICAL alert fires, this module sends a ``repository_dispatch`` -event to ``tapired/liquidity-monitoring`` with the protocol name and severity. -The receiving repo resolves which vaults/markets to act on from its own config +When a HIGH or CRITICAL alert fires, this module sends a signed webhook request +to the liquidity-monitoring service with the protocol name and severity. The +receiving service resolves which vaults/markets to act on from its own config (``emergency_config.json``, ``markets_config.py``, ``forced_caps.json``). -Requires the ``PAT_DISPATCH`` environment variable (fine-grained PAT -with ``actions:write`` on the target repo). +Requires the ``LIQUIDITY_WEBHOOK_SECRET`` environment variable. """ +import hashlib +import hmac +import json import os import time @@ -20,8 +22,9 @@ logger = get_logger("utils.dispatch") -TARGET_REPO = "tapired/liquidity-monitoring" -DISPATCH_URL = f"https://api.github.com/repos/{TARGET_REPO}/dispatches" +DEFAULT_WEBHOOK_URL = "http://127.0.0.1:8080/webhook/emergency" +WEBHOOK_URL_ENV = "LIQUIDITY_WEBHOOK_URL" +WEBHOOK_SECRET_ENV = "LIQUIDITY_WEBHOOK_SECRET" DEFAULT_COOLDOWN_SECONDS = 3600 # 60 minutes # Protocols that have emergency withdrawal config in liquidity-monitoring. @@ -47,6 +50,16 @@ def _record_dispatch(protocol: str) -> None: write_last_value_to_file(cache_filename, cache_key, time.time()) +def _serialize_payload(payload: dict) -> bytes: + """Serialize once so the signed bytes are exactly the bytes sent.""" + return json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _signature_header(secret: str, body: bytes) -> str: + digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + return f"sha256={digest}" + + def dispatch_emergency_withdrawal(alert: Alert) -> None: """Dispatch an emergency withdrawal to liquidity-monitoring. @@ -57,8 +70,8 @@ def dispatch_emergency_withdrawal(alert: Alert) -> None: ``DISPATCHABLE_PROTOCOLS``. Respects a per-protocol cooldown to avoid duplicate dispatches from repeated alerts. - The receiving workflow resolves vaults, markets, and chains from its - own ``emergency_config.json``. + The receiving webhook resolves vaults, markets, and chains from its own + ``emergency_config.json``. Args: alert: The alert that triggered the hook. @@ -78,9 +91,9 @@ def dispatch_emergency_withdrawal(alert: Alert) -> None: logger.info("Dispatch for %s is on cooldown, skipping", alert.protocol) return - token = os.getenv("PAT_DISPATCH") - if not token: - logger.warning("PAT_DISPATCH not set, cannot dispatch emergency withdrawal") + secret = os.getenv(WEBHOOK_SECRET_ENV) + if not secret: + logger.warning("%s not set, cannot dispatch emergency withdrawal", WEBHOOK_SECRET_ENV) return payload = { @@ -91,15 +104,16 @@ def dispatch_emergency_withdrawal(alert: Alert) -> None: "message": alert.message, }, } + body = _serialize_payload(payload) + webhook_url = os.getenv(WEBHOOK_URL_ENV, DEFAULT_WEBHOOK_URL) try: response = requests.post( - DISPATCH_URL, - json=payload, + webhook_url, + data=body, headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + "X-Hub-Signature-256": _signature_header(secret, body), }, timeout=10, )