From 40668834b0335c7fca73e5aef404d000e0065488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 19 Aug 2026 16:55:22 +0200 Subject: [PATCH 1/3] Remove monitoring of old test helpers --- ansible/roles/prometheus/vars/main.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index 1da7f053..26da57a2 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -1,8 +1,6 @@ dom0_hosts: - ams-slack-1.ooni.org - doams1-countly.ooni.nu - - mia-echoth.ooni.nu - - mia-httpth.ooni.nu blackbox_jobs: # TODO add these records to the ALB config @@ -48,18 +46,6 @@ blackbox_jobs: # cloudfront - "https://d3kr4emv7f56qa.cloudfront.net/bouncer/net-tests" - # IP addresses are used for test-helpers in monitoring configuration for some - # historical reason hopefully remembered by @hellais. - - name: "ooni tcp echo" - module: "ooni_tcp_echo_ok" - targets: - - "{{ lookup('dig', 'c.echo.th.ooni.io/A') }}:80" - - - name: "ooni http return json headers" - module: "ooni_http_return_json_headers_ok" - targets: - - "http://{{ lookup('dig', 'a.http.th.ooni.io/A') }}:80" - - name: "ooni explorer homepage" module: "http_2xx" targets: From f289d464803a068c0b5868b657732f7da17f5569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 20 Aug 2026 11:35:52 +0200 Subject: [PATCH 2/3] Add new test helpers checks --- ansible/roles/prometheus/files/alert_ooni.yml | 6 ++++++ ansible/roles/prometheus/vars/main.yml | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/ansible/roles/prometheus/files/alert_ooni.yml b/ansible/roles/prometheus/files/alert_ooni.yml index e43397ed..a84b2732 100644 --- a/ansible/roles/prometheus/files/alert_ooni.yml +++ b/ansible/roles/prometheus/files/alert_ooni.yml @@ -27,4 +27,10 @@ groups: summary: 'Mirror {{ $labels.instance }} lags by {{ $value | humanizeDuration }}' description: '{{ if eq $labels.instance "ooni.torproject.org:443" }}Run `make update-site`.{{ end }}' + - alert: TestHelperDown + expr: up{domain!=""} != 1 + for: 5m + annotations: + summary: 'test helper down: {{ $labels.instance }} ({{ $labels.domain }})' + ... diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index 26da57a2..c0cdea9f 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -51,6 +51,26 @@ blackbox_jobs: targets: - "https://explorer.ooni.org/" + # IP addresses are used for these test-helpers since that's what is handed + # out to probes by check-in. Keep them in sync with + # `generate_test_helpers_conf()` in ooni/backend + # ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py + - name: "ooni tcp echo" + module: "ooni_tcp_echo_ok" + ignore_instance_down: true # covered by the more specific TestHelperDown alert + labels: + domain: "echo.th.prod.ooni.io" + targets: + - "134.209.237.204:80" # echo.th.prod.ooni.io, tcp-echo test helper + + - name: "ooni http return json headers" + module: "ooni_http_return_json_headers_ok" + ignore_instance_down: true # covered by the more specific TestHelperDown alert + labels: + domain: "json.th.prod.ooni.io" + targets: + - "http://206.81.31.205:80" # json.th.prod.ooni.io + # API # - name: "ooni API measurements" From 49bc1d9d4e0bce61e15a2f44f601791435a94a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 20 Aug 2026 12:10:10 +0200 Subject: [PATCH 3/3] Lookup ip of test helper instead of hardcoding it --- ansible/roles/prometheus/vars/main.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index d50e0c8a..e9db75b1 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -54,17 +54,14 @@ blackbox_jobs: targets: - "https://explorer.ooni.org/" - # IP addresses are used for these test-helpers since that's what is handed - # out to probes by check-in. Keep them in sync with - # `generate_test_helpers_conf()` in ooni/backend - # ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py + # We use IP monitoring since this is what is served to users through check-in. - name: "ooni tcp echo" module: "ooni_tcp_echo_ok" ignore_instance_down: true # covered by the more specific TestHelperDown alert labels: domain: "echo.th.prod.ooni.io" targets: - - "134.209.237.204:80" # echo.th.prod.ooni.io, tcp-echo test helper + - "{{ lookup('dig', 'echo.th.prod.ooni.io/A') }}:80" - name: "ooni http return json headers" module: "ooni_http_return_json_headers_ok" @@ -72,7 +69,7 @@ blackbox_jobs: labels: domain: "json.th.prod.ooni.io" targets: - - "http://206.81.31.205:80" # json.th.prod.ooni.io + - "http://{{ lookup('dig', 'json.th.prod.ooni.io/A') }}:80" # API #