From f68dce824f3614b6113eb055a852f7cf4917e9fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=82ngelo=20Mois=C3=A9s=20Alves?=
<117550822+angeloINTJ@users.noreply.github.com>
Date: Sun, 2 Aug 2026 12:54:33 -0300
Subject: [PATCH] fix: the telemetry client would read a megabyte it never
wanted, and die doing it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
A campaign of 43 fault-injected servers against the four transports found that
three of them could put the device into a watchdog reboot loop, and that the
reboot had been hiding a second failure underneath it.
The reboots all came from two loops in the framework's HTTPClient with no upper
bound. handleHeaderResponse's _tcpTimeout is an INACTIVITY timeout — it resets
on every line — so a peer trickling one byte every 400 ms never trips it, and a
single readStringUntil('\n') spends 12.8 s inside one call on a 32-byte header
line, against an 8.388 s ceiling. disconnect() drains the body one byte at a
time "until available() is empty", which against a peer still sending means the
whole body. Both now carry one budget and feed the watchdog inside it, which is
only safe because the budget makes them terminate. Measured: huge1mb 4 reboots
to 0, drip 4 to 0 at two trickle rates, tls_slow20 2 to 0.
Without the reboots the device survived long enough to exhaust the lwIP pbuf
pool and go silent on the network while the serial CLI kept answering.
ClientContext::close() and abort() dropped the pcb without freeing _rx_buf, and
WiFiClient::stop() calls close() without releasing the context, so the refcount
never reached the unref() that would have discarded it. Freeing there took the
device from ~16 to ~144 connections before saturating; the pool going back to
its stock 24 (the cut to 12 saved 18 KB of BSS on the premise that 12 would not
saturate — it does) removed the saturation entirely.
Chasing what still killed the web server led below TCP: host ARP went
INCOMPLETE and ICMP got 100% loss while the firmware still announced its IP.
NET_READY's only liveness test is WiFi.status(), and cyw43 kept reporting
WL_CONNECTED with the link dead, so the state was never demoted and the working
reconnect path was never entered. The tell was an RSSI of +4 dBm — received
signal is negative by definition — which isNetworkHealthy() compared against a
-78 threshold and read as excellent. An implausible reading twice running now
demotes the state, and getRssi() reports it as a dead link.
Also, from the same campaign:
- the MQTT password typed on the config page was sent by the browser and never
read by commit_all, so a broker wanting credentials could not be reached
- an MQTT payload past the 8 KB client buffer failed deterministically, which
is a permanent stall rather than a retry; it now publishes record by record
- MQTT touched no metrics at all, so telSent/telFailed/latency stayed at zero
and update()'s adaptive interval, derived from smoothed latency, never engaged
- CSV mode emitted a 7-column header over 34-column rows
- a 4xx/5xx reply counted as neither sent nor failed and logged "HTTP OK"
- /api/ls lost one file per full batch of 20: `dir.next() && batchCount < 20`
advances the iterator before testing the count. 88 files listed as 84, and
because LittleFS iteration order varies, two listings dropped different ones
- the cursor could advance past a batch buildPayload had shortened under heap
pressure, and `pending` wrapped at 65535 with ~119k records on flash
Throughput and integrity held throughout: 48.6 records/s on HTTP and 49.7 on
MQTT at batch 50, zero reboots across 16 performance runs, and 1375 records
compared field by field against the on-flash .h5 decoded by the reference codec
matched exactly. TLS costs 35% of throughput on HTTPS and 12% on MQTT, because
HTTPS pays a handshake per POST while MQTT pays one and keeps the connection.
docs/telemetry-campaign-2026-08-02/ carries the method, the numbers, and the
defects — including the ones left open and the two conclusions this campaign
had to retract after retesting.
---
.../telemetry-campaign-2026-08-02/DEFEITOS.md | 431 ++++++++++++++++
.../METODOLOGIA.md | 122 +++++
.../RELATORIO.md | 320 ++++++++++++
src/NetworkManager.cpp | 49 +-
src/SystemDefs_Network.h | 10 +
src/TelemetryManager.cpp | 187 ++++++-
src/WebManager_Commit.cpp | 10 +
src/WebManager_Files.cpp | 35 +-
tools/arduino_pico_overrides/patch.sh | 73 ++-
.../patched_headers/lwipopts.h | 19 +-
.../patches/clientcontext_rx_leak.patch | 42 ++
.../patches/httpclient_read_deadlines.patch | 116 +++++
tools/telemetry_bench/README.md | 20 +
tools/telemetry_bench/bench.py | 363 +++++++++++++
tools/telemetry_bench/campaign.py | 168 +++++++
tools/telemetry_bench/enumerate_history.py | 133 +++++
tools/telemetry_bench/phase_drain.py | 183 +++++++
tools/telemetry_bench/phase_drain_full.py | 145 ++++++
tools/telemetry_bench/phase_mqtt.py | 341 +++++++++++++
tools/telemetry_bench/phase_mqtt_oversize.py | 113 +++++
tools/telemetry_bench/phase_payload.py | 148 ++++++
tools/telemetry_bench/phase_perf.py | 141 ++++++
tools/telemetry_bench/phase_survive.py | 226 +++++++++
tools/telemetry_bench/rescore.py | 82 +++
tools/telemetry_bench/revalidate.py | 219 ++++++++
tools/telemetry_bench/run_rest.sh | 31 ++
tools/telemetry_bench/run_rest2.sh | 35 ++
tools/telemetry_bench/run_rest3.sh | 15 +
tools/telemetry_bench/server_http.py | 393 +++++++++++++++
tools/telemetry_bench/server_mqtt.py | 475 ++++++++++++++++++
tools/telemetry_bench/soak24.py | 118 +++++
tools/telemetry_bench/summarize.py | 191 +++++++
32 files changed, 4926 insertions(+), 28 deletions(-)
create mode 100644 docs/telemetry-campaign-2026-08-02/DEFEITOS.md
create mode 100644 docs/telemetry-campaign-2026-08-02/METODOLOGIA.md
create mode 100644 docs/telemetry-campaign-2026-08-02/RELATORIO.md
create mode 100644 tools/arduino_pico_overrides/patches/clientcontext_rx_leak.patch
create mode 100644 tools/arduino_pico_overrides/patches/httpclient_read_deadlines.patch
create mode 100644 tools/telemetry_bench/README.md
create mode 100644 tools/telemetry_bench/bench.py
create mode 100644 tools/telemetry_bench/campaign.py
create mode 100644 tools/telemetry_bench/enumerate_history.py
create mode 100644 tools/telemetry_bench/phase_drain.py
create mode 100644 tools/telemetry_bench/phase_drain_full.py
create mode 100644 tools/telemetry_bench/phase_mqtt.py
create mode 100644 tools/telemetry_bench/phase_mqtt_oversize.py
create mode 100644 tools/telemetry_bench/phase_payload.py
create mode 100644 tools/telemetry_bench/phase_perf.py
create mode 100644 tools/telemetry_bench/phase_survive.py
create mode 100644 tools/telemetry_bench/rescore.py
create mode 100644 tools/telemetry_bench/revalidate.py
create mode 100755 tools/telemetry_bench/run_rest.sh
create mode 100755 tools/telemetry_bench/run_rest2.sh
create mode 100755 tools/telemetry_bench/run_rest3.sh
create mode 100644 tools/telemetry_bench/server_http.py
create mode 100644 tools/telemetry_bench/server_mqtt.py
create mode 100644 tools/telemetry_bench/soak24.py
create mode 100644 tools/telemetry_bench/summarize.py
diff --git a/docs/telemetry-campaign-2026-08-02/DEFEITOS.md b/docs/telemetry-campaign-2026-08-02/DEFEITOS.md
new file mode 100644
index 0000000..9495e78
--- /dev/null
+++ b/docs/telemetry-campaign-2026-08-02/DEFEITOS.md
@@ -0,0 +1,431 @@
+# Defeitos encontrados — campanha de telemetria 2026-08-02
+
+Cada item traz o mecanismo no código, a evidência que o provou e a correção
+proposta. Os que ainda dependem de medição em curso estão marcados
+**(a confirmar)** e serão fechados com o dado, não com a leitura do código.
+
+---
+
+## D1 — A senha MQTT digitada na web é descartada em silêncio
+
+**Severidade: alta.** Autenticação MQTT é impossível de configurar por qualquer
+caminho.
+
+**Mecanismo.** A página tem o campo (`WebUI.h:2577`):
+
+```html
+
+```
+
+e `wirePendingListeners()` (`WebUI.h:3055`) empurra **todo** input com `id`
+dentro de `#sysForm` para `Pending.sys`, por `id`. Ou seja, o navegador **envia**
+`sys.m_pass` no `_payload` do `/api/commit_all`.
+
+Do outro lado, `WebManager_Commit.cpp:570-575` trata `m_topic`, `m_cid`,
+`m_user`, `m_qos`, `m_retain` e `m_ka` — **não existe nenhum `has("m_pass")`**.
+O valor cai no chão sem log e sem erro. `cfg.mqttPass` só é escrito em
+`StorageManager.cpp:495`, que o zera nos defaults.
+
+Consequência: `mqttEnsureConnected()` sempre chama `connect()` com senha vazia
+(`TelemetryManager.cpp:710`), e qualquer broker que exija senha recusa o
+CONNECT. O usuário vê "MQTT failed: Bad credentials" com a senha certa digitada
+na tela.
+
+**Correção** (`WebManager_Commit.cpp`, junto de `m_user`):
+
+```cpp
+/* Vazio = manter a atual, como o placeholder do campo promete
+ * ("Leave empty to keep") e como t_key já faz com a máscara "***". */
+if (has("m_pass")) {
+ String mp = getStr("m_pass");
+ if (mp.length( ) > 0) safeCopy(cfg.mqttPass, mp.c_str( ), sizeof(cfg.mqttPass));
+}
+```
+
+---
+
+## D2 — O cabeçalho CSV descreve 7 colunas; as linhas têm 34
+
+**Severidade: alta.** Todo consumidor que confia no cabeçalho lê os valores nas
+colunas erradas.
+
+**Mecanismo.** `buildPayload()` monta o cabeçalho com **uma coluna por sensor
+ativo** (`TelemetryManager.cpp:1017-1025`):
+
+```cpp
+s = "timestamp";
+for (int i = 0; i < MAX_SENSORS; i++)
+ if (cfg.sensors[i].active) { snprintf(hdrBuf, ..., ";s%d_%s", i, cfg.sensors[i].hwId); ... }
+```
+
+mas `toCsvLine()` (`SystemDefs_Records.h:611`) emite **sempre** o layout fixo
+`epoch;s0..s15;h0..h15;press` = 34 campos, ativos ou não.
+
+**Evidência medida** (corpo cru capturado pelo sink, `results/phase_payload.json`):
+
+```
+cabeçalho (7): timestamp;s0_STM0009;s1_STM0010;s3_STH0003;s4_STB0001;s6_STZ9999;s10_STH0001
+linha (34): 1783140900;23.92;24.81;;25.14;;;;;;;23.99;;;;;;;;;68.3;;;;;;;68.5;;;;;;
+```
+
+Todas as 24 linhas do lote têm exatamente 34 campos. O comentário acima do
+código afirma "Header matches toCsvLine" — não casa.
+
+**Correção**: emitir o cabeçalho com o mesmo layout fixo das linhas, nomeando
+os slots ativos e deixando os inativos com nome posicional:
+
+```cpp
+s = "timestamp";
+for (int i = 0; i < MAX_SENSORS; i++) {
+ if (cfg.sensors[i].active && cfg.sensors[i].hwId[0])
+ snprintf(hdrBuf, sizeof(hdrBuf), ";s%d_%s", i, cfg.sensors[i].hwId);
+ else snprintf(hdrBuf, sizeof(hdrBuf), ";s%d", i);
+ s.concat(hdrBuf);
+}
+for (int i = 0; i < MAX_SENSORS; i++) { ... ";h%d..." ... }
+s.concat(";press");
+```
+
+O layout das linhas **não** muda — é o formato persistido e compatível com o
+upload; quem estava errado era o cabeçalho.
+
+---
+
+## D3 — A telemetria nunca alcança histórico além de 30 dias
+
+**Severidade: média** (é política, não bug — mas não está documentada e não há
+como contorná-la pela interface).
+
+**Mecanismo.** `collectBatch()` (`TelemetryManager.cpp:409-412`):
+
+```cpp
+if (lastCursor == 0) {
+ uint32_t lastRecorded = _storageRef->getLastRecordedTimestamp( );
+ if (lastRecorded > 86400UL * 30) lastCursor = lastRecorded - 86400UL * 30;
+}
+```
+
+`tel reset` zera o cursor, e é o único jeito de reenviar do começo. O piso de 30
+dias entra logo em seguida, e o corte por `minDay` descarta os arquivos
+anteriores. Não existe comando nem campo que peça mais.
+
+**Evidência medida.** Drenagem completa com `tel reset`, lote 50, intervalo 1 s:
+39 900 registros em 1009 s, começando em 2026-07-03 — enquanto o flash guarda
+desde 2026-05-03. O restante do arquivo é inalcançável por telemetria em
+qualquer tempo de execução.
+
+**Prova de que é política e não limite de armazenamento**: semeando
+`/config/t_cursor.bin` com 1 600 000 001 (logo acima de `HIST_EPOCH_MIN`) em vez
+de zero, o `if (lastCursor == 0)` não dispara e o dispositivo transmite o arquivo
+inteiro — ver a fase `drain_full`.
+
+**Correção proposta**: tornar a janela configurável (`t_backfill_days`, 0 = tudo)
+ou, no mínimo, registrar em log qual piso foi aplicado e por quê, para o operador
+não concluir que os dados sumiram.
+
+---
+
+## D4 — `/api/ls` trunca a listagem em silêncio
+
+**Severidade: alta.** Um cliente não consegue distinguir listagem parcial de
+completa, e o JSON sai bem-formado nos dois casos.
+
+**Mecanismo.** `handleApiLs()` (`WebManager_Files.cpp:186-187`):
+
+```cpp
+while (!dirDone) {
+ if (isHandlerOvertime( )) break; // <- sai e fecha o JSON como se tivesse acabado
+```
+
+`isHandlerOvertime()` compara com `_handlerDeadline`, fixado em
+`handlerStart + 6000` (`WebManager_Core.cpp:269`). Ao estourar, a função sai do
+laço, fecha `]}` e responde 200. Nada no corpo indica que faltou entrada.
+Compare com `handleApiHistoryMulti`, que **eleva** o prazo
+(`WEB_LONG_HANDLER_DEADLINE_MS`) justamente por ser um handler longo — o listador
+não faz isso.
+
+**Evidência medida.** Duas listagens de `/history` no mesmo dia devolveram
+**84 entradas cada, com conjuntos diferentes**:
+
+| arquivo | listagem A | listagem B | existe de fato? |
+|---|---|---|---|
+| `20260523.h5` | sim | não | **sim** — HTTP 200, 8166 B, md5 `3e26f8a71a` |
+| `20260524.h5` | não | sim | **sim** — HTTP 200, 8157 B, md5 `4017ee9696` |
+| `20260706.h5` | sim | não | sim |
+| `20260707.h5` | não | sim | sim (baixado, decodifica 1440 registros de 07-07) |
+
+Os dois arquivos de cada par existem e têm conteúdo distinto e correto; nenhuma
+listagem mostrou os dois. O gerenciador de arquivos da web sofre do mesmo
+problema, e qualquer inventário construído sobre `/api/ls` é subestimado por uma
+margem desconhecida.
+
+**Correção**: (a) sinalizar a truncagem no corpo — `"truncated":true` — para que
+nenhum cliente possa confundir; e (b) elevar o prazo desse handler como o
+`history_multi` faz. (a) é o essencial: sem ela, aumentar o prazo só empurra o
+limite para diretórios maiores.
+
+---
+
+## D5 — O cursor pode avançar por cima de um lote que `buildPayload` encurtou
+
+**Severidade: baixa hoje, alta se as constantes mudarem.** Latente: as margens
+atuais impedem que dispare.
+
+**Mecanismo.** `collectBatch()` devolve `newCursor` = maior epoch do lote
+**completo**. Em seguida `buildPayload()` pode **encurtar o lote** sob pressão de
+heap (`TelemetryManager.cpp:987-993`):
+
+```cpp
+if (freeHeap < estimatedSize + SEC_RESERVE) {
+ size_t safeCount = ...;
+ if (safeCount < batch.size( )) { batch.resize(safeCount); batch.shrink_to_fit( ); }
+}
+```
+
+mas `update()` segue usando o `newCursor` antigo:
+
+```cpp
+String payload = buildPayload(batch); // pode ter jogado registros fora
+batch.clear( );
+success = attemptHttpUpload(payload, newCursor); // avança até o lote INTEIRO
+```
+
+Se o encurtamento acontecer, o cursor pula os registros descartados e eles nunca
+são enviados — sem log. Vale para os dois ramos de `update()` e para
+`forceSync()`.
+
+**Por que não dispara hoje**: `safeBatchLimit()` já limitou o lote com uma
+reserva **maior** (32768 com TLS, 12288 sem) do que a de `buildPayload`
+(12288 / 6144), e com 350 B/registro contra 300. A desigualdade se mantém para
+qualquer heap. Ou seja, a segurança vem de um acoplamento implícito entre quatro
+constantes em duas funções — inverta qualquer uma e a perda passa a ser real.
+
+**Correção** (barata e torna o acoplamento desnecessário): fazer o cursor seguir
+o que o payload realmente contém.
+
+```cpp
+String payload = buildPayload(batch);
+/* buildPayload pode encurtar o lote sob pressão de heap; o cursor tem de
+ * seguir o que o payload contém, não o que foi coletado. O lote está em
+ * ordem crescente de epoch (arquivos ordenados, registros em ordem). */
+if (!batch.empty( )) newCursor = batch.back( ).epoch;
+```
+
+---
+
+## D6 — `pending` é `uint16_t` e envolve sem saturar
+
+**Severidade: baixa** (cosmético/diagnóstico), mas engana quem depende do número.
+
+**Mecanismo.** `refreshPendingCount()` acumula em `uint16_t total` com cast
+explícito (`TelemetryManager.cpp:1456,1496`):
+
+```cpp
+uint16_t total = 0;
+...
+total = (uint16_t)(total + hdr.pre.a);
+```
+
+Com o cursor zerado o laço conta **todos** os registros de **todos** os
+arquivos. O flash desta bancada guarda ~119 mil registros — bem acima de 65535 —
+e o valor envolve para um número plausível porém falso, que aparece no dashboard
+e em `/api/status`.
+
+Além disso `refreshPendingCount()` **não** aplica o piso de 30 dias que
+`collectBatch()` aplica: logo depois de um `tel reset`, o contador soma registros
+que o dispositivo jamais vai enviar.
+
+**Correção**: acumular em `uint32_t` e saturar na atribuição —
+`_pendingEstimate = (total > 0xFFFF) ? 0xFFFF : (uint16_t)total;` — e aplicar o
+mesmo piso de 30 dias do `collectBatch` para os dois contadores concordarem.
+
+---
+
+## D7 — QoS 1 e 2 são oferecidos na interface e nunca saem no fio **(a confirmar)**
+
+A página oferece `0 - At Most Once`, `1 - At Least Once`, `2 - Exactly Once`
+(`WebUI.h:2582-2586`) e o firmware valida e persiste `cfg.mqttQos` de 0 a 2
+(`WebManager_Commit.cpp:573`). Mas `attemptMqttPublish()` chama sempre a
+sobrecarga de três argumentos do PubSubClient:
+
+```cpp
+_mqttClient.publish(topic.c_str( ), linePayload.c_str( ), cfg.mqttRetain);
+```
+
+que publica **QoS 0** — o PubSubClient não implementa publish com QoS 1 ou 2.
+`cfg.mqttQos` não é lido em lugar nenhum do caminho de envio.
+
+Consequência prática: sem QoS 1 não há PUBACK, então `publish()` devolve
+sucesso assim que escreve no socket TCP. Um broker que morra no meio do publish
+é indistinguível de um que recebeu — e o cursor avança. O teste
+`mq_drop_on_publish` mede exatamente isso.
+
+*A confirmar pelas flags do PUBLISH registradas pelo broker instrumentado.*
+
+---
+
+## D8 — Payload MQTT acima de ~8 KB trava a telemetria para sempre **(a confirmar)**
+
+`attemptMqttPublish()` cresce o buffer para caber o payload, mas com teto rígido:
+
+```cpp
+if (payload.length( ) > _mqttClient.getBufferSize( )) {
+ uint16_t needed = min((size_t)8192, payload.length( ) + 64);
+ _mqttClient.setBufferSize(needed);
+}
+```
+
+`PubSubClient::publish()` recusa qualquer pacote que não caiba no buffer. Acima
+de ~8 KB o publish falha **de forma determinística**: o retry falha igual, o
+backoff escala até o teto de 300 s e a telemetria fica parada para sempre — sem
+reboot e sem mensagem que explique.
+
+É alcançável pela tela de configuração: lote 50 com um `t_line` longo
+(o campo aceita 512 bytes) passa de 8 KB com os sensores desta bancada.
+
+*A confirmar pela fase `mqtt_oversize`.*
+
+---
+
+## D9 — Um único campo de pressão por registro, rotulado pelo slot errado
+
+**Severidade: média** para quem tiver mais de um sensor de pressão.
+
+`BinaryHistoryRecord` tem `pressure` como **escalar**, não um por slot. Em
+`collectBatch()`:
+
+```cpp
+else if (chOf[c] == CH_PRESS) rec.pressure = BinaryHistoryRecord::floatToI16x10(v);
+```
+
+o laço percorre os canais do schema e **o último vence**. E em
+`formatLineJsonBuf()` o rótulo vem do **primeiro** slot ativo capaz de pressão:
+
+```cpp
+for (int i = 0; i < MAX_SENSORS; i++)
+ if (cfg.sensors[i].active && ... sensorHasChannel(..., CH_PRESS)) { pHwid = cfg.sensors[i].hwId; break; }
+```
+
+Com dois sensores de pressão provisionados, o valor publicado é o do **último**
+slot sob o nome do **primeiro** — e o do primeiro nunca é publicado. Canais NaN
+são pulados, então um slot fantasma sem leitura não estraga nada; dois sensores
+reais, sim.
+
+**Correção**: ou passar `pressure` a um vetor por slot no registro de telemetria,
+ou (mínimo) rotular com o slot de onde o valor veio, guardando o índice junto
+com o valor em `collectBatch`.
+
+---
+
+## D14 — Vazamento de PBUF sob respostas HTTP grandes (descoberto na revalidação)
+
+**Severidade: alta.** O servidor web morre e não volta sem reboot.
+
+Corrigido o D10, os reboots por watchdog somem — e o dispositivo passa a
+sobreviver tempo suficiente para expor um problema que os reboots vinham
+apagando a cada 8,4 s.
+
+**Evidência.** Contra o servidor que responde HTTP 200 com 1 MB de corpo, após
+~19–67 conexões seguidas:
+
+```
+show net status → PBUF pool: 12 em uso / pico 12 / 12 total, 1158 falhas
+```
+
+O dispositivo continua vivo: IP 192.168.3.24, RSSI −42 dBm, CLI serial
+respondendo comandos normalmente. O que morre é só a pilha de rede — sem buffer
+não há pacote de resposta, e o web fica mudo.
+
+**Não é ocupação transitória, é vazamento.** Com a telemetria desligada
+(`tel interval 0`) o pool fica em 12/12 e o contador de falhas continua subindo
+(519 → 666). Um `reload confirm` devolve tudo:
+`PBUF pool: 0 em uso / pico 1 / 12 total, 0 falhas`, web HTTP 200.
+
+**Agravante de projeto**: `PBUF_POOL_SIZE` foi reduzido de 24 para 12 no patch de
+lwIP (`tools/arduino_pico_overrides`), economizando 18 KB de RAM. Com 12 entradas
+a margem é estreita.
+
+**Atribuição corrigida.** Primeiro atribuí o vazamento ao `stop( )` que eu havia
+acrescentado antes do `http.end( )` em `attemptHttpUpload`. Removi o `stop( )`,
+regravei e refiz: **mesmo resultado**. A hipótese estava errada; o `stop( )` saiu
+de vez (sem benefício medido e com risco). O vazamento antecede as mudanças desta
+campanha.
+
+**Não corrigido**, deliberadamente: localizar onde os pbufs da resposta abortada
+deixam de ser liberados exige instrumentar o lwIP, e não quis propor correção que
+não pudesse validar na bancada. Duas direções para quem for atacar:
+
+1. o caminho de `tcp_abort`/close com fila de recepção pendente, que é onde a
+ resposta grande é largada pela metade;
+2. um teto no tamanho de corpo que a telemetria aceita — recusar antes de
+ começar a ler é mais barato do que abortar no meio.
+
+
+---
+
+## D15 — O aparelho sai da rede WiFi e o firmware não percebe
+
+**Severidade: alta.** Fica inacessível até alguém reiniciar, anunciando-se saudável.
+
+Descoberto ao investigar por que o servidor web morria sob rajada de respostas
+grandes mesmo depois de o pool de PBUF parar de saturar (D14).
+
+**Não era rede nem buffer.** A prova foi separar as camadas, do host:
+
+```
+ping 192.168.3.24 → 4 pacotes, 0 recebidos, 100% de perda
+ip neigh → 192.168.3.24 dev enp6s0 INCOMPLETE
+```
+
+ARP incompleto está **abaixo** de TCP e de IP: o aparelho não estava na rede.
+Enquanto isso, pela serial:
+
+```
+IP: 192.168.3.24
+RSSI: 4 dBm
+PBUF pool: 7 em uso / pico 20 / 24 total, 0 falhas
+```
+
+Zero falhas de buffer, IP anunciado, e um **RSSI de +4 dBm** — força de sinal
+recebido é negativa por definição, então o ioctl do cyw43 havia parado de
+devolver dado real.
+
+**Mecanismo** (`NetworkManager.cpp`, `case NET_READY`):
+
+```cpp
+if (WiFi.status( ) != WL_CONNECTED) { ...rebaixa e reconecta... }
+```
+
+O único teste de vida é a palavra do próprio driver, e ele seguia respondendo
+`WL_CONNECTED` com o enlace morto. `isConnected( )` é `_state == NET_READY`, o
+estado nunca era rebaixado, e o caminho de reconexão — que existe e funciona —
+nunca era acionado.
+
+Agravante: `isNetworkHealthy( )` é `isConnected( ) && getRssi( ) > RSSI_MIN_THRESHOLD`
+com o limiar em −78. O valor corrompido de **+4 passa folgado**: a leitura
+quebrada não escapava da detecção, ela **confirmava saúde**.
+
+**Correção.** O RSSI já era amostrado ali mesmo, 1×/min. Ele é o sinal
+independente que faltava:
+
+- `NET_READY` rebaixa para `NET_DISCONNECT_PENDING` após **duas** leituras fora
+ da faixa plausível (`RSSI_IMPLAUSIBLE_HIGH = 0`, `RSSI_IMPLAUSIBLE_LOW = -120`).
+ Duas, não uma: um valor esquisito pode ser glitch, e reconectar à toa custa
+ mais do que esperar um minuto. Detecção em ~2 min em vez de nunca.
+- `getRssi( )` devolve −100 para leitura implausível, então `isNetworkHealthy( )`
+ também reprova e a telemetria para de tentar.
+- Cadência mantida em 1×/min de propósito: RSSI é ioctl vivo, e martelá-lo é
+ risco próprio — a assinatura `C0=[WIFI]` por leitura-viva-que-bloqueia já está
+ registrada no histórico do projeto.
+
+**O que NÃO está resolvido, e a ressalva do teste.** A causa de o rádio morrer
+segue desconhecida; a correção troca "fora da rede até reiniciar" por "detectado
+e reconectado", que é o dano operacional real, mas não impede a queda.
+
+E na corrida de validação **o rádio não morreu** — RSSI ficou em −46 dBm o tempo
+todo, e o aparelho atravessou 150 s de rajada e voltou a responder em ~40 s. Ou
+seja: está provado que o aparelho aguenta e se recupera, **não** que o novo
+caminho de detecção dispara corretamente quando o cyw43 corrompe. Reproduzir a
+morte do rádio não é determinístico, e não vou registrar como verificado o que
+não vi disparar.
diff --git a/docs/telemetry-campaign-2026-08-02/METODOLOGIA.md b/docs/telemetry-campaign-2026-08-02/METODOLOGIA.md
new file mode 100644
index 0000000..e25d3c3
--- /dev/null
+++ b/docs/telemetry-campaign-2026-08-02/METODOLOGIA.md
@@ -0,0 +1,122 @@
+# Campanha de telemetria — metodologia e bancada
+
+**Data:** 2026-08-02
+**Alvo:** Raspberry Pi Pico W, firmware SIMUT **2.0.1-alpha**, env `pico_w_test`
+(`SIMUT_CLI_FULL=1` — a CLI reduzida do release não tem `tel …`, então a suíte
+não roda nela).
+**IP do dispositivo:** 192.168.3.24 · **host de teste:** 192.168.3.31
+**Serial:** `/dev/serial/by-id/usb-Raspberry_Pi_Pico_W_E6642815E34C1824-if00`
+
+## 1. Por que três instrumentos
+
+Cada medição é lida de três lugares independentes, porque nenhum deles sozinho
+distingue "o dispositivo está bem" de "o dispositivo está mudo":
+
+| instrumento | responde | cegueira que ele tem |
+|---|---|---|
+| **servidor de teste** | o que de fato chegou (bytes, registros, epochs, relógio de parede) | não vê o dispositivo |
+| **`/api/status`** (autenticado) | o que o dispositivo acredita: `ts`/`tf`/`tl`/`tb`, heap, maior bloco, `pending` | morre junto com o dispositivo |
+| **serial (CDC)** | reboot (o USB re-enumera), `[FTL]`, `SOFT PANIC`, `HW WATCHDOG` | não é confiável durante um travamento do Core 0 |
+
+`/api/status` **exige sessão**: uma sonda sem login recebe `{"error":"Forbidden"}`
+e um coletor descuidado lê isso como "dispositivo respondendo". A biblioteca da
+bancada faz login uma vez e refaz a sessão em 401/403.
+
+## 2. O que a bancada precisou saber antes de medir
+
+**Backoff.** `TelemetryManager::escalateBackoff()` dobra o intervalo a partir de
+5 s até 300 s. Uma janela de medição aberta logo depois de trocar o servidor cai
+**dentro** de um backoff herdado e mede o temporizador, não o transporte — foi
+exatamente o que aconteceu na primeira tentativa de HTTPS (0 conexões em 45 s, e
+o servidor estava perfeito). `forceSync()` chama `resetBackoff()` **antes** de
+qualquer outra coisa, então `tel sync` é a única forma de zerar o backoff sem
+reboot. Toda janela começa com `tel reset` + `tel sync`.
+
+**O que exige reboot.** `telTransport`, o cliente MQTT e o TLS do MQTT são lidos
+**uma única vez**, em `TelemetryManager::begin()`. Trocar HTTP↔MQTT ou
+MQTT↔MQTTS custa um `POST /api/commit_all` (que reinicia). Já servidor, porta,
+lote, intervalo e modo do caminho HTTP são relidos a cada envio e mudam pela CLI
+sem reboot.
+
+Consequência de projeto para os testes de falha: **todos rodam na mesma porta**,
+e o modo de falha vem de reiniciar o processo do servidor — nunca de
+reconfigurar o dispositivo. Isso mantém o dispositivo estável entre falhas e
+elimina o reboot como variável.
+
+**Ponto de partida repetível.** `tel reset` apaga o cursor; o próximo
+`collectBatch` cai no piso de `lastRecorded − 30 dias`. Toda corrida começa daí,
+então todas veem a mesma fila grande e os números comparam.
+
+## 3. Servidores de teste
+
+Escritos para este trabalho (`scratchpad/telbench/`), em Python puro, porque o
+ponto é **errar de propósito** — nenhum servidor real tem chave para "responda
+metade do cabeçalho e feche".
+
+### `server_http.py` — sink HTTP/HTTPS instrumentado
+
+Modos: `ok`, `error500`, `error401`, `blackhole` (aceita e nunca responde),
+`slow N`, `half` (linha de status parcial + FIN), `rst` (RST no accept),
+`rst_mid` (RST no meio dos cabeçalhos), `garbage` (bytes não-HTTP), `huge`
+(corpo de 1 MB), `drip` (1 byte a cada N ms), `close_early`.
+Falhas de TLS antes de qualquer HTTP: `--tls-fault blackhole|garbage|rst|slow`.
+TLS fixado em 1.2 com cert autoassinado, igual ao servidor real do usuário.
+
+### `server_mqtt.py` — broker MQTT 3.1.1 instrumentado
+
+Implementado do formato de fio (não é wrapper do mosquitto) para poder mentir:
+`rst`, `no_connack`, `slow_connack`, `half_connack` (2 dos 4 bytes),
+`connack_refuse/badproto/badid/unavail`, `drop_after_connack`,
+`drop_on_publish`, `rst_on_publish`, `garbage`, `no_pingresp`. Fala
+CONNECT/CONNACK, PUBLISH (QoS 0 e 1), PUBACK, SUBSCRIBE/SUBACK,
+PINGREQ/PINGRESP e DISCONNECT — o suficiente para o PubSubClient do firmware.
+Registra os campos do CONNECT (usuário, senha, client id, keepalive, will) e as
+flags de cada PUBLISH (QoS, retain, dup), que é a única forma de checar se a
+config prometida na web chega ao fio.
+
+## 4. Estrutura de um teste de sobrevivência
+
+"Não reiniciou" é metade da pergunta. A outra metade é se o dispositivo jogou
+dado fora em silêncio. Por isso cada falha tem três atos:
+
+1. **linha de base** — sink bom, `tel reset` + `tel sync`, anota o último epoch
+ aceito (E1);
+2. **falha** — troca o servidor pelo defeituoso e roda a janela inteira;
+3. **recuperação** — sink bom de volta, `tel sync`, anota o primeiro epoch
+ aceito (E2).
+
+Se **E2 > E1 + 1 intervalo**, o dispositivo avançou o cursor por cima de
+registros que servidor nenhum confirmou: perda de dados. Sem os atos 1 e 3, uma
+falha que come histórico é indistinguível de uma que o dispositivo ignorou.
+
+## 5. Endereços que não são servidores
+
+Duas falhas não podem ser produzidas por um socket escutando, e por isso trocam
+o endereço em vez do modo:
+
+- `syn_blackhole` → `192.0.2.1` (TEST-NET-1, RFC 5737): roteado para lugar
+ nenhum, o SYN é engolido em vez de recusado, e o `connect()` bloqueia em vez
+ de falhar rápido;
+- `dns_fail` → `nao-existe.invalid`: a resolução é que bloqueia.
+
+## 6. Integridade, não só vazão
+
+Vazão sem conferir valor não vale nada. A fase de payload:
+
+- captura os corpos crus das requisições nos três modos (`json`, `csv`,
+ `custom`) e confere que são o que prometem;
+- baixa os `.h5` dos mesmos dias por `/download` e decodifica com o codec de
+ referência (`tools/history_v5.py`), comparando **valor a valor** contra o que
+ chegou pela telemetria.
+
+## 7. Configuração original do usuário (restaurada no fim)
+
+Salva em `results/ORIGINAL_CONFIG.json`:
+
+```json
+{"t_transport": 0, "t_sec": true, "t_srv": "192.168.3.206", "t_port": 8443,
+ "t_path": "/api.php", "t_int": 10000, "t_bat": 50, "t_mode": 0,
+ "m_topic": "simut/data", "m_qos": 0, "m_retain": false, "m_ka": 60, "h_int": 1}
+```
+
+Usuário web descartável criado para a campanha: `telb` (admin), apagado no fim.
diff --git a/docs/telemetry-campaign-2026-08-02/RELATORIO.md b/docs/telemetry-campaign-2026-08-02/RELATORIO.md
new file mode 100644
index 0000000..83beac4
--- /dev/null
+++ b/docs/telemetry-campaign-2026-08-02/RELATORIO.md
@@ -0,0 +1,320 @@
+# Telemetria SIMUT — campanha de desempenho e sobrevivência
+
+**Data:** 2026-08-02 · **Firmware:** 2.0.1-alpha (`pico_w_test`) · **Alvo:** Raspberry Pi Pico W
+**Metodologia e bancada:** [METODOLOGIA.md](METODOLOGIA.md) · **Defeitos detalhados:** [DEFEITOS.md](DEFEITOS.md)
+
+---
+
+## 1. Resumo executivo
+
+Foram medidos os quatro transportes (HTTP, HTTPS, MQTT, MQTTS) em quatro tamanhos
+de lote cada, exercidos **43 modos de falha de servidor** escritos para errar de
+propósito, drenado o histórico do dispositivo duas vezes e conferido valor a valor
+o que chegou contra o que está no flash.
+
+**O que está sólido:**
+
+- **Vazão**: até 49,7 registros/s. Os quatro transportes escalam linearmente com
+ o tamanho do lote, sem falha e sem reboot em 16 corridas de desempenho.
+- **Integridade**: 100% dos 1375 registros conferidos batem com o flash,
+ decodificados pelo codec de referência.
+- **Tratamento de erro de rede**: 40 dos 43 modos de falha foram absorvidos como
+ uma linha de log e um backoff — DNS que não resolve, SYN engolido, conexão
+ recusada, RST, meia resposta, lixo binário, CONNACK recusado, broker que morre
+ no meio do publish.
+- **Cursor**: em HTTP e HTTPS, **zero** registros perdidos em 21 modos de falha.
+
+**O que estava quebrado (14 defeitos; 9 corrigidos e verificados, 5 abertos):**
+
+- **3 modos de falha derrubavam o aparelho em laço de reboot.** Os três foram
+ corrigidos e verificados (§7.3): `huge1mb` 4→0, `drip` 4→0 (em duas taxas de
+ gotejamento), `tls_slow20` 2→0 (e um caso mais duro, 40 s, também passa).
+- **Um payload MQTT acima de 8 KB parava a telemetria para sempre**, em silêncio.
+- **A senha MQTT digitada na web nunca chegava ao broker.**
+- **O transporte MQTT era invisível para as métricas**, e por isso a cadência
+ adaptativa nunca funcionava nele.
+- **`/api/ls` truncava listagens sem avisar** — duas listagens do mesmo diretório
+ devolviam conjuntos diferentes.
+- **O modo CSV emitia um cabeçalho de 7 colunas para linhas de 34.**
+- **Uma resposta 4xx/5xx não contava como falha** e era registrada como "HTTP OK".
+
+---
+
+## 2. Desempenho
+
+Cada corrida: 90 s, `tel reset` + `tel sync` antes de abrir a janela (senão a
+medição cai dentro de um backoff herdado e mede o temporizador, não o transporte).
+
+### 2.1 Vazão por transporte e tamanho de lote (registros/s)
+
+| lote | HTTP | HTTPS | MQTT | MQTTS |
+|---|---|---|---|---|
+| 1 | 0,97 | 0,71 | 2,10 | 0,96 |
+| 5 | — | — | 4,27 | 4,97 |
+| 10 | 10,16 | 6,93 | 9,67 | 9,67 |
+| 25 | 25,44 | 16,83 | — | — |
+| **50** | **48,61** | **31,67** | **49,67** | **43,89** |
+
+### 2.2 Latência por POST (medida no dispositivo, HTTP/HTTPS)
+
+| lote | HTTP mediana | HTTP máx | HTTPS mediana | HTTPS máx |
+|---|---|---|---|---|
+| 1 | 14 ms | 22 ms | 506 ms | 556 ms |
+| 10 | 20 ms | 35 ms | 550 ms | 640 ms |
+| 25 | 29 ms | 34 ms | 610 ms | 697 ms |
+| 50 | 46 ms | 64 ms | 715 ms | 884 ms |
+
+### 2.3 O achado de desempenho que muda uma decisão de projeto
+
+**O TLS custa 35% da vazão no HTTPS e 12% no MQTT.**
+
+A razão é estrutural, não de implementação: o HTTPS abre e fecha uma conexão TLS
+**por POST** — 490 ms de handshake que aparecem inteiros na latência de lote 1
+(506 ms contra 14 ms do HTTP puro). O MQTT paga esse handshake **uma vez** e
+mantém a conexão: as quatro corridas MQTTS de 90 s inteiras usaram **1 conexão**.
+
+Para telemetria cifrada com vazão, a recomendação é **MQTTS, não HTTPS**.
+
+O preço do MQTTS é heap: 69 KB livres contra 96–102 KB dos demais, porque o
+contexto BearSSL fica residente. Ainda é folga confortável.
+
+### 2.4 Custo de memória
+
+| transporte | heap livre mín | maior bloco mín |
+|---|---|---|
+| HTTP | 95 656 B | 85 740 B |
+| HTTPS | 95 592 B | 74 967 B |
+| MQTT | 96 576 B | — |
+| MQTTS | 69 056 B | — |
+
+Nenhuma tendência de queda ao longo das corridas: o heap volta ao mesmo valor a
+cada ciclo.
+
+---
+
+## 3. Integridade dos dados
+
+Duas verificações independentes.
+
+**Valor a valor.** 1375 epochs recebidos pela telemetria foram comparados campo a
+campo contra os mesmos registros lidos dos `.h5` do dispositivo e decodificados
+pelo codec de referência (`tools/history_v5.py`): **1375 de 1375 conferem
+(100%)**, 0 ausentes no disco.
+
+**Formatos de payload.** Os três modos foram capturados crus no servidor:
+
+- `json` — `[{"ts":1783134000,"tSTM0009":22.00,"tSTM0010":22.86,...}]` ✔
+- `custom` — `{"dev":"simut","mac":"28:cd:c1:15:4e:99","data":[{"ts":...,"tSTM0009":23.92}]}` ✔
+ (reescrita de chave `t0_ID` → `tSTM0009` e remoção de token sem canal funcionam)
+- `csv` — **cabeçalho não descreve as linhas** (defeito D2, corrigido)
+
+---
+
+## 4. Sobrevivência — 43 modos de falha
+
+Estrutura de cada teste: linha de base com servidor bom → falha → recuperação com
+servidor bom. A diferença entre o último epoch aceito antes e o primeiro depois
+mede se o dispositivo avançou o cursor por cima de dado que ninguém recebeu.
+
+### 4.1 HTTP (14 modos)
+
+| falha | veredito | reboots | reg. perdidos |
+|---|---|---|---|
+| refused, blackhole, slow20, half, rst, rst_mid, garbage, close_early, syn_blackhole, dns_fail | sobreviveu | 0 | 0 |
+| error401, error500 | sobreviveu, **mas falha invisível** (D11) | 0 | 0 |
+| **huge1mb** | **REBOOT ×4 + FTL** | 4 | 0 |
+| **drip** | **REBOOT ×4 + FTL** | 4 | 0 |
+
+### 4.2 HTTPS (7 modos)
+
+| falha | veredito | reboots | reg. perdidos |
+|---|---|---|---|
+| tls_blackhole, tls_garbage, tls_rst, tls_refused, tls_error500, tls_blackhole_http | sobreviveu | 0 | 0 |
+| **tls_slow20** | **REBOOT ×2 + FTL** | 2 | 0 |
+
+O `tls_blackhole` (aceita TCP e nunca fala) **sobreviver** é a prova de que o
+prazo global de handshake TLS que já existia no repositório funciona: o
+dispositivo desiste em 15 s e reporta erro. O que ainda matava era o **dreno
+pós-falha**, não o handshake.
+
+### 4.3 MQTT (12 modos) e MQTTS (5 modos)
+
+Todos sobreviveram — **zero reboots em 17 modos**. O cliente MQTT trata
+corretamente broker ausente, RST no accept, CONNACK que nunca chega, CONNACK
+lento, meio CONNACK, CONNACK recusado, queda pós-CONNACK, queda no publish, RST
+no publish, lixo binário e PINGREQ ignorado.
+
+---
+
+## 5. Descarga do histórico
+
+### 5.1 Pelo caminho normal (`tel reset`)
+
+| métrica | valor |
+|---|---|
+| duração | 1009 s |
+| envios HTTP | 799 |
+| registros aceitos | 39 900 |
+| epochs únicos | 39 659 |
+| taxa sustentada | 39,5 reg/s |
+| reboots | 0 |
+| primeiro registro | 2026-07-03 |
+| último registro | agora |
+
+### 5.2 O teto de 30 dias
+
+`tel reset` zera o cursor, e `collectBatch` então recusa olhar mais para trás do
+que `lastRecorded − 30 dias`. Com **92 dias de histórico no flash**, isso deixa
+dois terços do arquivo inalcançáveis pela telemetria, em qualquer tempo de
+execução. Não há comando nem campo de configuração que peça mais.
+
+**É política, não limite de armazenamento — e isso foi provado.** Semeando
+`/config/t_cursor.bin` com 1 600 000 001 em vez de zero, o `if (lastCursor == 0)`
+não dispara e o aparelho transmitiu **o arquivo inteiro**: 124 800 registros em
+3038 s (41,07 reg/s), 2496 POSTs, 14,66 MB, **zero reboots**, cobrindo
+2026-05-03 → 2026-08-02.
+
+| caminho | epochs únicos | cobertura do que há no flash |
+|---|---|---|
+| `tel reset` (padrão do produto) | 39 659 | **31,7 %** |
+| cursor semeado | 124 609 | **99,64 %** |
+
+Inventário real, obtido por data em vez de confiar no `/api/ls`: **88 arquivos,
+125 058 epochs únicos, 0 erros de decodificação**.
+
+---
+
+## 6. Correções aplicadas
+
+Onze defeitos corrigidos, compilando nos dois ambientes
+(`pico_w_test` 96,9%, `pico_w_release` 92,3%) e com os 40 testes nativos passando.
+
+| # | defeito | correção |
+|---|---|---|
+| D10 | servidor lento/grande = laço de reboot | patch de framework: prazo e feed **entre caracteres** na leitura de cabeçalho, mais dreno limitado no `disconnect( )`. Resolve os três casos (§7.3) |
+| D8 | payload MQTT > 8 KB = parada permanente | publica registro a registro quando o payload não cabe |
+| D1 | senha MQTT descartada | `commit_all` passa a ler `m_pass`; vazio = manter |
+| D2 | cabeçalho CSV de 7 colunas para linhas de 34 | cabeçalho passa a nomear as 34 colunas reais |
+| D4 | `/api/ls` perdia 1 arquivo a cada lote de 20 | `batchCount < 20 && dir.next( )` — conta antes de avançar; mais `"truncated"` e prazo longo como defesa |
+| D5 | cursor pula lote encurtado por heap | cursor segue `batch.back().epoch` nos 3 caminhos de envio |
+| D6 | `pending` estoura em 65535 | acumulador de 32 bits saturado, e o mesmo piso de 30 dias do `collectBatch` |
+| D11 | 4xx/5xx não contava como falha | ramo próprio, `telFailed++` e log de erro honesto |
+| D12 | MQTT invisível às métricas | as duas rotas alimentam `telSent`/`telFailed`/`telTotalBytes`/`telLastLatencyMs` e a média de latência |
+
+Não corrigidos (decisão de produto ou risco documentado): D3 (teto de 30 dias),
+D7 (QoS 1/2 oferecido mas não implementável com PubSubClient), D9 (um campo de
+pressão por registro).
+
+---
+
+## 7. Revalidação com o firmware corrigido
+
+Firmware gravado (`pico_w_test`, 96,9%) e os testes refeitos. Para não medir um
+efeito colateral em vez do alvo, a ordem foi invertida e há reboot entre grupos:
+o `huge1mb` esgota o pool de PBUF (D14) e mataria a web de todos os testes
+seguintes, então roda por último.
+
+### 7.1 Corrigido e verificado
+
+| defeito | evidência antes | evidência depois |
+|---|---|---|
+| **D2** cabeçalho CSV | 7 colunas para linhas de 34 | **34/34, `match: true`** |
+| **D4** `/api/ls` incompleto | 84 de 88, conjuntos variando entre chamadas | **88 de 88, estável em 4 chamadas, `missing: []`** |
+| **D1** senha MQTT | CONNECT com `pass: ""` | **`pass: "benchsecret"`** no fio |
+| **D11** 4xx/5xx invisível | `devFail+0`, log "HTTP OK ... code 500" | **`devFail+4`**, e recupera |
+| **D10** `huge1mb` | 4 reboots + `[FTL] HW WATCHDOG` | **0 reboots** (3 corridas) |
+
+Fidelidade MQTT: 6 de 7 verificações passam (client id, usuário, **senha**,
+keepalive, tópico, retain). A que falha é `qos1_honoured` — D7, não corrigido
+porque o PubSubClient não implementa publish em QoS 1/2.
+
+**Sem regressão de desempenho**: HTTP lote 50 = 44,44 reg/s (era 48,61), HTTPS
+lote 50 = 27,78 reg/s (era 31,67), 0 falhas e 0 reboots nas duas. Dentro da
+variação de bancada.
+
+### 7.2 A causa real do D4 não era a que eu diagnosticou
+
+A primeira hipótese foi o guarda de tempo (`isHandlerOvertime`). Errada: a
+listagem corrigida devolvia 84 de 88 com `truncated: false` — o laço terminava
+normalmente e ainda assim perdia arquivos. O mecanismo é ordem de avaliação:
+
+```cpp
+while (dir.next( ) && batchCount < 20) // <- avança ANTES de testar o limite
+```
+
+Quando `batchCount` chega a 20, `dir.next( )` já moveu o iterador e a entrada é
+descartada. **Cada lote cheio perde uma**: 88 arquivos, 4 lotes cheios, 84
+listados. E como a ordem de iteração do LittleFS varia, variam quais somem — o
+que explica as duas listagens divergentes. Correção: `batchCount < 20 && dir.next( )`.
+O marcador `"truncated"` e o prazo maior ficaram como defesa em profundidade.
+
+### 7.3 As três mortes por watchdog — resolvidas
+
+| falha | reboots antes | reboots depois |
+|---|---|---|
+| `huge1mb` (corpo de 1 MB) | 4 | **0** (3 corridas) |
+| `drip` 400 ms/byte | 4 | **0** |
+| `drip` 900 ms/byte (mais duro) | — | **0** |
+| `tls_slow20` (dorme 20 s) | 2 | **0** |
+| `tls_slow40` (dorme 40 s, mais duro) | — | **0** |
+
+Zero linhas `[FTL]` em todos, e as falhas contadas corretamente (`devFail+3/+4`).
+
+**A primeira correção não bastava, e eu cheguei a reportar `drip` como resolvido
+antes da hora.** Aquele "sobreviveu" veio de um aparelho já degradado pelo
+`huge1mb` imediatamente anterior — com o pool de PBUF esgotado ele nem abria
+conexão, portanto nunca alcançava o caminho que trava. Com reboot entre os casos
+o resultado real apareceu: 3 reboots.
+
+**A raiz que faltava.** O orçamento global que pus em `handleHeaderResponse` era
+checado **entre linhas**, mas o bloqueio está dentro de uma única chamada:
+
+```cpp
+String headerLine = _client()->readStringUntil('\n');
+```
+
+`readStringUntil` lê caractere a caractere sem alimentar o watchdog. A
+400 ms/byte, `Content-Type: application/json\r\n` (32 bytes) consome **12,8 s
+numa só chamada** — o dobro do teto de 8,388 s do RP2040. O prazo nunca chegava
+a ser consultado.
+
+A correção substitui essa leitura por um laço próprio que honra o prazo **no
+meio da linha** e alimenta o watchdog **entre caracteres**, com o mesmo contrato
+(acumula até `\n`, não o inclui, devolve o que tem em timeout). O feed é seguro
+exatamente porque o prazo faz o laço terminar.
+
+Isso explica por que o `tls_slow20` caiu junto: o handshake TLS lento entra pelo
+mesmo `handleHeaderResponse`.
+
+**Sobre os "DATA-LOSS" que aparecem nessas corridas**: são o ruído já
+documentado. No `tls_slow20` o gap deu +660 s e no `tls_slow40`, no mesmo
+cenário, **−13 140 s** — fisicamente impossível. Ambos com `faultRecs=0` e
+`conns=0`: o servidor defeituoso não recebeu nada em nenhum dos dois, então não
+há assimetria que sustente perda em um e não no outro.
+
+### 7.4 D14 — o reboot escondia um vazamento de PBUF
+
+Sem os reboots do `huge1mb`, o aparelho sobrevive tempo suficiente para expor um
+segundo problema, independente e até então mascarado:
+
+```
+show net status → PBUF pool: 12 em uso / pico 12 / 12 total, 1158 falhas
+```
+
+O dispositivo continua vivo — IP, link a −42 dBm, CLI serial respondendo — mas o
+servidor web fica mudo por falta de buffer. **Não se recupera sozinho**: com a
+telemetria desligada o pool segue 12/12 e as falhas sobem (519 → 666). Só
+`reload confirm` devolve (`0 em uso / pico 1 / 12 total, 0 falhas`, web 200).
+
+Atribuí isso ao `stop( )` que eu havia acrescentado; removi, refiz e o vazamento
+continuou igual. A hipótese estava errada — o vazamento antecede as mudanças
+desta campanha, e o que mudou foi ele deixar de ser apagado a cada 8,4 s.
+
+Agravante: `PBUF_POOL_SIZE` foi reduzido de 24 para 12 no patch de lwIP do
+projeto, para economizar 18 KB de RAM. A margem é estreita.
+
+### 7.5 Estado da bancada ao final
+
+Configuração original restaurada (`192.168.3.206:8443/api.php`, HTTPS,
+intervalo 10 s, lote 50, JSON), usuário descartável `telb` removido, aparelho
+saudável: `IP 192.168.3.24 · RSSI −44 dBm · PBUF 1/12 · 0 falhas · web HTTP 200`.
diff --git a/src/NetworkManager.cpp b/src/NetworkManager.cpp
index 81c182b..714ccbb 100644
--- a/src/NetworkManager.cpp
+++ b/src/NetworkManager.cpp
@@ -277,9 +277,46 @@ void NetworkManager::update( ) {
} else {
/* Sample RSSI once per minute when connected. */
static uint32_t _rssiSampleAt = 0;
+ static uint8_t _rssiImplausible = 0;
if (timeSince(_rssiSampleAt, 60000)) {
_rssiSampleAt = millis( );
- MetricsManager::instance( ).observeRssi(WiFi.RSSI( ));
+ const int32_t rssi = WiFi.RSSI( );
+ MetricsManager::instance( ).observeRssi(rssi);
+
+ /* Second liveness signal, because WiFi.status( ) above is not one.
+ *
+ * Measured 2026-08-02: after a burst of large HTTP responses the device
+ * dropped off the network completely — host ARP went INCOMPLETE and ICMP
+ * got 100% loss — while WiFi.status( ) still said WL_CONNECTED. Nothing
+ * demoted the state, so nothing ever reconnected and the device stayed
+ * dark until a reboot, still reporting its IP.
+ *
+ * The tell was in the reading right here: RSSI came back as +4 dBm.
+ * Received signal strength is negative by definition, so a non-negative
+ * value means the cyw43 ioctl is no longer returning real data — and it
+ * is worse than useless, because isNetworkHealthy( ) compares it against
+ * RSSI_MIN_THRESHOLD and an impossible +4 sails past, actively
+ * confirming health.
+ *
+ * Two consecutive bad samples before acting: one could be a glitchy
+ * read, and a needless reconnect costs more than a minute of waiting.
+ * Detection lands within ~2 minutes instead of never. Sampling stays at
+ * once a minute on purpose — RSSI is a live ioctl and hammering it is
+ * its own hazard. */
+ if (rssi >= RSSI_IMPLAUSIBLE_HIGH || rssi < RSSI_IMPLAUSIBLE_LOW) {
+ if (++_rssiImplausible >= 2) {
+ LOG_CODE(LOG_WARN, "NET", SYS_WIFI_DISCONNECT, (int)rssi,
+ TRL("Implausible RSSI twice — link presumed dead, reconnecting"));
+ _rssiImplausible = 0;
+ WiFi.disconnect(false);
+ _reconnectDelay = 5000;
+ resetNtpBackoff( );
+ _state = NET_DISCONNECT_PENDING;
+ _stateTimer = millis( );
+ }
+ } else {
+ _rssiImplausible = 0;
+ }
}
}
break;
@@ -437,7 +474,15 @@ void NetworkManager::getMacAddress(char* buf, size_t len) {
String NetworkManager::getSubnetMask( ) { return WiFi.subnetMask( ).toString( ); }
String NetworkManager::getGateway( ) { return WiFi.gatewayIP( ).toString( ); }
String NetworkManager::getDns( ) { return WiFi.dnsIP( ).toString( ); }
-int32_t NetworkManager::getRssi( ) { return (!isConnected( )) ? -100 : WiFi.RSSI( ); }
+int32_t NetworkManager::getRssi( ) {
+ if (!isConnected( )) return -100;
+ /* An implausible reading means the radio is not answering with real data, so
+ * report it as a dead link rather than letting a positive dBm pass the
+ * RSSI_MIN_THRESHOLD gate in isNetworkHealthy( ). See the NET_READY case. */
+ const int32_t rssi = WiFi.RSSI( );
+ if (rssi >= RSSI_IMPLAUSIBLE_HIGH || rssi < RSSI_IMPLAUSIBLE_LOW) return -100;
+ return rssi;
+}
String NetworkManager::getFormattedTime( ) {
time_t now = getEpoch( );
diff --git a/src/SystemDefs_Network.h b/src/SystemDefs_Network.h
index 57d6394..5151b34 100644
--- a/src/SystemDefs_Network.h
+++ b/src/SystemDefs_Network.h
@@ -48,6 +48,16 @@ constexpr uint32_t NET_TLS_HANDSHAKE_MS = 15000;
*/
constexpr int32_t RSSI_MIN_THRESHOLD = -78;
+/* Plausible range for a received-signal RSSI, in dBm. Anything outside it means
+ * the cyw43 ioctl is not returning real data — measured on the bench as +4 dBm
+ * while the device was completely off the network (host ARP INCOMPLETE, 100%
+ * ICMP loss) and WiFi.status( ) still reported WL_CONNECTED. Used by
+ * NetworkManager as a second liveness signal, because a non-negative reading
+ * otherwise sails past RSSI_MIN_THRESHOLD and confirms health instead of
+ * denying it. */
+constexpr int32_t RSSI_IMPLAUSIBLE_HIGH = 0;
+constexpr int32_t RSSI_IMPLAUSIBLE_LOW = -120;
+
#ifdef SIMUT_MDNS
/** Minimum interval between MDNS.update() calls (ms). */
constexpr uint32_t MDNS_UPDATE_INTERVAL_MS = 2000;
diff --git a/src/TelemetryManager.cpp b/src/TelemetryManager.cpp
index 5cd914e..8eb74f7 100644
--- a/src/TelemetryManager.cpp
+++ b/src/TelemetryManager.cpp
@@ -44,6 +44,13 @@
* @param minDay Data limite no formato "YYYYMMDD".
* @return true se deve ser pulado.
*/
+/* Largest MQTT buffer the client will be asked for, and the fixed-header plus
+ * topic-length overhead a PUBLISH carries on top of its payload. Named because
+ * the ceiling is the difference between "this batch is published record by
+ * record" and "telemetry stops forever" — see attemptMqttPublish. */
+static constexpr size_t MQTT_BUFFER_CEILING = 8192;
+static constexpr size_t MQTT_PACKET_OVERHEAD = 16;
+
static bool historyDayIsBefore(const String &fileName, const char *minDay) {
if (fileName.length( ) < 8) return false;
for (int i = 0; i < 8; i++) {
@@ -281,6 +288,12 @@ void TelemetryManager::update( ) {
_dumpPayload(payload.c_str( ), payload.length( ), "MQTT");
_dumpPayloadNext = false;
}
+ /* buildPayload can drop records off the end under heap pressure. The
+ * cursor has to follow what the payload actually carries, or those
+ * records are skipped for good, silently. The batch is in ascending
+ * epoch order (files sorted, records in time order within a file), so
+ * the last surviving element is the high-water mark. */
+ if (!batch.empty( )) newCursor = batch.back( ).epoch;
success = attemptMqttPublish(payload, batch, newCursor);
/* batch and payload go out of scope here and free memory */
} else {
@@ -295,6 +308,10 @@ void TelemetryManager::update( ) {
_dumpPayloadNext = false;
}
+ /* Same reason as the MQTT branch above: read the high-water mark off the
+ * batch buildPayload left behind, before it is thrown away. */
+ if (!batch.empty( )) newCursor = batch.back( ).epoch;
+
/* Free batch to reduce RAM peak before TLS handshake */
batch.clear( );
batch.shrink_to_fit( );
@@ -634,8 +651,9 @@ bool TelemetryManager::attemptHttpUpload(String& payload, uint32_t newCursor) {
watchdog_update( );
if (code > 0) {
- LOG_CODE(LOG_INFO, "TEL", SYS_TEL_SENT, code, "HTTP OK: " + String(payload.length( )) + " bytes, code " + String(code));
if (code >= 200 && code < 300) {
+ LOG_CODE(LOG_INFO, "TEL", SYS_TEL_SENT, code,
+ "HTTP OK: " + String(payload.length( )) + " bytes, code " + String(code));
_storageRef->setLastSentTimestamp(newCursor);
success = true;
auto& m = MetricsManager::instance( ).data( );
@@ -646,11 +664,43 @@ bool TelemetryManager::attemptHttpUpload(String& payload, uint32_t newCursor) {
_smoothedLatencyMs = (_smoothedLatencyMs == 0)
? postLatency
: (_smoothedLatencyMs * 7 + (uint32_t)postLatency * 3) / 10;
+ } else {
+ /* A reply that arrived is not a delivery. This branch used to fall
+ * through the same INFO line as success — a server answering 500 to
+ * every batch logged "HTTP OK ... code 500" and left both telSent and
+ * telFailed untouched, so the dashboard read "Falhas: 0" while nothing
+ * was getting through. The cursor was already held back correctly; what
+ * was missing was saying so. */
+ LOG_CODE(LOG_ERROR, "TEL", SYS_TEL_FAIL, code,
+ "HTTP rejected: " + String(payload.length( )) + " bytes, code " + String(code));
+ MetricsManager::instance( ).data( ).telFailed++;
}
} else {
LOG_CODE(LOG_ERROR, "TEL", SYS_TEL_FAIL, code, String(TRL("HTTP error: ")) + http.errorToString(code));
MetricsManager::instance( ).data( ).telFailed++;
}
+
+ /* Close the socket before end( ).
+ *
+ * Everything this cycle needs is the status code, already read. Leaving the
+ * connection open hands it to HTTPClient::disconnect( ), which drains
+ * whatever the peer is still sending so the socket stays reusable — and
+ * against a peer that never stops sending, that path still reaches the
+ * 8.388 s watchdog even with the framework deadline in place.
+ *
+ * Measured, A/B, same servers and same windows:
+ *
+ * with this stop( ) huge1mb 0 reboots, drip 0 reboots
+ * without this stop( ) huge1mb 0 reboots, drip 3 reboots + [FTL]
+ *
+ * It was removed once, on the theory that closing without reading was what
+ * exhausted the lwIP pbuf pool. That theory was wrong: the pool is exhausted
+ * in BOTH builds (D14 — a separate defect the watchdog reboots used to hide),
+ * and removing the stop( ) only brought the drip kill back. Put it back.
+ */
+ if (cfg.telEncryption) { if (_httpSecurePtr) _httpSecurePtr->stop( ); }
+ else client.stop( );
+
http.end( );
}
@@ -773,7 +823,18 @@ bool TelemetryManager::mqttEnsureConnected( ) {
bool TelemetryManager::attemptMqttPublish(String& payload, std::vector& batch, uint32_t newCursor) {
if (!_mqttInitialized) return false;
- if (!mqttEnsureConnected( )) return false;
+ /* Metrics are recorded here for the same reason attemptHttpUpload records
+ * them: without it this whole transport is invisible. Measured on the bench
+ * — 386 publishes carrying 384 records, and telSent / telFailed / telBytes /
+ * telLastLatencyMs all still read zero, so the dashboard and `show metrics`
+ * said nothing had ever been sent. The functional half of that is worse than
+ * the cosmetic one: update( ) derives its effective interval from
+ * _smoothedLatencyMs, which only the HTTP path was feeding, so the adaptive
+ * pacing never engaged on MQTT at all. */
+ const uint32_t pubStart = millis( );
+ auto& m = MetricsManager::instance( ).data( );
+
+ if (!mqttEnsureConnected( )) { m.telFailed++; return false; }
SystemConfig &cfg = _storageRef->getConfig( );
String topic = String(cfg.mqttTopic);
@@ -782,6 +843,7 @@ bool TelemetryManager::attemptMqttPublish(String& payload, std::vector MQTT_BUFFER_CEILING) {
+ LOG_CODE(LOG_WARN, "TEL", SYS_TEL_QUEUE, (int)batch.size( ),
+ "MQTT payload " + String(payload.length( )) +
+ " B over buffer ceiling — publishing per record");
+ int published = 0;
+ for (size_t i = 0; i < batch.size( ); i++) {
+ feedWdt( );
+ String linePayload;
+ if (cfg.telMode == TEL_MODE_JSON) {
+ linePayload = formatLineJson(batch[i], cfg);
+ } else if (cfg.telMode == TEL_MODE_CSV) {
+ char csvBuf[256];
+ batch[i].toCsvLine(csvBuf, sizeof(csvBuf));
+ linePayload = String(csvBuf);
+ } else {
+ linePayload = formatLineCustom(batch[i], cfg);
+ }
+ if (!_mqttClient.publish(topic.c_str( ), linePayload.c_str( ), cfg.mqttRetain)) break;
+ published++;
+ sentBytes += (uint32_t)linePayload.length( );
+ _mqttClient.loop( );
+ }
+ if (published > 0) {
+ LOG_CODE(LOG_INFO, "TEL", SYS_TEL_MQTT_PUB, published,
+ "MQTT split publish " + String(published) + "/" + String(batch.size( )));
+ _storageRef->setLastSentTimestamp(batch[published - 1].epoch);
+ success = (published == (int)batch.size( ));
+ }
+ } else {
+
if (payload.length( ) > _mqttClient.getBufferSize( )) {
- uint16_t needed = min((size_t)8192, payload.length( ) + 64);
- _mqttClient.setBufferSize(needed);
+ _mqttClient.setBufferSize((uint16_t)min((size_t)MQTT_BUFFER_CEILING,
+ payload.length( ) + 64));
}
bool ok;
@@ -847,12 +957,29 @@ bool TelemetryManager::attemptMqttPublish(String& payload, std::vectorsetLastSentTimestamp(newCursor);
+ sentBytes = (uint32_t)payload.length( );
success = true;
} else {
LOG_CODE(LOG_ERROR, "TEL", SYS_TEL_FAIL, _mqttClient.state( ),
"MQTT publish failed (payload " + String(payload.length( )) + " bytes)");
}
}
+ }
+
+ /* Same bookkeeping attemptHttpUpload does, so the two transports report
+ * through the same counters and the dashboard means the same thing whichever
+ * one is configured. */
+ const uint32_t pubLatency = millis( ) - pubStart;
+ if (success) {
+ m.telSent++;
+ m.telTotalBytes += sentBytes;
+ m.telLastLatencyMs = pubLatency;
+ _smoothedLatencyMs = (_smoothedLatencyMs == 0)
+ ? pubLatency
+ : (_smoothedLatencyMs * 7 + pubLatency * 3) / 10;
+ } else {
+ m.telFailed++;
+ }
return success;
}
@@ -941,6 +1068,9 @@ bool TelemetryManager::forceSync( ) {
_dumpPayloadNext = false;
}
+ /* Same as update( ): the cursor follows the payload, not the collection. */
+ if (!batch.empty( )) newCursor = batch.back( ).epoch;
+
bool ok;
if (cfg.telTransport == TEL_TRANSPORT_MQTT) {
ok = attemptMqttPublish(payload, batch, newCursor);
@@ -974,9 +1104,13 @@ String TelemetryManager::buildPayload(std::vector& batch) {
LogManager::TraceScope _tB(0, MOD_TEL_BUILD);
SystemConfig &cfg = _storageRef->getConfig( );
- /* Estimate size: JSON ~300 bytes/record with 12 sensors */
+ /* Estimate size: JSON ~300 bytes/record with 12 sensors.
+ * CSV needs a bigger fixed part: its header names all 34 columns of the
+ * row layout (~440 B), which does not fit in the 256 B slack the other
+ * modes use and would force the String to reallocate on every batch. */
size_t perLine = (cfg.telMode == TEL_MODE_CSV) ? 120 : 300;
- size_t estimatedSize = batch.size( ) * perLine + 256;
+ size_t fixedPart = (cfg.telMode == TEL_MODE_CSV) ? 640 : 256;
+ size_t estimatedSize = batch.size( ) * perLine + fixedPart;
/* Check heap and reduce batch if needed.
* Differentiated reserve by TLS — 12K with encryption,
@@ -990,7 +1124,7 @@ String TelemetryManager::buildPayload(std::vector& batch) {
batch.resize(safeCount);
batch.shrink_to_fit( ); /* release effective capacity */
}
- estimatedSize = batch.size( ) * perLine + 256;
+ estimatedSize = batch.size( ) * perLine + fixedPart;
}
String s;
@@ -1012,16 +1146,29 @@ String TelemetryManager::buildPayload(std::vector& batch) {
}
s.concat(']');
} else if (cfg.telMode == TEL_MODE_CSV) {
- /* Header matches toCsvLine, which dropped the ambT;ambH pair with the
- * ambient slot. */
+ /* The header has to name every column toCsvLine emits, and toCsvLine emits
+ * the fixed layout `epoch;s0..s15;h0..h15;press` — all 16 slots, active or
+ * not, then all 16 humidities, then pressure. Naming only the active slots
+ * produced a 7-column header over 34-column rows, so anything reading by
+ * header index read the wrong values. The rows are the persisted, upload-
+ * compatible format and do not change; the header was what lied. */
s = "timestamp";
char hdrBuf[32];
for (int i = 0; i < MAX_SENSORS; i++) {
- if (cfg.sensors[i].active) {
+ if (cfg.sensors[i].active && cfg.sensors[i].hwId[0])
snprintf(hdrBuf, sizeof(hdrBuf), ";s%d_%s", i, cfg.sensors[i].hwId);
+ else
+ snprintf(hdrBuf, sizeof(hdrBuf), ";s%d", i);
s.concat(hdrBuf);
}
+ for (int i = 0; i < MAX_SENSORS; i++) {
+ if (cfg.sensors[i].active && cfg.sensors[i].hwId[0])
+ snprintf(hdrBuf, sizeof(hdrBuf), ";h%d_%s", i, cfg.sensors[i].hwId);
+ else
+ snprintf(hdrBuf, sizeof(hdrBuf), ";h%d", i);
+ s.concat(hdrBuf);
}
+ s.concat(";press");
s.concat('\n');
char csvBuf[256];
for (size_t i = 0; i < batch.size( ); i++) {
@@ -1425,6 +1572,14 @@ void TelemetryManager::refreshPendingCount( ) {
uint32_t lastCursor = _storageRef->getLastSentTimestamp( );
+ /* The same 30-day floor collectBatch applies when the cursor is zero.
+ * Without it, the count right after `tel reset` includes every record on
+ * flash — including the ones the sender will never reach — so the dashboard
+ * shows a backlog that can only ever shrink to a non-zero number. */
+ if (lastCursor == 0) {
+ uint32_t lastRecorded = _storageRef->getLastRecordedTimestamp( );
+ if (lastRecorded > 86400UL * 30) lastCursor = lastRecorded - 86400UL * 30;
+ }
std::vector files;
{
@@ -1453,7 +1608,11 @@ void TelemetryManager::refreshPendingCount( ) {
timeinfo.tm_year + 1900, timeinfo.tm_mon + 1, timeinfo.tm_mday);
}
- uint16_t total = 0;
+ /* 32-bit accumulator, saturated on the way out. It used to be uint16_t with
+ * an explicit cast on every add, so an archive holding more than 65535
+ * pending records wrapped to a plausible-looking wrong number on the
+ * dashboard — this bench holds ~119k. */
+ uint32_t total = 0;
/* Same reason as collectBatch — was reading 28 B raw
@@ -1493,7 +1652,7 @@ void TelemetryManager::refreshPendingCount( ) {
if (!got) break;
if (hdr.t0 > lastCursor) {
- total = (uint16_t)(total + hdr.pre.a);
+ total += hdr.pre.a;
} else {
straddleT0 = hdr.t0;
straddleCount = hdr.pre.a;
@@ -1522,7 +1681,7 @@ void TelemetryManager::refreshPendingCount( ) {
feedWdt( );
}
- _pendingEstimate = total;
+ _pendingEstimate = (total > 0xFFFFu) ? (uint16_t)0xFFFFu : (uint16_t)total;
_pendingDirty = false;
}
diff --git a/src/WebManager_Commit.cpp b/src/WebManager_Commit.cpp
index 4efba86..202407a 100644
--- a/src/WebManager_Commit.cpp
+++ b/src/WebManager_Commit.cpp
@@ -570,6 +570,16 @@ void WebManager::handleApiCommitAll( ) {
if (has("m_topic")) safeCopy(cfg.mqttTopic, getStr("m_topic").c_str( ), sizeof(cfg.mqttTopic));
if (has("m_cid")) safeCopy(cfg.mqttClientId, getStr("m_cid").c_str( ), sizeof(cfg.mqttClientId));
if (has("m_user")) safeCopy(cfg.mqttUser, getStr("m_user").c_str( ), sizeof(cfg.mqttUser));
+ /* The page has always had this field and the browser has always sent
+ * it (every input with an id inside #sysForm stages into Pending.sys)
+ * — nothing here read it, so the MQTT password went in the bin and
+ * a broker that wants credentials could never be reached. Empty
+ * means keep, which is what the field's placeholder promises and
+ * what t_key already does with its "***" mask. */
+ if (has("m_pass")) {
+ String mp = getStr("m_pass");
+ if (mp.length( ) > 0) safeCopy(cfg.mqttPass, mp.c_str( ), sizeof(cfg.mqttPass));
+ }
if (has("m_qos")) { int v; if (parseIntStrict(getNum("m_qos"), v) && isInRange(v, 0, 2)) cfg.mqttQos = (uint8_t)v; }
if (has("m_retain")) cfg.mqttRetain = (getNum("m_retain") != "0");
if (has("m_ka")) { int v; if (parseIntStrict(getNum("m_ka"), v) && isInRange(v, 5, 600)) cfg.mqttKeepAlive = (uint16_t)v; }
diff --git a/src/WebManager_Files.cpp b/src/WebManager_Files.cpp
index 53aa43c..680b9cb 100644
--- a/src/WebManager_Files.cpp
+++ b/src/WebManager_Files.cpp
@@ -161,12 +161,19 @@ void WebManager::handleApiLs( ) {
HeavyTaskGuard htg(_storageRef);
if (!htg.isLocked( )) { _server.send(503, "application/json", "{\"error\":\"System Busy\"}"); return; }
+ /* Listing a big directory belongs with the other long handlers: /history
+ * holds one file per day, and every entry costs a flash read plus a chunked
+ * write, which contends with telemetry for the same flash lock. At the
+ * default 6 s budget an archive of ~90 days was being cut off mid-scan. */
+ const uint32_t savedDeadline = _handlerDeadline;
+ _handlerDeadline = millis( ) + WEB_LONG_HANDLER_DEADLINE_MS;
+
_server.setContentLength(CONTENT_LENGTH_UNKNOWN);
_server.send(200, "application/json", "");
char buf[256];
snprintf(buf, sizeof(buf), "{\"path\":\"%s\",\"entries\":[", dirPath.c_str( ));
- if (!safeSend(buf)) return;
+ if (!safeSend(buf)) { _handlerDeadline = savedDeadline; return; }
bool first = true;
@@ -183,8 +190,15 @@ void WebManager::handleApiLs( ) {
dir = LittleFS.openDir(dirPath);
}
+ /* Set when the enumeration is cut short, and reported in the body. A caller
+ * that cannot tell a partial listing from a complete one will treat missing
+ * files as deleted: two listings of the same /history minutes apart came
+ * back with 84 entries each and DIFFERENT contents, and both looked like
+ * well-formed, finished JSON. */
+ bool truncated = false;
+
while (!dirDone) {
- if (isHandlerOvertime( )) break;
+ if (isHandlerOvertime( )) { truncated = true; break; }
struct DirEntry { String name; size_t size; bool isDir; };
DirEntry batch[20];
@@ -192,7 +206,15 @@ void WebManager::handleApiLs( ) {
{
ReadGuard rg(_storageRef);
- while (dir.next( ) && batchCount < 20) {
+ /* Count first, THEN advance. Written the other way round —
+ * `dir.next( ) && batchCount < 20` — the iterator moves before the count is
+ * tested, so the entry that would have filled the batch is consumed and
+ * never recorded: every full batch of 20 silently lost one file. Measured:
+ * 88 files on flash, 84 in the listing, and because LittleFS iteration
+ * order varies, two listings minutes apart dropped DIFFERENT files
+ * (20260523 vs 20260524 — both present, both intact, neither ever shown
+ * together). Nothing in the response said anything was missing. */
+ while (batchCount < 20 && dir.next( )) {
/* feedWdt( ) under the read lock — feedWatchdog( ) would run the light yield,
* which reaches enterFlashReadLock( ) on this same core and self-deadlocks on
* a non-recursive mutex. See the note in handleApiHistoryDays. */
@@ -218,7 +240,7 @@ void WebManager::handleApiLs( ) {
jsonEscapeFilename(dName, dEscaped, sizeof(dEscaped));
snprintf(buf, sizeof(buf), "%s{\"n\":\"%s\",\"t\":\"d\",\"s\":0}",
first ? "" : ",", dEscaped);
- if (!safeSend(buf)) return;
+ if (!safeSend(buf)) { _handlerDeadline = savedDeadline; return; }
first = false;
continue;
}
@@ -241,14 +263,15 @@ void WebManager::handleApiLs( ) {
snprintf(buf, sizeof(buf), "%s{\"n\":\"%s\",\"t\":\"f\",\"s\":%u%s}",
first ? "" : ",", escaped, (unsigned)batch[i].size,
prot ? ",\"p\":1" : "");
- if (!safeSend(buf)) return;
+ if (!safeSend(buf)) { _handlerDeadline = savedDeadline; return; }
first = false;
}
feedWatchdog( );
}
- safeSend("]}");
+ safeSend(truncated ? "],\"truncated\":true}" : "]}");
+ _handlerDeadline = savedDeadline;
}
void WebManager::handleApiMkdir( ) {
diff --git a/tools/arduino_pico_overrides/patch.sh b/tools/arduino_pico_overrides/patch.sh
index 5b467e6..7206ba7 100755
--- a/tools/arduino_pico_overrides/patch.sh
+++ b/tools/arduino_pico_overrides/patch.sh
@@ -85,6 +85,66 @@ else
patch -p1 -d "$FW" < "$BEARSSL_PATCH"
fi
+# 2c. Prazos nos laços de leitura do HTTPClient (HTTPClient.cpp)
+#
+# Dois laços do upstream não têm limite superior nenhum:
+#
+# - handleHeaderResponse(): _tcpTimeout é prazo de INATIVIDADE (lastDataTime
+# reinicia a cada linha recebida), então um servidor que responde 1 byte a
+# cada 400 ms nunca o dispara e o laço roda para sempre.
+# - disconnect(): drena o corpo byte a byte "até available() zerar". Contra um
+# peer que ainda está transmitindo, o socket enche na mesma velocidade em
+# que esvazia, e o laço percorre o corpo inteiro.
+#
+# Medido na bancada em 2026-08-02: resposta de 1 MB = 4 reboots por watchdog
+# em 2 min, autópsia C0=[TEL_SEND] em todos. Um servidor de telemetria
+# defeituoso (ou hostil) derrubava o aparelho em laço.
+#
+# Patch: um orçamento único em cada laço + watchdog_update() dentro — seguro
+# exatamente porque o orçamento faz os dois terminarem. Mesmo padrão do
+# prazo de handshake TLS acima.
+HTTPC="$FW/libraries/HTTPClient/src/HTTPClient.cpp"
+HTTPC_PATCH="$OVR/patches/httpclient_read_deadlines.patch"
+if [ ! -f "$OVR/originals/HTTPClient.cpp" ]; then
+ mkdir -p "$OVR/originals"
+ cp -v "$HTTPC" "$OVR/originals/"
+fi
+if grep -q "SIMUT override — bound the header read as a whole" "$HTTPC"; then
+ echo "[patch] HTTPClient já tem prazos de leitura — nada a fazer"
+else
+ echo "[patch] aplicando prazos nos laços de leitura do HTTPClient"
+ patch -p1 -d "$FW" < "$HTTPC_PATCH"
+fi
+
+# 2d. Vazamento de pbufs de recepcao no ClientContext (ClientContext.h)
+#
+# close( ) e abort( ) desanexam todos os callbacks e largam o _pcb, mas
+# deixavam _rx_buf intacto. Sem o pcb esse dado nao pode mais ser entregue a
+# ninguem, entao os pbufs do pool ficam presos — e WiFiClient::stop( ) chama
+# close( ) SEM soltar o ClientContext, entao o refcount nunca chega ao
+# unref( ) que teria feito o discard.
+#
+# Medido na bancada em 2026-08-02 contra um servidor que responde 1 MB: apos
+# ~19 conexoes, "PBUF pool: 12 em uso / pico 12 / 12 total, 1158 falhas". O
+# aparelho seguia vivo (IP, link -42 dBm, CLI serial respondendo) mas o
+# servidor web ficava mudo, e NAO se recuperava: com a telemetria desligada o
+# pool continuava 12/12 e as falhas subindo. So reboot devolvia.
+#
+# O pool tem so 12 entradas (PBUF_POOL_SIZE 24->12 no lwipopts patchado, para
+# economizar 18 KB de RAM), entao a margem e estreita.
+CTXH="$FW/libraries/WiFi/src/include/ClientContext.h"
+CTXH_PATCH="$OVR/patches/clientcontext_rx_leak.patch"
+if [ ! -f "$OVR/originals/ClientContext.h" ]; then
+ mkdir -p "$OVR/originals"
+ cp -v "$CTXH" "$OVR/originals/"
+fi
+if grep -q "SIMUT override: release buffered RX before abandoning the pcb" "$CTXH"; then
+ echo "[patch] ClientContext ja libera o RX no close/abort — nada a fazer"
+else
+ echo "[patch] aplicando liberacao de _rx_buf no close/abort do ClientContext"
+ patch -p1 -d "$FW" < "$CTXH_PATCH"
+fi
+
# 3. Invalida cache PIO (lwip src + lib WiFi)
# A lib WiFi tem cache próprio em lib*/WiFi/ — sem apagá-lo o .cpp patchado
# não recompila e o build "passa" ainda com o handshake sem prazo.
@@ -94,12 +154,23 @@ for build in "$ROOT/.pio/build"/*/FrameworkArduino/lwip; do
echo "[patch] cache invalidado: $build"
fi
done
-for wifiobj in "$ROOT/.pio/build"/*/lib*/WiFi/WiFiClientSecureBearSSL.cpp.o; do
+# Todos os .o da lib WiFi, nao so o do BearSSL: ClientContext.h e um HEADER
+# incluido por varios deles, entao apagar um objeto so deixaria o vazamento de
+# pbuf linkado enquanto o build "passa".
+for wifiobj in "$ROOT/.pio/build"/*/lib*/WiFi/*.o; do
if [ -f "$wifiobj" ]; then
rm -f "$wifiobj"
echo "[patch] cache invalidado: $wifiobj"
fi
done
+# HTTPClient tem cache próprio pelo mesmo motivo da lib WiFi: sem apagar o .o,
+# o build "passa" ainda linkando os laços de leitura sem prazo.
+for httpobj in "$ROOT/.pio/build"/*/lib*/HTTPClient/HTTPClient.cpp.o; do
+ if [ -f "$httpobj" ]; then
+ rm -f "$httpobj"
+ echo "[patch] cache invalidado: $httpobj"
+ fi
+done
echo ""
echo "[patch] DONE. Próximo \`pio run\` recompila lwIP com PBUF_POOL_SIZE=12."
diff --git a/tools/arduino_pico_overrides/patched_headers/lwipopts.h b/tools/arduino_pico_overrides/patched_headers/lwipopts.h
index 9950cc5..a445dfb 100644
--- a/tools/arduino_pico_overrides/patched_headers/lwipopts.h
+++ b/tools/arduino_pico_overrides/patched_headers/lwipopts.h
@@ -34,11 +34,20 @@ extern unsigned long __lwip_rand(void);
#define MEM_SIZE (__LWIP_MEMMULT * 16384)
#define MEMP_NUM_TCP_SEG (32)
#define MEMP_NUM_ARP_QUEUE (10)
-/* SIMUT patch: 24 → 12 (save ~18 KB BSS). 1-2 conexões TCP simultâneas no
- * device típico — não satura mesmo com 12 envelopes pré-alocados. UDP/BT
- * defaults preservados (reduzir UDP_PCB quebra mDNS, BT changes quebram
- * RSSI via cyw43 chip compartilhado). */
-#define PBUF_POOL_SIZE (__LWIP_MEMMULT > 1 ? 32 : 12)
+/* SIMUT: de volta ao default 24, revertendo o corte para 12.
+ *
+ * A justificativa do corte era "1-2 conexões TCP simultâneas no device típico
+ * — não satura mesmo com 12 envelopes". Medido em 2026-08-02, satura: contra
+ * um servidor que responde 1 MB, o pool ia a 12/12 e o servidor web ficava
+ * mudo, sem recuperar. Parte disso era um vazamento de _rx_buf no
+ * ClientContext (corrigido em patches/clientcontext_rx_leak.patch, que sozinho
+ * levou o aparelho de ~16 para ~144 conexões antes de saturar), mas resta uma
+ * segunda fonte não localizada, e com 12 entradas a margem para ela é nenhuma.
+ *
+ * Custo: ~18 KB de BSS. Com o firmware em ~30% de RAM, cabe.
+ * UDP/BT seguem em default (reduzir UDP_PCB quebra mDNS; mexer em BT quebra o
+ * RSSI via chip cyw43 compartilhado). */
+#define PBUF_POOL_SIZE (__LWIP_MEMMULT > 1 ? 32 : 24)
#define LWIP_ARP 7
#define LWIP_ETHERNET 1
#define LWIP_ICMP 1
diff --git a/tools/arduino_pico_overrides/patches/clientcontext_rx_leak.patch b/tools/arduino_pico_overrides/patches/clientcontext_rx_leak.patch
new file mode 100644
index 0000000..efc4eff
--- /dev/null
+++ b/tools/arduino_pico_overrides/patches/clientcontext_rx_leak.patch
@@ -0,0 +1,42 @@
+--- a/libraries/WiFi/src/include/ClientContext.h
++++ b/libraries/WiFi/src/include/ClientContext.h
+@@ -66,6 +66,9 @@
+ err_t abort() {
+ if (_pcb) {
+ DEBUGV(":abort\r\n");
++ /* SIMUT override: release buffered RX before abandoning the pcb.
++ * See the note above close( ) — same leak, same reason. */
++ discard_received();
+ tcp_arg(_pcb, nullptr);
+ tcp_sent(_pcb, nullptr);
+ tcp_recv(_pcb, nullptr);
+@@ -81,6 +84,29 @@
+ err_t err = ERR_OK;
+ if (_pcb) {
+ DEBUGV(":close\r\n");
++ /* SIMUT override: release buffered RX before abandoning the pcb.
++ *
++ * close( ) detaches every callback and drops _pcb, but left _rx_buf
++ * alone. Once the pcb is gone that data can never be delivered to
++ * anyone, so the pool pbufs holding it are simply leaked — and
++ * WiFiClient::stop( ) calls close( ) without dropping the
++ * ClientContext, so the refcount never reaches the unref( ) path
++ * that would have discarded them.
++ *
++ * Measured on the bench against a server answering with a 1 MB
++ * body: after ~19 connections `show net status` reported
++ * "PBUF pool: 12 em uso / pico 12 / 12 total, 1158 falhas". The
++ * device kept its IP and a -42 dBm link and the serial CLI kept
++ * answering, but the web server went silent with nothing left to
++ * build a packet from, and it did not recover — with telemetry
++ * switched off entirely the pool stayed at 12/12 and the failure
++ * count kept climbing. Only a reboot returned it.
++ *
++ * Reading after the PEER closes is unaffected: that path leaves
++ * _pcb alone and does not come through here. close( ) means WE are
++ * done with the connection.
++ */
++ discard_received();
+ tcp_arg(_pcb, nullptr);
+ tcp_sent(_pcb, nullptr);
+ tcp_recv(_pcb, nullptr);
diff --git a/tools/arduino_pico_overrides/patches/httpclient_read_deadlines.patch b/tools/arduino_pico_overrides/patches/httpclient_read_deadlines.patch
new file mode 100644
index 0000000..4acba01
--- /dev/null
+++ b/tools/arduino_pico_overrides/patches/httpclient_read_deadlines.patch
@@ -0,0 +1,116 @@
+--- a/libraries/HTTPClient/src/HTTPClient.cpp
++++ b/libraries/HTTPClient/src/HTTPClient.cpp
+@@ -27,6 +27,7 @@
+ #include "HTTPClient.h"
+ #include
+ #include
++#include /* SIMUT override: feed inside bounded read loops */
+ #include "base64.h"
+ extern "C" char *strptime(const char *__restrict, const char *__restrict, struct tm *__restrict); // Not exposed by headers?
+
+@@ -348,10 +349,33 @@
+ */
+ void HTTPClient::disconnect(bool preserveClient) {
+ if (connected()) {
++ /* SIMUT override — bound the drain.
++ *
++ * Upstream reads until available() reports empty, one byte at a time,
++ * with no deadline. Against a peer that is still streaming, the socket
++ * refills as fast as it drains, so this loop runs for the whole body:
++ * measured on the bench, a 1 MB response rebooted the RP2040 four
++ * times in two minutes via the 8.388 s hardware watchdog, with the
++ * autopsy pointing at C0=[TEL_SEND] every time. The response had
++ * already been parsed — this loop exists only to leave the socket
++ * reusable, which is never worth the device.
++ *
++ * Read in blocks, give up after _tcpTimeout, and feed the watchdog:
++ * the feed is safe because the deadline makes the loop provably
++ * terminate. Whatever is left unread, stop() below discards.
++ */
+ if (_client()->available() > 0) {
+ DEBUG_HTTPCLIENT("[HTTP-Client][end] still data in buffer (%d), clean up.\n", _client()->available());
++ const unsigned long drainStart = millis();
++ uint8_t sink[64];
+ while (_client()->available() > 0) {
+- _client()->read();
++ if ((millis() - drainStart) > _tcpTimeout) {
++ DEBUG_HTTPCLIENT("[HTTP-Client][end] drain deadline hit, dropping socket\n");
++ _canReuse = false;
++ break;
++ }
++ watchdog_update();
++ _client()->read(sink, sizeof(sink));
+ }
+ }
+
+@@ -1085,11 +1109,69 @@
+ unsigned long lastDataTime = millis();
+ String date;
+
++ /* SIMUT override — bound the header read as a whole.
++ *
++ * _tcpTimeout below is an INACTIVITY timeout: lastDataTime is reset on
++ * every line received, so a peer that trickles the response never trips
++ * it and this loop has no upper bound at all. Measured on the bench, a
++ * server answering one byte every 400 ms held Core 0 here past the
++ * RP2040's 8.388 s watchdog ceiling and rebooted the device.
++ *
++ * One budget across the whole response, plus a watchdog feed that is safe
++ * precisely because the budget makes the loop terminate. Twice the
++ * inactivity timeout leaves room for a legitimately slow-but-progressing
++ * server while still cutting off a trickle well inside a bounded time.
++ */
++ const unsigned long headerStart = millis();
++ const unsigned long headerBudget = _tcpTimeout * 2;
++
+ while (connected()) {
++ if ((millis() - headerStart) > headerBudget) {
++ DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] overall timeout\n");
++ _canReuse = false;
++ return HTTPC_ERROR_READ_TIMEOUT;
++ }
++ watchdog_update();
+ size_t len = _client()->available();
+ if (len > 0) {
+ int headerSeparator = -1;
+- String headerLine = _client()->readStringUntil('\n');
++ /* SIMUT override — read the line ourselves.
++ *
++ * readStringUntil( ) is ONE call that blocks character by character
++ * with nothing feeding the watchdog inside it, so the budget check
++ * above — which only runs between lines — cannot save a line that
++ * is slow on its own. Measured: a peer trickling one byte every
++ * 400 ms puts `Content-Type: application/json\r\n` (32 bytes) at
++ * 12.8 s inside a single readStringUntil, well past the RP2040's
++ * 8.388 s ceiling, and the device reboots mid-header.
++ *
++ * Same contract as readStringUntil('\n'): accumulate up to the
++ * newline, do not include it, give back what we have on timeout.
++ * The difference is that the overall budget and the watchdog feed
++ * now apply between characters too. */
++ String headerLine;
++ {
++ unsigned long charWait = millis();
++ while (true) {
++ if ((millis() - headerStart) > headerBudget) {
++ break;
++ }
++ watchdog_update();
++ int c = _client()->read();
++ if (c < 0) {
++ if ((millis() - charWait) > _tcpTimeout) {
++ break;
++ }
++ delay(1);
++ continue;
++ }
++ charWait = millis();
++ if (c == '\n') {
++ break;
++ }
++ headerLine += (char)c;
++ }
++ }
+
+ lastDataTime = millis();
+
diff --git a/tools/telemetry_bench/README.md b/tools/telemetry_bench/README.md
new file mode 100644
index 0000000..c0a87f1
--- /dev/null
+++ b/tools/telemetry_bench/README.md
@@ -0,0 +1,20 @@
+# Bancada de telemetria
+
+Servidores instrumentados e orquestração usados na campanha de 2026-08-02
+(ver `docs/telemetry-campaign-2026-08-02/`). Python 3 + pyserial + requests.
+
+- `server_http.py` — sink HTTP/HTTPS com injeção de falha (`--mode`, `--tls-fault`)
+- `server_mqtt.py` — broker MQTT 3.1.1 escrito do formato de fio, para poder
+ mentir (CONNACK recusado, meio CONNACK, queda no publish…)
+- `bench.py` — serial com detecção de reboot, cliente web, ciclo de vida dos servidores
+- `campaign.py` — controle do alvo; `phase_*.py` — as fases; `revalidate.py` — regressões
+- `rescore.py` — recontagem de perda de dados contando o que a falha recebeu
+- `soak24.py` — soak longo, monitorado só pela serial
+
+Os certificados de teste **não** estão versionados. Gerar antes de usar TLS:
+
+ mkdir -p certs && openssl req -x509 -newkey rsa:2048 -keyout certs/key.pem \
+ -out certs/cert.pem -days 365 -nodes -subj "/CN=/O=SimutBench" \
+ -addext "subjectAltName=IP:"
+
+Os endereços do alvo e do host estão fixos no topo de `campaign.py` e `bench.py`.
diff --git a/tools/telemetry_bench/bench.py b/tools/telemetry_bench/bench.py
new file mode 100644
index 0000000..aeefec7
--- /dev/null
+++ b/tools/telemetry_bench/bench.py
@@ -0,0 +1,363 @@
+#!/usr/bin/env python3
+"""Bench library for the telemetry test campaign.
+
+One process owns the serial port for a whole run, so reboot detection and CLI
+driving cannot fight each other for /dev/ttyACM*. Everything the tests need —
+serial, web, servers, metrics sampling — lives here so the test scripts stay
+about the experiment.
+
+Reboot detection is deliberately belt-and-braces: the USB CDC drops when the
+device resets, but a fast reset can be missed by a reader that happens to be
+between reads, so uptime going backwards in `show metrics` is checked too.
+"""
+import glob
+import json
+import os
+import re
+import signal
+import subprocess
+import sys
+import threading
+import time
+
+import serial
+import requests
+import hashlib
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+TARGET_GLOB = '/dev/serial/by-id/usb-Raspberry_Pi_Pico_W_*-if00'
+HAND_GLOB = '/dev/serial/by-id/usb-Raspberry_Pi_Pico_[0-9A-Z]*-if00'
+HOST_IP = '192.168.3.31'
+
+
+def target_port():
+ m = glob.glob(TARGET_GLOB)
+ return os.path.realpath(m[0]) if m else None
+
+
+def hand_port():
+ for p in glob.glob(HAND_GLOB):
+ if '_W_' in p:
+ continue
+ return os.path.realpath(p)
+ return None
+
+
+# ---------------------------------------------------------------------------
+# serial
+# ---------------------------------------------------------------------------
+
+class Target:
+ """Owns the target's USB CDC for the lifetime of a run.
+
+ A background thread drains the port continuously into a timestamped log and
+ watches for the markers that mean the device died: the port vanishing
+ (reset re-enumerates USB), the boot banner, and the fatal-log line the
+ firmware prints after a watchdog reboot.
+ """
+
+ BOOT_MARKERS = (
+ re.compile(r'\[BOOT'),
+ re.compile(r'SIMUT v?\d'),
+ re.compile(r'Iniciando|Booting|=== SIMUT'),
+ )
+ FATAL = re.compile(r'\[FTL\]|SOFT PANIC|HW WATCHDOG|PANIC')
+
+ def __init__(self, logpath, echo=False):
+ self.logpath = logpath
+ self.echo = echo
+ self.lf = open(logpath, 'a', buffering=1)
+ self.ser = None
+ self.buf = ''
+ self.lock = threading.Lock()
+ self.stop = False
+ self.port_drops = 0 # USB re-enumerations = hard reboots
+ self.fatal_lines = []
+ self.boot_lines = []
+ self.all_lines = []
+ self.t0 = time.time()
+ self._open()
+ self.rx = threading.Thread(target=self._reader, daemon=True)
+ self.rx.start()
+
+ # -- plumbing ----------------------------------------------------------
+ def _open(self):
+ p = target_port()
+ if not p:
+ return False
+ try:
+ s = serial.Serial(p, 115200, timeout=0.15)
+ s.dtr = True
+ time.sleep(0.25)
+ self.ser = s
+ self._log(f'--- serial open {p}')
+ return True
+ except Exception as e:
+ self._log(f'--- serial open failed: {e}')
+ return False
+
+ def _log(self, line):
+ stamp = f'{time.time() - self.t0:9.2f} '
+ self.lf.write(stamp + line + '\n')
+ if self.echo:
+ print(stamp + line, flush=True)
+
+ def _feed(self, text):
+ with self.lock:
+ self.buf += text
+ while '\n' in self.buf:
+ line, _, self.buf = self.buf.partition('\n')
+ line = line.rstrip('\r')
+ if not line:
+ continue
+ self.all_lines.append((time.time() - self.t0, line))
+ self._log(line)
+ if self.FATAL.search(line):
+ self.fatal_lines.append((time.time() - self.t0, line))
+ for m in self.BOOT_MARKERS:
+ if m.search(line):
+ self.boot_lines.append((time.time() - self.t0, line))
+ break
+
+ def _reader(self):
+ while not self.stop:
+ if self.ser is None:
+ if not self._open():
+ time.sleep(0.4)
+ continue
+ try:
+ data = self.ser.read(4096)
+ if data:
+ self._feed(data.decode('utf-8', 'replace'))
+ except Exception as e:
+ # Port vanished: the device re-enumerated, i.e. it reset.
+ self._log(f'--- serial dropped ({e})')
+ self.port_drops += 1
+ try:
+ self.ser.close()
+ except Exception:
+ pass
+ self.ser = None
+ time.sleep(0.8)
+
+ # -- commands ----------------------------------------------------------
+ def send(self, cmd, wait=2.5, quiet=0.45):
+ """Send one CLI line and collect the reply until the prompt goes quiet."""
+ if self.ser is None:
+ self._open()
+ if self.ser is None:
+ return ''
+ with self.lock:
+ mark = len(self.all_lines)
+ try:
+ self.ser.write((cmd + '\r\n').encode())
+ self.ser.flush()
+ except Exception as e:
+ self._log(f'--- write failed: {e}')
+ return ''
+ deadline = time.time() + wait
+ last_len = mark
+ quiet_at = None
+ while time.time() < deadline:
+ time.sleep(0.08)
+ with self.lock:
+ n = len(self.all_lines)
+ if n != last_len:
+ last_len = n
+ quiet_at = time.time() + quiet
+ deadline = max(deadline, time.time() + 0.5)
+ elif quiet_at and time.time() > quiet_at:
+ break
+ with self.lock:
+ return '\n'.join(l for _, l in self.all_lines[mark:])
+
+ def cmds(self, *cmds, wait=2.5):
+ return [self.send(c, wait=wait) for c in cmds]
+
+ # -- state -------------------------------------------------------------
+ def metrics(self, wait=3.5):
+ txt = self.send('show metrics', wait=wait)
+ d = {'raw': txt}
+ pats = {
+ 'uptime': r'Uptime:\s*(\d+):(\d+):(\d+)',
+ 'heap': r'Heap:\s*(\d+)\s*B\s*\(min:\s*(\d+)',
+ 'largest': r'Maior bloco:\s*(\d+)\s*B\s*\(min:\s*(\d+)',
+ 'wifi_conns': r'WiFi conns:\s*(\d+)',
+ 'mqtt_conns': r'MQTT conns:\s*(\d+)',
+ 'tel_sent': r'Enviadas:\s*(\d+)',
+ 'tel_failed': r'Falhas:\s*(\d+)',
+ 'tel_retries': r'Retries:\s*(\d+)',
+ 'tel_bytes': r'Bytes:\s*(\d+)',
+ 'tel_lat': r'Ult\. lat:\s*(\d+)',
+ 'reads_ok': r'Leituras OK:\s*(\d+)',
+ 'reads_err': r'Leituras erro:\s*(\d+)',
+ 'flash_ops': r'Flash ops:\s*(\d+)',
+ 'core1_exposed': r'Core1 exposto:\s*(\d+)',
+ 'core1_hb': r'Heartbeat:\s*(\d+)\s*ms',
+ 'core1_running': r'Rodando:\s*(\d+)',
+ }
+ for k, p in pats.items():
+ m = re.search(p, txt)
+ if not m:
+ d[k] = None
+ continue
+ if k == 'uptime':
+ d[k] = int(m.group(1)) * 3600 + int(m.group(2)) * 60 + int(m.group(3))
+ elif k in ('heap', 'largest'):
+ d[k] = int(m.group(1))
+ d[k + '_min'] = int(m.group(2))
+ else:
+ d[k] = int(m.group(1))
+ return d
+
+ def close(self):
+ self.stop = True
+ time.sleep(0.3)
+ try:
+ if self.ser:
+ self.ser.close()
+ except Exception:
+ pass
+ self.lf.close()
+
+
+# ---------------------------------------------------------------------------
+# web
+# ---------------------------------------------------------------------------
+
+def sha256_frontend(password):
+ """The login page hashes each UTF-16 code unit as one byte — latin-1."""
+ return hashlib.sha256(password.encode('latin-1')).hexdigest()
+
+
+class Web:
+ def __init__(self, host, timeout=20):
+ self.base = f'http://{host}'
+ self.s = requests.Session()
+ self.timeout = timeout
+
+ def get(self, path, **kw):
+ kw.setdefault('allow_redirects', False)
+ return self.s.get(self.base + path, timeout=self.timeout, **kw)
+
+ def post(self, path, **kw):
+ kw.setdefault('allow_redirects', False)
+ return self.s.post(self.base + path, timeout=self.timeout, **kw)
+
+ def login(self, user, password):
+ r = self.get('/api/login_init')
+ if r.status_code != 200:
+ return False, f'login_init HTTP {r.status_code}'
+ nonce = r.json().get('nonce', '')
+ r = self.post('/api/login', data={
+ 'user': user, 'pass': sha256_frontend(password), 'nonce': nonce,
+ }, headers={'Content-Type': 'application/x-www-form-urlencoded'})
+ if 'SIMUTSESS' not in self.s.cookies.get_dict():
+ return False, f'no session cookie (HTTP {r.status_code}) {r.text[:120]}'
+ return True, 'ok'
+
+ def commit(self, sys_fields):
+ """POST /api/commit_all — applies config and reboots the device."""
+ payload = json.dumps({'sys': sys_fields}, separators=(',', ':'))
+ return self.post('/api/commit_all', data={'_payload': payload},
+ headers={'Content-Type': 'application/x-www-form-urlencoded'})
+
+ def config(self):
+ r = self.get('/api/config')
+ return r.json() if r.status_code == 200 else {'_http': r.status_code,
+ '_body': r.text[:200]}
+
+
+def wait_web(host, timeout=90, path='/api/login_init'):
+ """Block until the web server answers again (post-reboot)."""
+ t0 = time.time()
+ while time.time() - t0 < timeout:
+ try:
+ r = requests.get(f'http://{host}{path}', timeout=4)
+ if r.status_code < 500:
+ return round(time.time() - t0, 1)
+ except Exception:
+ pass
+ time.sleep(1)
+ return None
+
+
+# ---------------------------------------------------------------------------
+# servers
+# ---------------------------------------------------------------------------
+
+class Server:
+ """A test server subprocess with its stats file."""
+
+ def __init__(self, kind, name, outdir, **kw):
+ self.kind = kind # 'http' | 'mqtt'
+ self.name = name
+ self.outdir = outdir
+ self.stats_path = os.path.join(outdir, f'{name}.stats.json')
+ self.records_path = os.path.join(outdir, f'{name}.records.ndjson')
+ self.log_path = os.path.join(outdir, f'{name}.server.log')
+ for p in (self.stats_path, self.records_path):
+ if os.path.exists(p):
+ os.remove(p)
+ script = 'server_http.py' if kind == 'http' else 'server_mqtt.py'
+ cmd = [sys.executable, os.path.join(HERE, script),
+ '--stats', self.stats_path, '--records', self.records_path]
+ for k, v in kw.items():
+ flag = '--' + k.replace('_', '-')
+ if v is True:
+ cmd.append(flag)
+ elif v is False or v is None:
+ continue
+ else:
+ cmd += [flag, str(v)]
+ self.cmd = cmd
+ self.lf = open(self.log_path, 'w')
+ self.proc = subprocess.Popen(cmd, cwd=HERE, stdout=self.lf,
+ stderr=subprocess.STDOUT,
+ preexec_fn=os.setsid)
+ time.sleep(0.7)
+
+ def alive(self):
+ return self.proc.poll() is None
+
+ def stats(self):
+ try:
+ with open(self.stats_path) as fh:
+ return json.load(fh)
+ except Exception:
+ return {}
+
+ def records(self):
+ out = []
+ try:
+ with open(self.records_path) as fh:
+ for line in fh:
+ line = line.strip()
+ if line:
+ try:
+ out.append(json.loads(line))
+ except Exception:
+ pass
+ except Exception:
+ pass
+ return out
+
+ def stop(self):
+ try:
+ os.killpg(os.getpgid(self.proc.pid), signal.SIGTERM)
+ self.proc.wait(timeout=5)
+ except Exception:
+ try:
+ os.killpg(os.getpgid(self.proc.pid), signal.SIGKILL)
+ except Exception:
+ pass
+ try:
+ self.lf.close()
+ except Exception:
+ pass
+
+
+def kill_stale():
+ subprocess.run(['pkill', '-f', 'server_http.py'], capture_output=True)
+ subprocess.run(['pkill', '-f', 'server_mqtt.py'], capture_output=True)
+ time.sleep(0.4)
diff --git a/tools/telemetry_bench/campaign.py b/tools/telemetry_bench/campaign.py
new file mode 100644
index 0000000..a16bed0
--- /dev/null
+++ b/tools/telemetry_bench/campaign.py
@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Telemetry test campaign driver.
+
+Phases
+------
+ perf latency/throughput for HTTP, HTTPS, MQTT, MQTTS + batch sweep
+ drain point telemetry at a sink, reset the cursor, drain the history
+ survive cycle each fault mode past the device and watch it live or die
+
+Everything is measured from three independent places so no single instrument
+can lie: the server (what actually arrived), /api/status (what the device
+thinks), and the serial log (whether it rebooted).
+"""
+import argparse
+import json
+import os
+import re
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from bench import Target, Web, Server, kill_stale, wait_web, HOST_IP # noqa: E402
+
+import requests # noqa: E402
+
+DEV = '192.168.3.24'
+WEB_USER, WEB_PASS = 'telb', 'Bench2026x'
+PORT_HTTP, PORT_HTTPS, PORT_MQTT, PORT_MQTTS = 18080, 18443, 11883, 18883
+
+OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'results')
+os.makedirs(OUT, exist_ok=True)
+
+
+def log(msg):
+ print(f'[{time.strftime("%H:%M:%S")}] {msg}', flush=True)
+
+
+# ---------------------------------------------------------------------------
+# device control
+# ---------------------------------------------------------------------------
+
+_SESS = {'web': None}
+
+
+def web_session(force=False):
+ """One logged-in session, reused. /api/status is behind auth, so an
+ unauthenticated poll returns Forbidden and reads as 'device fine' — the
+ exact instrument failure this campaign is meant to avoid."""
+ if _SESS['web'] is None or force:
+ w = Web(DEV)
+ ok, why = w.login(WEB_USER, WEB_PASS)
+ if not ok:
+ return None
+ _SESS['web'] = w
+ return _SESS['web']
+
+
+def status(dev=DEV, timeout=6):
+ w = web_session()
+ if w is None:
+ return {'_err': 'noauth'}
+ try:
+ r = w.get('/api/status')
+ if r.status_code == 200:
+ return r.json()
+ if r.status_code in (401, 403, 302):
+ # Session died with the device: re-login once, then report.
+ w = web_session(force=True)
+ if w is None:
+ return {'_err': 'noauth'}
+ r = w.get('/api/status')
+ if r.status_code == 200:
+ return r.json()
+ return {'_http': r.status_code}
+ except Exception as e:
+ return {'_err': type(e).__name__}
+
+
+def cfg_http(t, server, port, crypto, batch=None, interval=None, mode=None,
+ path=None):
+ """Set the HTTP-side telemetry knobs over the serial CLI (no reboot)."""
+ cmds = ['enable', 'configure terminal',
+ f'tel server {server}', f'tel port {port}',
+ f'tel crypto {"on" if crypto else "off"}']
+ if path is not None:
+ cmds.append(f'tel path {path}')
+ if batch is not None:
+ cmds.append(f'tel batch {batch}')
+ if interval is not None:
+ cmds.append(f'tel interval {interval}')
+ if mode is not None:
+ cmds.append(f'tel mode {mode}')
+ cmds += ['end', 'write memory']
+ out = []
+ for c in cmds:
+ out.append(t.send(c, wait=3.0 if c == 'write memory' else 1.6))
+ return out
+
+
+def tel_reset(t):
+ t.send('enable', wait=1.2)
+ return t.send('tel reset', wait=4.0)
+
+
+def tel_sync(t, wait=25.0):
+ """Force one send. forceSync() resets the backoff before doing anything,
+ which is what makes a measurement window start at t=0 instead of somewhere
+ inside an escalated retry timer."""
+ t.send('enable', wait=1.2)
+ return t.send('tel sync', wait=wait)
+
+
+def commit(web, fields, expect_reboot=True):
+ """Push config through /api/commit_all; the device reboots on success."""
+ r = web.commit(fields)
+ ok = r.status_code in (200, 302)
+ log(f'commit_all -> HTTP {r.status_code} {r.text[:120]}')
+ if ok and expect_reboot:
+ time.sleep(4)
+ back = wait_web(DEV, timeout=120)
+ log(f'device web back after {back}s')
+ return ok, back
+ return ok, None
+
+
+def sample_loop(seconds, period=2.0, servers=(), tag=''):
+ """Poll /api/status while a run is in flight."""
+ samples = []
+ t0 = time.time()
+ while time.time() - t0 < seconds:
+ s = status()
+ s['t'] = round(time.time() - t0, 1)
+ samples.append(s)
+ time.sleep(period)
+ return samples
+
+
+def summarize_samples(samples):
+ up = [s.get('sys', {}).get('uptime') for s in samples if 'sys' in s]
+ heap = [s.get('sys', {}).get('heap_f') for s in samples if 'sys' in s]
+ lb = [s.get('sys', {}).get('heap_lb') for s in samples if 'sys' in s]
+ pend = [s.get('sys', {}).get('pending') for s in samples if 'sys' in s]
+ resets = 0
+ for a, b in zip(up, up[1:]):
+ if a is not None and b is not None and b < a:
+ resets += 1
+ unreachable = sum(1 for s in samples if '_err' in s or '_http' in s)
+ return {
+ 'n': len(samples),
+ 'uptime_resets': resets,
+ 'unreachable': unreachable,
+ 'heap_min': min([h for h in heap if h], default=None),
+ 'heap_max': max([h for h in heap if h], default=None),
+ 'largest_min': min([x for x in lb if x], default=None),
+ 'largest_max': max([x for x in lb if x], default=None),
+ 'pending_first': pend[0] if pend else None,
+ 'pending_last': pend[-1] if pend else None,
+ 'uptime_first': up[0] if up else None,
+ 'uptime_last': up[-1] if up else None,
+ }
+
+
+def save(name, obj):
+ p = os.path.join(OUT, name)
+ with open(p, 'w') as fh:
+ json.dump(obj, fh, indent=1, default=str)
+ log(f'saved {p}')
+ return p
diff --git a/tools/telemetry_bench/enumerate_history.py b/tools/telemetry_bench/enumerate_history.py
new file mode 100644
index 0000000..ee9875a
--- /dev/null
+++ b/tools/telemetry_bench/enumerate_history.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Enumerate /history by date instead of trusting /api/ls.
+
+`handleApiLs` breaks out of its enumeration loop on `isHandlerOvertime()` (a
+6 s budget) and then closes the JSON as if it had finished. Two listings of the
+same directory came back with 84 entries each and *different* contents;
+`20260523.h5` and `20260524.h5` both download fine (200, different md5) and no
+single listing showed both. A client cannot tell a truncated listing from a
+complete one, so any inventory built on /api/ls is an undercount of unknown
+size.
+
+Asking for each date directly removes the listing from the loop entirely: a
+404 means the day is genuinely absent, a 200 means it is there whatever the
+listing said.
+"""
+import datetime as dt
+import hashlib
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, '/home/angelo/Documentos/simut/tools')
+import campaign as C # noqa: E402
+import history_v5 as h5 # noqa: E402
+
+RAW = os.path.join(C.OUT, 'history_full')
+os.makedirs(RAW, exist_ok=True)
+
+
+def fetch(w, name, tries=3):
+ for i in range(tries):
+ try:
+ r = w.get('/download?file=/history/' + name)
+ except Exception:
+ time.sleep(1.5)
+ continue
+ if r.status_code == 200:
+ return r.content
+ if r.status_code == 404:
+ return None
+ # 403 = session gone, 503 = HeavyTaskGuard busy: both are retryable
+ w = C.web_session(force=True)
+ time.sleep(1.5)
+ return False # distinct from None: "could not decide"
+
+
+def main():
+ start = dt.date(2026, 4, 1)
+ end = dt.date.today()
+ w = C.web_session(force=True)
+
+ present, absent, unknown = [], [], []
+ d = start
+ while d <= end:
+ name = d.strftime('%Y%m%d') + '.h5'
+ dst = os.path.join(RAW, name)
+ if os.path.exists(dst) and os.path.getsize(dst) > 0:
+ present.append((name, os.path.getsize(dst)))
+ d += dt.timedelta(days=1)
+ continue
+ blob = fetch(w, name)
+ if blob is None:
+ absent.append(name)
+ elif blob is False:
+ unknown.append(name)
+ else:
+ with open(dst, 'wb') as fh:
+ fh.write(blob)
+ present.append((name, len(blob)))
+ d += dt.timedelta(days=1)
+ time.sleep(0.2)
+
+ epochs = set()
+ per_file, errors = [], []
+ for name, size in present:
+ try:
+ blob = open(os.path.join(RAW, name), 'rb').read()
+ eps = [ts for _s, ts, _v in h5.read_series(blob, 60)]
+ epochs.update(eps)
+ per_file.append({'file': name, 'bytes': size, 'records': len(eps),
+ 'first': min(eps) if eps else None,
+ 'last': max(eps) if eps else None,
+ 'md5': hashlib.md5(blob).hexdigest()[:12]})
+ except Exception as ex:
+ errors.append({'file': name, 'err': f'{type(ex).__name__}: {ex}'})
+
+ # What did /api/ls claim, for the record?
+ try:
+ listed = [e['n'] for e in w.get('/api/ls?dir=/history').json()['entries']]
+ except Exception:
+ listed = []
+
+ out = {
+ 'scanned_days': (end - start).days + 1,
+ 'files_present': len(present),
+ 'files_absent': len(absent),
+ 'files_unknown': unknown,
+ 'total_bytes': sum(s for _, s in present),
+ 'total_records': sum(p['records'] for p in per_file),
+ 'unique_epochs': len(epochs),
+ 'epoch_min': min(epochs) if epochs else None,
+ 'epoch_max': max(epochs) if epochs else None,
+ 'decode_errors': errors,
+ 'api_ls_reported': len(listed),
+ 'api_ls_missed': sorted(set(n for n, _ in present) - set(listed)),
+ 'api_ls_extra': sorted(set(listed) - set(n for n, _ in present)),
+ 'absent_days': absent,
+ 'per_file': per_file,
+ }
+ C.save('history_inventory.json', out)
+ C.log(json.dumps({k: out[k] for k in
+ ('files_present', 'files_absent', 'total_bytes',
+ 'total_records', 'unique_epochs', 'api_ls_reported',
+ 'api_ls_missed', 'files_unknown')}, indent=1))
+
+ # Re-score both drains against the true inventory.
+ for src, key in (('phase_drain.json', 'telemetry'),
+ ('phase_drain_full.json', None)):
+ p = os.path.join(C.OUT, src)
+ if not os.path.exists(p):
+ continue
+ with open(p) as fh:
+ j = json.load(fh)
+ n = (j[key] if key else j).get('unique_epochs')
+ C.log(f'{src}: {n} epochs = '
+ f'{round(100.0 * (n or 0) / max(1, len(epochs)), 2)}% of the true '
+ f'{len(epochs)} on flash')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/phase_drain.py b/tools/telemetry_bench/phase_drain.py
new file mode 100644
index 0000000..a0acdcd
--- /dev/null
+++ b/tools/telemetry_bench/phase_drain.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Phase B — drain as much history as the telemetry path will give up.
+
+Two independent readings of the same history, so the drain can be judged rather
+than just described:
+
+ telemetry `tel reset` then let the device push until it goes quiet
+ ground truth download every .h5 over /download and decode it with the
+ reference codec in tools/history_v5.py
+
+The difference between the two epoch sets is the answer to "todo o histórico
+possível" — including the part of the archive the telemetry path structurally
+cannot reach.
+"""
+import json
+import os
+import subprocess
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import campaign as C # noqa: E402
+from bench import Target, Server, kill_stale # noqa: E402
+
+REPO = '/home/angelo/Documentos/simut'
+sys.path.insert(0, os.path.join(REPO, 'tools'))
+
+RAW = os.path.join(C.OUT, 'history_raw')
+os.makedirs(RAW, exist_ok=True)
+
+
+def drain(t, seconds_max=2400, quiet_s=90, batch=50, interval=1000):
+ kill_stale()
+ srv = Server('http', 'drain', C.OUT, port=C.PORT_HTTP, mode='ok')
+ C.cfg_http(t, C.HOST_IP, C.PORT_HTTP, crypto=False, batch=batch,
+ interval=interval, mode='json', path='/ingest')
+ C.tel_reset(t)
+ C.tel_sync(t, wait=30)
+
+ t0 = time.time()
+ last_rec = 0
+ last_change = time.time()
+ timeline = []
+ boots0 = t.port_drops
+ while time.time() - t0 < seconds_max:
+ st = srv.stats()
+ n = st.get('records') or 0
+ s = C.status()
+ timeline.append({
+ 't': round(time.time() - t0, 1),
+ 'records': n,
+ 'requests': st.get('requests'),
+ 'pending': s.get('sys', {}).get('pending'),
+ 'heap': s.get('sys', {}).get('heap_f'),
+ 'lb': s.get('metr', {}).get('lb'),
+ 'tf': s.get('metr', {}).get('tf'),
+ 'tl': s.get('metr', {}).get('tl'),
+ 'uptime': s.get('sys', {}).get('uptime'),
+ })
+ if n != last_rec:
+ last_rec = n
+ last_change = time.time()
+ elif time.time() - last_change > quiet_s:
+ C.log(f'drained: no new record for {quiet_s}s at {n} records')
+ break
+ if len(timeline) % 10 == 0:
+ C.log(f' t={timeline[-1]["t"]}s records={n} pending={timeline[-1]["pending"]}')
+ time.sleep(3)
+
+ st = srv.stats()
+ recs = srv.records()
+ srv.stop()
+ epochs = sorted({r['ts'] for r in recs if isinstance(r.get('ts'), int)})
+ return {
+ 'wall_s': round(time.time() - t0, 1),
+ 'requests': st.get('requests'),
+ 'records': st.get('records'),
+ 'records_written': len(recs),
+ 'unique_epochs': len(epochs),
+ 'bytes_in': st.get('bytes_in'),
+ 'epoch_min': epochs[0] if epochs else None,
+ 'epoch_max': epochs[-1] if epochs else None,
+ 'duplicates': len(recs) - len(epochs),
+ 'usb_drops': t.port_drops - boots0,
+ 'srv_ms_p50': st.get('server_ms_p50'),
+ 'srv_ms_max': st.get('server_ms_max'),
+ 'timeline': timeline,
+ 'batch': batch, 'interval': interval,
+ }, epochs
+
+
+def download_all():
+ """Pull every history file over /download and decode with the reference codec."""
+ import history_v5 as h5
+ w = C.web_session()
+ entries = w.get('/api/ls?dir=/history').json()['entries']
+ files = [e for e in entries if e['n'].endswith('.h5')]
+ got, failed = [], []
+ for e in sorted(files, key=lambda x: x['n']):
+ dst = os.path.join(RAW, e['n'])
+ if os.path.exists(dst) and os.path.getsize(dst) == e['s']:
+ got.append((e['n'], e['s']))
+ continue
+ try:
+ r = w.get('/download?file=/history/' + e['n'])
+ if r.status_code == 200 and len(r.content) == e['s']:
+ with open(dst, 'wb') as fh:
+ fh.write(r.content)
+ got.append((e['n'], e['s']))
+ else:
+ failed.append((e['n'], r.status_code, len(r.content), e['s']))
+ except Exception as ex:
+ failed.append((e['n'], type(ex).__name__, str(ex)[:60], e['s']))
+ time.sleep(0.15)
+
+ all_epochs = set()
+ per_file = []
+ decode_errors = []
+ for name, size in got:
+ try:
+ blob = open(os.path.join(RAW, name), 'rb').read()
+ eps = [ts for _sch, ts, _vals in
+ h5.read_series(blob, nominal_interval_s=60)]
+ all_epochs.update(eps)
+ per_file.append({'file': name, 'bytes': size, 'records': len(eps),
+ 'first': min(eps) if eps else None,
+ 'last': max(eps) if eps else None})
+ except Exception as ex:
+ decode_errors.append({'file': name, 'err': f'{type(ex).__name__}: {ex}'})
+ return {
+ 'files_listed': len(files), 'files_downloaded': len(got),
+ 'download_failures': failed,
+ 'total_bytes': sum(s for _, s in got),
+ 'total_records': sum(p['records'] for p in per_file),
+ 'unique_epochs': len(all_epochs),
+ 'epoch_min': min(all_epochs) if all_epochs else None,
+ 'epoch_max': max(all_epochs) if all_epochs else None,
+ 'decode_errors': decode_errors,
+ 'per_file': per_file,
+ }, all_epochs
+
+
+def main():
+ seconds_max = int(sys.argv[1]) if len(sys.argv) > 1 else 2400
+ kill_stale()
+ t = Target(os.path.join(C.OUT, 'serial_drain.log'))
+ time.sleep(1)
+
+ C.log('=== draining via telemetry ===')
+ d, tel_epochs = drain(t, seconds_max=seconds_max)
+ C.log(json.dumps({k: d[k] for k in
+ ('wall_s', 'requests', 'records', 'unique_epochs',
+ 'epoch_min', 'epoch_max', 'duplicates', 'usb_drops')}))
+ t.close()
+
+ C.log('=== downloading ground truth ===')
+ g, disk_epochs = download_all()
+ C.log(json.dumps({k: g[k] for k in
+ ('files_listed', 'files_downloaded', 'total_records',
+ 'unique_epochs', 'epoch_min', 'epoch_max')}))
+
+ missing = sorted(disk_epochs - set(tel_epochs))
+ extra = sorted(set(tel_epochs) - disk_epochs)
+ out = {
+ 'telemetry': d,
+ 'ground_truth': g,
+ 'coverage': {
+ 'on_disk': len(disk_epochs),
+ 'via_telemetry': len(tel_epochs),
+ 'pct': round(100.0 * len(tel_epochs) / max(1, len(disk_epochs)), 2),
+ 'missing_count': len(missing),
+ 'missing_first': missing[0] if missing else None,
+ 'missing_last': missing[-1] if missing else None,
+ 'extra_count': len(extra),
+ 'extra_sample': extra[:10],
+ },
+ }
+ C.save('phase_drain.json', out)
+ C.log(json.dumps(out['coverage'], indent=1))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/phase_drain_full.py b/tools/telemetry_bench/phase_drain_full.py
new file mode 100644
index 0000000..3b37554
--- /dev/null
+++ b/tools/telemetry_bench/phase_drain_full.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Phase B2 — drain the WHOLE archive, past the firmware's 30-day floor.
+
+`tel reset` zeroes the cursor, and collectBatch then refuses to look further
+back than `lastRecorded − 30 days`:
+
+ if (lastCursor == 0) {
+ uint32_t lastRecorded = _storageRef->getLastRecordedTimestamp( );
+ if (lastRecorded > 86400UL * 30) lastCursor = lastRecorded - 86400UL * 30;
+ }
+
+so anything older is unreachable through telemetry no matter how long it runs.
+The floor is a policy in that one branch, not a storage limit — and this proves
+it. The cursor lives in a 4-byte file, `/config/t_cursor.bin`; seeding it with
+HIST_EPOCH_MIN instead of zero skips the fallback entirely and the device
+happily streams the entire archive.
+
+Run order matters: `tel reset` must come first (it clears the RAM cache AND
+deletes the file), and the seeded file has to land before the next read.
+"""
+import json
+import os
+import struct
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import campaign as C # noqa: E402
+from bench import Target, Server, kill_stale # noqa: E402
+
+SEED_EPOCH = 1600000001 # just past HIST_EPOCH_MIN (1.6e9)
+
+
+def seed_cursor(value):
+ """Upload a 4-byte little-endian cursor. /api/upload ignores `dir`, so the
+ destination path has to travel in the filename."""
+ w = C.web_session()
+ blob = struct.pack(' 1 else 4200
+ kill_stale()
+ t = Target(os.path.join(C.OUT, 'serial_drain_full.log'))
+ time.sleep(1)
+
+ srv = Server('http', 'drain_full', C.OUT, port=C.PORT_HTTP, mode='ok')
+ C.cfg_http(t, C.HOST_IP, C.PORT_HTTP, crypto=False, batch=50,
+ interval=1000, mode='json', path='/ingest')
+ C.tel_reset(t)
+ time.sleep(2)
+ code, body = seed_cursor(SEED_EPOCH)
+ C.log(f'seed /config/t_cursor.bin -> HTTP {code} {body}')
+ time.sleep(2)
+ # Verify the seed landed where it was aimed. A silently-misplaced upload
+ # would leave the 30-day fallback in charge and this run would quietly
+ # re-measure the previous phase while claiming to have beaten the cap.
+ w = C.web_session()
+ try:
+ listing = w.get('/api/ls?dir=/config').json().get('entries', [])
+ except Exception as e:
+ listing = [{'err': str(e)}]
+ seed_ok = any(e.get('n') == 't_cursor.bin' and e.get('s') == 4
+ for e in listing)
+ C.log(f'/config listing: {listing} seed_ok={seed_ok}')
+ C.tel_sync(t, wait=40)
+
+ t0 = time.time()
+ last_rec, last_change = 0, time.time()
+ timeline = []
+ boots0 = t.port_drops
+ while time.time() - t0 < seconds_max:
+ st = srv.stats()
+ n = st.get('records') or 0
+ s = C.status()
+ timeline.append({
+ 't': round(time.time() - t0, 1), 'records': n,
+ 'requests': st.get('requests'),
+ 'pending': s.get('sys', {}).get('pending'),
+ 'heap': s.get('sys', {}).get('heap_f'),
+ 'lb': s.get('metr', {}).get('lb'),
+ 'tf': s.get('metr', {}).get('tf'), 'tl': s.get('metr', {}).get('tl'),
+ 'uptime': s.get('sys', {}).get('uptime'),
+ })
+ if n != last_rec:
+ last_rec, last_change = n, time.time()
+ elif time.time() - last_change > 120:
+ C.log(f'drained: quiet for 120s at {n} records')
+ break
+ if len(timeline) % 20 == 0:
+ C.log(f' t={timeline[-1]["t"]}s records={n} '
+ f'pending={timeline[-1]["pending"]} heap={timeline[-1]["heap"]}')
+ time.sleep(3)
+
+ st = srv.stats()
+ recs = srv.records()
+ srv.stop()
+ t.close()
+
+ epochs = sorted({r['ts'] for r in recs if isinstance(r.get('ts'), int)})
+ with_press = [r for r in recs if any(k.startswith('p') for k in r)]
+ out = {
+ 'seed_epoch': SEED_EPOCH,
+ 'seed_upload_http': code,
+ 'seed_present_in_fs': seed_ok,
+ 'wall_s': round(time.time() - t0, 1),
+ 'requests': st.get('requests'), 'records': st.get('records'),
+ 'unique_epochs': len(epochs), 'duplicates': len(recs) - len(epochs),
+ 'bytes_in': st.get('bytes_in'),
+ 'epoch_min': epochs[0] if epochs else None,
+ 'epoch_max': epochs[-1] if epochs else None,
+ 'records_with_pressure': len(with_press),
+ 'first_pressure_record': with_press[0] if with_press else None,
+ 'usb_drops': t.port_drops - boots0,
+ 'timeline': timeline,
+ }
+ if out['wall_s']:
+ out['records_per_s'] = round((out['records'] or 0) / out['wall_s'], 2)
+ C.save('phase_drain_full.json', out)
+
+ # Compare against the ground truth the 30-day drain already collected.
+ prev = os.path.join(C.OUT, 'phase_drain.json')
+ if os.path.exists(prev):
+ with open(prev) as fh:
+ p = json.load(fh)
+ disk = p['ground_truth']
+ out['vs_disk'] = {
+ 'disk_unique_epochs': disk['unique_epochs'],
+ 'full_drain_unique': len(epochs),
+ 'pct': round(100.0 * len(epochs) / max(1, disk['unique_epochs']), 2),
+ 'thirty_day_drain_unique': p['telemetry']['unique_epochs'],
+ 'thirty_day_pct': p['coverage']['pct'],
+ }
+ C.save('phase_drain_full.json', out)
+ C.log(json.dumps(out['vs_disk'], indent=1))
+ C.log(json.dumps({k: out[k] for k in
+ ('wall_s', 'requests', 'records', 'unique_epochs',
+ 'epoch_min', 'epoch_max', 'records_per_s', 'usb_drops')}))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/phase_mqtt.py b/tools/telemetry_bench/phase_mqtt.py
new file mode 100644
index 0000000..eb38122
--- /dev/null
+++ b/tools/telemetry_bench/phase_mqtt.py
@@ -0,0 +1,341 @@
+#!/usr/bin/env python3
+"""Phases A2/C2 — MQTT and MQTTS: throughput, and survival against a broken broker.
+
+Two facts about the firmware shape this file.
+
+`telTransport` and the MQTT client's server/port/TLS are read exactly once, in
+TelemetryManager::begin(). Nothing re-reads them, so switching to MQTT — or from
+MQTT to MQTTS — costs a reboot through /api/commit_all. Everything after that
+runs on ONE port, and the fault modes are produced by restarting the broker
+process rather than by re-pointing the device.
+
+attemptMqttPublish splits at 5 records: five or fewer are published one message
+per record, more than five as a single payload. The batch sweep straddles that
+threshold on purpose.
+"""
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import campaign as C # noqa: E402
+from bench import Target, Server, kill_stale, wait_web # noqa: E402
+
+HIST_INTERVAL_S = 60
+
+
+def switch(web, transport, tls, port, batch=50, interval=1000):
+ fields = {
+ 't_transport': str(transport),
+ 't_sec': '1' if tls else '0',
+ 't_srv': C.HOST_IP,
+ 't_port': str(port),
+ 't_int': str(interval),
+ 't_bat': str(batch),
+ 't_mode': '0',
+ 'm_topic': 'simut/data',
+ 'm_qos': '1',
+ 'm_retain': '0',
+ 'm_ka': '30',
+ }
+ ok, back = C.commit(web, fields)
+ return ok, back
+
+
+def broker(name, port, tls, **kw):
+ args = dict(port=port, tls=tls, cert='certs/cert.pem', key='certs/key.pem')
+ args.update(kw)
+ return Server('mqtt', name, C.OUT, **args)
+
+
+def measure(t, srv, seconds, label):
+ s0 = C.status()
+ m0 = s0.get('metr', {})
+ boots0, fatal0 = t.port_drops, len(t.fatal_lines)
+ samples = []
+ t0 = time.time()
+ while time.time() - t0 < seconds:
+ s = C.status()
+ s['_t'] = round(time.time() - t0, 1)
+ samples.append(s)
+ time.sleep(2)
+ m1 = C.status().get('metr', {})
+ st = srv.stats() if srv else {}
+
+ def d(k):
+ a, b = m0.get(k), m1.get(k)
+ return (b - a) if isinstance(a, int) and isinstance(b, int) else None
+
+ lat = [s.get('metr', {}).get('tl') for s in samples if s.get('metr')]
+ lat = sorted(x for x in lat if isinstance(x, int) and x > 0)
+ heap = [s.get('sys', {}).get('heap_f') for s in samples if s.get('sys')]
+ lb = [s.get('metr', {}).get('lb') for s in samples if s.get('metr')]
+ up = [s.get('sys', {}).get('uptime') for s in samples if s.get('sys')]
+ resets = sum(1 for a, b in zip(up, up[1:])
+ if a is not None and b is not None and b < a)
+ row = {
+ 'label': label, 'seconds': seconds,
+ 'dev_sent': d('ts'), 'dev_failed': d('tf'), 'dev_retries': d('tr'),
+ 'dev_bytes': d('tb'), 'dev_mqtt_conns': d('mq'),
+ 'dev_lat_min': lat[0] if lat else None,
+ 'dev_lat_med': lat[len(lat) // 2] if lat else None,
+ 'dev_lat_max': lat[-1] if lat else None,
+ 'heap_min': min([h for h in heap if h], default=None),
+ 'heap_max': max([h for h in heap if h], default=None),
+ 'largest_min': min([x for x in lb if x], default=None),
+ 'uptime_resets': resets, 'usb_drops': t.port_drops - boots0,
+ 'fatal_lines': len(t.fatal_lines) - fatal0,
+ 'unreachable_polls': sum(1 for s in samples if '_err' in s or '_http' in s),
+ 'srv_conns': st.get('conns'), 'srv_connects': st.get('connects'),
+ 'srv_connacks': st.get('connacks'), 'srv_publishes': st.get('publishes'),
+ 'srv_records': st.get('records'), 'srv_bytes': st.get('bytes_in'),
+ 'srv_pings': st.get('pings'),
+ 'srv_tls_ok': st.get('tls_ok'), 'srv_tls_fail': st.get('tls_failures'),
+ 'srv_qos_seen': st.get('qos_seen'), 'srv_retain_seen': st.get('retain_seen'),
+ 'srv_client_ids': st.get('client_ids'), 'srv_will_topics': st.get('will_topics'),
+ 'srv_status_msgs': st.get('status_msgs'),
+ 'srv_epoch_min': st.get('epoch_min'), 'srv_epoch_max': st.get('epoch_max'),
+ 'srv_msg_bytes': [m.get('bytes') for m in (st.get('msgs') or [])[-5:]],
+ 'srv_msg_n': [m.get('n') for m in (st.get('msgs') or [])[-5:]],
+ }
+ if row['srv_records'] is not None:
+ row['records_per_s'] = round(row['srv_records'] / seconds, 2)
+ return row
+
+
+def perf_sweep(t, port, tls, batches, seconds, tag):
+ rows = []
+ for b in batches:
+ label = f'{"mqtts" if tls else "mqtt"}_batch{b}'
+ C.log(f'--- {label}')
+ kill_stale()
+ srv = broker(f'{tag}_{label}', port, tls, mode='ok')
+ # Batch size is plain config, no reboot needed — only transport/TLS are
+ # frozen at begin().
+ t.send('enable', wait=1.2)
+ t.send('configure terminal', wait=1.2)
+ t.send(f'tel batch {b}', wait=1.2)
+ t.send('tel interval 1000', wait=1.2)
+ t.send('end', wait=1.2)
+ t.send('write memory', wait=3.0)
+ C.tel_reset(t)
+ C.tel_sync(t, wait=30)
+ time.sleep(3)
+ row = measure(t, srv, seconds, label)
+ row['batch'] = b
+ rows.append(row)
+ C.log(json.dumps({k: row[k] for k in
+ ('label', 'srv_publishes', 'srv_records', 'records_per_s',
+ 'dev_lat_med', 'dev_failed', 'srv_qos_seen', 'heap_min',
+ 'usb_drops', 'srv_connects')}))
+ srv.stop()
+ return rows
+
+
+def run_fault(t, name, port, tls, seconds, server_kw=None, no_server=False, batch=10):
+ C.log(f'=== mqtt fault: {name} ({seconds}s)')
+ # act 1 — healthy broker, note where the cursor got to
+ kill_stale()
+ pre_srv = broker(f'{name}_pre', port, tls, mode='ok')
+ t.send('enable', wait=1.2)
+ t.send('configure terminal', wait=1.2)
+ t.send(f'tel batch {batch}', wait=1.2)
+ t.send('tel interval 1000', wait=1.2)
+ t.send('end', wait=1.2)
+ t.send('write memory', wait=3.0)
+ C.tel_reset(t)
+ C.tel_sync(t, wait=30)
+ time.sleep(12)
+ pre = pre_srv.stats()
+ pre = {'publishes': pre.get('publishes'), 'records': pre.get('records'),
+ 'epoch_min': pre.get('epoch_min'), 'epoch_max': pre.get('epoch_max')}
+ pre_srv.stop()
+ C.log(f' baseline: {pre}')
+
+ kill_stale()
+ srv = None
+ if not no_server:
+ srv = broker(f'{name}_fault', port, tls, **(server_kw or {}))
+ C.tel_sync(t, wait=30)
+ row = measure(t, srv, seconds, name)
+ if srv:
+ srv.stop()
+
+ kill_stale()
+ post_srv = broker(f'{name}_post', port, tls, mode='ok')
+ C.tel_sync(t, wait=30)
+ time.sleep(30)
+ post = post_srv.stats()
+ post = {'publishes': post.get('publishes'), 'records': post.get('records'),
+ 'epoch_min': post.get('epoch_min'), 'epoch_max': post.get('epoch_max')}
+ post_srv.stop()
+ C.log(f' recovery: {post}')
+
+ gap = lost = None
+ if pre.get('epoch_max') and post.get('epoch_min'):
+ gap = post['epoch_min'] - pre['epoch_max']
+ lost = max(0, (gap // HIST_INTERVAL_S) - 1)
+ row.update({'fault': name, 'pre': pre, 'post': post,
+ 'cursor_gap_s': gap, 'records_skipped': lost,
+ 'server_kw': server_kw, 'no_server': no_server, 'tls': tls})
+ v = []
+ if row['usb_drops'] or row['uptime_resets']:
+ v.append('REBOOT')
+ if row['fatal_lines']:
+ v.append('FATAL')
+ if lost:
+ v.append(f'DATA-LOSS({lost} rec)')
+ if row['unreachable_polls'] >= 3:
+ v.append(f'WEB-STALL({row["unreachable_polls"]} polls)')
+ if not post.get('records'):
+ v.append('NO-RECOVERY')
+ row['verdict'] = ' '.join(v) if v else 'SURVIVED'
+ C.log(f' -> {row["verdict"]} (devFail+{row["dev_failed"]} '
+ f'conns={row["srv_conns"]} connacks={row["srv_connacks"]} heapMin={row["heap_min"]})')
+ return row
+
+
+FAULTS = [
+ ('mq_refused', dict(no_server=True)),
+ ('mq_rst', dict(server_kw={'mode': 'rst'})),
+ ('mq_no_connack', dict(server_kw={'mode': 'no_connack', 'delay': 180})),
+ ('mq_slow_connack', dict(server_kw={'mode': 'slow_connack', 'delay': 20})),
+ ('mq_half_connack', dict(server_kw={'mode': 'half_connack', 'delay': 180})),
+ ('mq_connack_refuse', dict(server_kw={'mode': 'connack_refuse'})),
+ ('mq_connack_unavail', dict(server_kw={'mode': 'connack_unavail'})),
+ ('mq_drop_after_connack', dict(server_kw={'mode': 'drop_after_connack'})),
+ ('mq_drop_on_publish', dict(server_kw={'mode': 'drop_on_publish'})),
+ ('mq_rst_on_publish', dict(server_kw={'mode': 'rst_on_publish'})),
+ ('mq_garbage', dict(server_kw={'mode': 'garbage'})),
+ ('mq_no_pingresp', dict(server_kw={'mode': 'no_pingresp'})),
+]
+
+TLS_FAULTS = [
+ ('mqs_tls_blackhole', dict(server_kw={'mode': 'ok', 'tls_fault': 'blackhole', 'delay': 180})),
+ ('mqs_tls_garbage', dict(server_kw={'mode': 'ok', 'tls_fault': 'garbage'})),
+ ('mqs_tls_rst', dict(server_kw={'mode': 'ok', 'tls_fault': 'rst'})),
+ ('mqs_refused', dict(no_server=True)),
+ ('mqs_drop_on_publish', dict(server_kw={'mode': 'drop_on_publish'})),
+]
+
+
+def config_fidelity(t, web, port, seconds=70):
+ """Does the wire carry what the config page promised?
+
+ Every MQTT knob the UI exposes is checked against the bytes that actually
+ reach the broker: credentials and client id in the CONNECT, QoS and retain
+ in the PUBLISH flags, keepalive in the CONNECT header. A field the firmware
+ accepts and then never transmits is invisible from the device side — the
+ broker is the only place it shows up.
+ """
+ C.log('=== MQTT config fidelity ===')
+ kill_stale()
+ srv = broker('fidelity', port, False, mode='ok')
+ fields = {
+ 't_transport': '1', 't_sec': '0', 't_srv': C.HOST_IP, 't_port': str(port),
+ 't_int': '2000', 't_bat': '3', 't_mode': '0',
+ 'm_topic': 'bench/telemetry/data', 'm_cid': 'benchcid7',
+ 'm_user': 'benchuser', 'm_pass': 'benchsecret',
+ 'm_qos': '1', 'm_retain': '1', 'm_ka': '45',
+ }
+ C.commit(web, fields)
+ C.web_session(force=True)
+ time.sleep(6)
+ C.tel_sync(t, wait=30)
+ time.sleep(seconds)
+ st = srv.stats()
+ srv.stop()
+
+ wanted = {'m_cid': 'benchcid7', 'm_user': 'benchuser', 'm_pass': 'benchsecret',
+ 'm_qos': 1, 'm_retain': True, 'm_ka': 45,
+ 'm_topic': 'bench/telemetry/data'}
+ # The broker is up before the commit, so the device's PRE-reboot client
+ # connects to it first, carrying the OLD settings. Reading frame 0 scored
+ # every field as ignored and would have reported four defects that do not
+ # exist. Take the frame that belongs to the config under test — the one
+ # whose will topic came from the new topic — and fall back to the last.
+ frames = st.get('connect_frames') or [{}]
+ want_will = wanted['m_topic'].rsplit('/', 1)[0] + '/status'
+ cf = next((f for f in frames if f.get('willTopic') == want_will), frames[-1])
+ got = {
+ 'clientId': cf.get('clientId'), 'user': cf.get('user'),
+ 'pass': cf.get('pass'), 'keepalive': cf.get('keepalive'),
+ 'willTopic': cf.get('willTopic'),
+ 'qos_seen': st.get('qos_seen'), 'retain_seen': st.get('retain_seen'),
+ 'topics': sorted({m['topic'] for m in (st.get('msgs') or [])}),
+ 'publishes': st.get('publishes'), 'records': st.get('records'),
+ }
+ checks = {
+ 'client_id_honoured': got['clientId'] == wanted['m_cid'],
+ 'user_honoured': got['user'] == wanted['m_user'],
+ 'password_honoured': got['pass'] == wanted['m_pass'],
+ 'keepalive_honoured': got['keepalive'] == wanted['m_ka'],
+ 'topic_honoured': wanted['m_topic'] in (got['topics'] or []),
+ 'qos1_honoured': bool(got['qos_seen']) and set(map(int, got['qos_seen'])) == {1},
+ 'retain_honoured': bool(got['retain_seen']) and got['retain_seen'].get('True', 0) > 0,
+ }
+ C.log('fidelity wanted=' + json.dumps(wanted))
+ C.log('fidelity got=' + json.dumps(got))
+ C.log('fidelity checks=' + json.dumps(checks))
+ return {'wanted': wanted, 'got': got, 'checks': checks,
+ 'frame_used': cf, 'connect_frames': frames}
+
+
+def main():
+ seconds = int(sys.argv[1]) if len(sys.argv) > 1 else 90
+ stage = sys.argv[2] if len(sys.argv) > 2 else 'all'
+ kill_stale()
+ t = Target(os.path.join(C.OUT, f'serial_mqtt_{stage}.log'))
+ time.sleep(1)
+ web = C.web_session()
+ out = {'perf': [], 'faults': []}
+
+ if stage in ('all', 'plain'):
+ C.log('=== switching to MQTT plain (reboot) ===')
+ kill_stale()
+ srv = broker('boot_mqtt', C.PORT_MQTT, False, mode='ok')
+ ok, back = switch(web, 1, False, C.PORT_MQTT)
+ C.log(f'switch ok={ok} web back in {back}s')
+ C.web_session(force=True)
+ time.sleep(5)
+ srv.stop()
+ out['perf'] += perf_sweep(t, C.PORT_MQTT, False, [1, 5, 10, 50], seconds, 'mqtt')
+ C.save('phase_mqtt_plain.json', out)
+ out['fidelity'] = config_fidelity(t, C.web_session(), C.PORT_MQTT)
+ # config_fidelity leaves credentials and QoS 1 set; put the transport
+ # back on the plain settings the fault runs assume.
+ C.commit(C.web_session(), {
+ 't_transport': '1', 't_sec': '0', 't_srv': C.HOST_IP,
+ 't_port': str(C.PORT_MQTT), 't_int': '1000', 't_bat': '10',
+ 'm_topic': 'simut/data', 'm_cid': '', 'm_user': '',
+ 'm_qos': '0', 'm_retain': '0', 'm_ka': '30'})
+ C.web_session(force=True)
+ time.sleep(5)
+ C.save('phase_mqtt_plain.json', out)
+ for name, kw in FAULTS:
+ out['faults'].append(run_fault(t, name, C.PORT_MQTT, False, seconds, **kw))
+ C.save('phase_mqtt_plain.json', out)
+
+ if stage in ('all', 'tls'):
+ C.log('=== switching to MQTTS (reboot) ===')
+ kill_stale()
+ srv = broker('boot_mqtts', C.PORT_MQTTS, True, mode='ok')
+ ok, back = switch(web, 1, True, C.PORT_MQTTS)
+ C.log(f'switch ok={ok} web back in {back}s')
+ C.web_session(force=True)
+ time.sleep(5)
+ srv.stop()
+ out2 = {'perf': [], 'faults': []}
+ out2['perf'] += perf_sweep(t, C.PORT_MQTTS, True, [1, 5, 10, 50], seconds, 'mqtts')
+ C.save('phase_mqtt_tls.json', out2)
+ for name, kw in TLS_FAULTS:
+ out2['faults'].append(run_fault(t, name, C.PORT_MQTTS, True, seconds, **kw))
+ C.save('phase_mqtt_tls.json', out2)
+
+ t.close()
+ C.log('done')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/phase_mqtt_oversize.py b/tools/telemetry_bench/phase_mqtt_oversize.py
new file mode 100644
index 0000000..a40d3cd
--- /dev/null
+++ b/tools/telemetry_bench/phase_mqtt_oversize.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Targeted test — an MQTT payload bigger than the client buffer can ever be.
+
+attemptMqttPublish grows the PubSubClient buffer to fit the payload, but clamps
+the request at 8192:
+
+ uint16_t needed = min((size_t)8192, payload.length( ) + 64);
+ _mqttClient.setBufferSize(needed);
+
+PubSubClient::publish() refuses any packet that does not fit the buffer, so a
+payload past ~8 KB cannot be published *at all* — and since the failure is
+deterministic, the retry never succeeds either. That is a permanent stall, not a
+transient error, and it is reachable from the config page: batch 50 with a long
+custom line template gets there with the sensors this bench already has.
+
+The test builds exactly that configuration, runs it against a healthy broker,
+and checks whether anything is published. It then verifies the device is still
+alive and recovers once the batch comes back down.
+"""
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import campaign as C # noqa: E402
+from bench import Target, Server, kill_stale # noqa: E402
+
+# ~205 bytes per record once the tokens expand: comfortably over 8192 at 50.
+LONG_LINE = ('{"timestamp_utc_seconds":{TS},"ch_a_temperature_celsius":{t0},'
+ '"ch_b_temperature_celsius":{t1},"ch_c_temperature_celsius":{t3},'
+ '"ch_c_relative_humidity":{u3},"ch_d_temperature":{t4},'
+ '"ch_e_temperature":{t10},"ch_e_humidity":{u10}}')
+
+
+def run(t, batch, mode, label, seconds=70):
+ kill_stale()
+ srv = Server('mqtt', f'oversize_{label}', C.OUT, port=C.PORT_MQTT,
+ mode='ok')
+ t.send('enable', wait=1.2)
+ t.send('configure terminal', wait=1.2)
+ t.send(f'tel batch {batch}', wait=1.2)
+ t.send('tel interval 1000', wait=1.2)
+ t.send(f'tel mode {mode}', wait=1.2)
+ t.send('end', wait=1.2)
+ t.send('write memory', wait=3.0)
+ C.tel_reset(t)
+ C.tel_sync(t, wait=30)
+ boots0 = t.port_drops
+ time.sleep(seconds)
+ st = srv.stats()
+ srv.stop()
+ s = C.status()
+ msgs = st.get('msgs') or []
+ return {
+ 'label': label, 'batch': batch, 'mode': mode,
+ 'publishes': st.get('publishes'), 'records': st.get('records'),
+ 'connects': st.get('connects'), 'connacks': st.get('connacks'),
+ 'largest_msg_bytes': max([m.get('bytes', 0) for m in msgs], default=0),
+ 'msg_bytes': [m.get('bytes') for m in msgs[-5:]],
+ 'usb_drops': t.port_drops - boots0,
+ 'heap': s.get('sys', {}).get('heap_f'),
+ 'dev_failed': s.get('metr', {}).get('tf'),
+ 'dev_sent': s.get('metr', {}).get('ts'),
+ 'uptime': s.get('sys', {}).get('uptime'),
+ }
+
+
+def main():
+ kill_stale()
+ t = Target(os.path.join(C.OUT, 'serial_mqtt_oversize.log'))
+ time.sleep(1)
+ web = C.web_session()
+
+ C.log('=== switching to MQTT plain with a long custom line template ===')
+ kill_stale()
+ boot = Server('mqtt', 'oversize_boot', C.OUT, port=C.PORT_MQTT, mode='ok')
+ C.commit(web, {
+ 't_transport': '1', 't_sec': '0', 't_srv': C.HOST_IP,
+ 't_port': str(C.PORT_MQTT), 't_int': '1000', 't_bat': '50',
+ 't_mode': '2',
+ 't_glob': '[{DATA}]',
+ 't_line': LONG_LINE,
+ 't_sep': ',',
+ 'm_topic': 'simut/data', 'm_qos': '0', 'm_retain': '0', 'm_ka': '30',
+ })
+ C.web_session(force=True)
+ time.sleep(5)
+ boot.stop()
+
+ out = {}
+ # Small batch first: proves the template itself publishes fine.
+ out['small_batch5'] = run(t, 5, 'custom', 'small_batch5')
+ C.log(json.dumps(out['small_batch5']))
+ # Then the same template at batch 50, where the payload passes 8 KB.
+ out['big_batch50'] = run(t, 50, 'custom', 'big_batch50')
+ C.log(json.dumps(out['big_batch50']))
+ # And back down, to show the stall is the payload size and not damage.
+ out['recover_batch5'] = run(t, 5, 'custom', 'recover_batch5')
+ C.log(json.dumps(out['recover_batch5']))
+
+ big = out['big_batch50']
+ out['verdict'] = ('STALL: nothing published at batch 50'
+ if not big['publishes'] else
+ f'published {big["publishes"]} msgs, '
+ f'largest {big["largest_msg_bytes"]} B')
+ C.log('VERDICT: ' + out['verdict'])
+ C.save('phase_mqtt_oversize.json', out)
+ t.close()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/phase_payload.py b/tools/telemetry_bench/phase_payload.py
new file mode 100644
index 0000000..d64f16a
--- /dev/null
+++ b/tools/telemetry_bench/phase_payload.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Phase D — payload builders and value integrity.
+
+Two questions the throughput numbers cannot answer:
+
+ 1. Do the three payload modes (json / csv / custom) actually produce what
+ they claim? Raw request bodies are captured verbatim and checked.
+ 2. Are the *values* that arrive the values that are on flash? The same
+ records are read back from the .h5 files with the reference codec and
+ compared field by field. Throughput is worthless if the numbers are wrong.
+"""
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, '/home/angelo/Documentos/simut/tools')
+import campaign as C # noqa: E402
+from bench import Target, Server, kill_stale # noqa: E402
+import history_v5 as h5 # noqa: E402
+
+
+def capture(t, mode, seconds=45, batch=5):
+ kill_stale()
+ srv = Server('http', f'payload_{mode}', C.OUT, port=C.PORT_HTTP, mode='ok',
+ raw_dump=6)
+ C.cfg_http(t, C.HOST_IP, C.PORT_HTTP, crypto=False, batch=batch,
+ interval=2000, mode=mode, path='/ingest')
+ C.tel_reset(t)
+ C.tel_sync(t, wait=30)
+ time.sleep(seconds)
+ st = srv.stats()
+ srv.stop()
+ return {
+ 'mode': mode,
+ 'requests': st.get('requests'),
+ 'records_parsed': st.get('records'),
+ 'bytes': st.get('bytes_in'),
+ 'bodies': st.get('raw_bodies', [])[:3],
+ 'avg_bytes_per_request': (round(st['bytes_in'] / st['requests'], 1)
+ if st.get('requests') else None),
+ }
+
+
+def integrity(t, seconds=60, batch=25):
+ """Send a slice of history, then read the same epochs off flash and compare."""
+ kill_stale()
+ srv = Server('http', 'integrity', C.OUT, port=C.PORT_HTTP, mode='ok',
+ raw_dump=3)
+ C.cfg_http(t, C.HOST_IP, C.PORT_HTTP, crypto=False, batch=batch,
+ interval=1000, mode='json', path='/ingest')
+ C.tel_reset(t)
+ C.tel_sync(t, wait=30)
+ time.sleep(seconds)
+ recs = srv.records()
+ srv.stop()
+ if not recs:
+ return {'error': 'no records received'}
+
+ by_epoch = {}
+ for r in recs:
+ ts = r.get('ts')
+ if isinstance(ts, int):
+ by_epoch[ts] = r
+
+ # Ground truth for the days those epochs fall on.
+ import datetime as dt
+ days = sorted({dt.datetime.fromtimestamp(e).strftime('%Y%m%d') for e in by_epoch})
+ w = C.web_session()
+ disk = {}
+ schema_keys = None
+ for d in days:
+ r = w.get(f'/download?file=/history/{d}.h5')
+ if r.status_code != 200:
+ continue
+ for sch, epoch, vals in h5.read_series(r.content, nominal_interval_s=60):
+ if schema_keys is None:
+ schema_keys = [(c.id, c.kind, c.scale_exp) for c in sch]
+ if epoch in by_epoch:
+ disk[epoch] = (sch, vals)
+
+ compared = matched = 0
+ mismatches = []
+ missing_on_disk = []
+ for epoch, rec in sorted(by_epoch.items()):
+ if epoch not in disk:
+ missing_on_disk.append(epoch)
+ continue
+ sch, vals = disk[epoch]
+ compared += 1
+ ok = True
+ for c, v in zip(sch, vals):
+ if v == h5.H5_NAN:
+ continue
+ fv = v * (10.0 ** c.scale_exp)
+ # Find the matching key in the telemetry record by value, since
+ # the JSON key encodes slot+hwId rather than the schema id.
+ hit = any(abs(float(x) - fv) < 0.051
+ for k, x in rec.items()
+ if k != 'ts' and isinstance(x, (int, float)))
+ if not hit:
+ ok = False
+ if len(mismatches) < 12:
+ mismatches.append({'epoch': epoch, 'chan_id': c.id,
+ 'disk_value': round(fv, 3),
+ 'record': rec})
+ break
+ if ok:
+ matched += 1
+ return {
+ 'received': len(recs), 'unique_epochs': len(by_epoch),
+ 'days_fetched': days,
+ 'compared': compared, 'matched': matched,
+ 'match_pct': round(100.0 * matched / max(1, compared), 2),
+ 'missing_on_disk': len(missing_on_disk),
+ 'missing_sample': missing_on_disk[:5],
+ 'mismatches': mismatches,
+ 'schema': schema_keys,
+ 'sample_record': recs[0],
+ }
+
+
+def main():
+ kill_stale()
+ t = Target(os.path.join(C.OUT, 'serial_payload.log'))
+ time.sleep(1)
+ out = {}
+ for mode in ('json', 'csv', 'custom'):
+ C.log(f'--- payload mode {mode}')
+ out[mode] = capture(t, mode)
+ C.log(json.dumps({k: out[mode][k] for k in
+ ('requests', 'records_parsed', 'avg_bytes_per_request')}))
+ for b in out[mode]['bodies'][:1]:
+ C.log(' body: ' + b[:300])
+ C.save('phase_payload.json', out)
+
+ C.log('--- restoring json mode for the integrity check')
+ out['integrity'] = integrity(t)
+ C.log(json.dumps({k: out['integrity'].get(k) for k in
+ ('received', 'unique_epochs', 'compared', 'matched',
+ 'match_pct', 'missing_on_disk')}))
+ C.save('phase_payload.json', out)
+ t.close()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/phase_perf.py b/tools/telemetry_bench/phase_perf.py
new file mode 100644
index 0000000..4b19253
--- /dev/null
+++ b/tools/telemetry_bench/phase_perf.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Phase A — telemetry throughput and latency, per transport and batch size.
+
+Each run starts from the same place: `tel reset` puts the cursor back to the
+firmware's 30-day floor, so every run has the same large backlog to chew on and
+the numbers compare. Without that the first run drains the queue and every run
+after it measures an idle device.
+
+Three instruments per run:
+ server what actually arrived (requests, records, bytes, wall clock)
+ /api/status what the device believes (telSent/telFailed/latency/heap)
+ serial whether it rebooted
+"""
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import campaign as C # noqa: E402
+from bench import Target, Server, kill_stale, wait_web # noqa: E402
+
+
+def run_window(t, srv, seconds, label, period=2.0):
+ """Sample the device while telemetry runs, then reduce to one row."""
+ s0 = C.status()
+ m0 = s0.get('metr', {})
+ boots0 = t.port_drops
+ fatal0 = len(t.fatal_lines)
+ samples = []
+ t0 = time.time()
+ while time.time() - t0 < seconds:
+ s = C.status()
+ s['_t'] = round(time.time() - t0, 1)
+ samples.append(s)
+ time.sleep(period)
+ s1 = C.status()
+ m1 = s1.get('metr', {})
+ st = srv.stats() if srv else {}
+
+ lat = [s.get('metr', {}).get('tl') for s in samples if s.get('metr')]
+ lat = [x for x in lat if isinstance(x, int) and x > 0]
+ heap = [s.get('sys', {}).get('heap_f') for s in samples if s.get('sys')]
+ lb = [s.get('metr', {}).get('lb') for s in samples if s.get('metr')]
+ up = [s.get('sys', {}).get('uptime') for s in samples if s.get('sys')]
+ resets = sum(1 for a, b in zip(up, up[1:])
+ if a is not None and b is not None and b < a)
+
+ def d(k):
+ a, b = m0.get(k), m1.get(k)
+ return (b - a) if isinstance(a, int) and isinstance(b, int) else None
+
+ lat_sorted = sorted(set(lat))
+ row = {
+ 'label': label,
+ 'seconds': seconds,
+ 'dev_sent': d('ts'), 'dev_failed': d('tf'), 'dev_retries': d('tr'),
+ 'dev_bytes': d('tb'),
+ 'dev_lat_samples': sorted(lat),
+ 'dev_lat_min': min(lat) if lat else None,
+ 'dev_lat_med': lat_sorted[len(lat_sorted) // 2] if lat_sorted else None,
+ 'dev_lat_max': max(lat) if lat else None,
+ 'heap_min': min([h for h in heap if h], default=None),
+ 'heap_max': max([h for h in heap if h], default=None),
+ 'largest_min': min([x for x in lb if x], default=None),
+ 'largest_max': max([x for x in lb if x], default=None),
+ 'pending_start': s0.get('sys', {}).get('pending'),
+ 'pending_end': s1.get('sys', {}).get('pending'),
+ 'uptime_resets': resets,
+ 'usb_drops': t.port_drops - boots0,
+ 'fatal_lines': len(t.fatal_lines) - fatal0,
+ 'unreachable_polls': sum(1 for s in samples if '_err' in s or '_http' in s),
+ 'srv_requests': st.get('requests') or st.get('publishes'),
+ 'srv_records': st.get('records'),
+ 'srv_bytes': st.get('bytes_in'),
+ 'srv_conns': st.get('conns'),
+ 'srv_connects': st.get('connects'),
+ 'srv_tls_ok': st.get('tls_ok'),
+ 'srv_tls_fail': st.get('tls_failures'),
+ 'srv_ms_p50': st.get('server_ms_p50'),
+ 'srv_epoch_min': st.get('epoch_min'),
+ 'srv_epoch_max': st.get('epoch_max'),
+ }
+ if row['srv_records'] is not None:
+ row['records_per_s'] = round(row['srv_records'] / seconds, 2)
+ row['bytes_per_s'] = round((row['srv_bytes'] or 0) / seconds, 1)
+ if row['srv_requests']:
+ row['s_between_sends'] = round(seconds / row['srv_requests'], 2)
+ row['records_per_send'] = round((row['srv_records'] or 0) / row['srv_requests'], 1)
+ return row, samples
+
+
+def http_matrix(t, tls, port, batches, seconds, tag):
+ rows = []
+ for b in batches:
+ label = f'{"https" if tls else "http"}_batch{b}'
+ C.log(f'--- {label}')
+ kill_stale()
+ srv = Server('http', f'{tag}_{label}', C.OUT, port=port, mode='ok',
+ tls=tls, cert='certs/cert.pem', key='certs/key.pem')
+ C.cfg_http(t, C.HOST_IP, port, crypto=tls, batch=b, interval=1000,
+ mode='json', path='/ingest')
+ C.tel_reset(t)
+ # `tel sync` calls resetBackoff() before it does anything else, so it
+ # is the only way to clear an escalated backoff without a reboot.
+ # Skipping it measures the backoff timer, not the transport.
+ C.tel_sync(t)
+ time.sleep(3)
+ row, samples = run_window(t, srv, seconds, label)
+ row['batch'] = b
+ row['tls'] = tls
+ rows.append(row)
+ C.log(json.dumps({k: row[k] for k in
+ ('label', 'srv_requests', 'srv_records', 'records_per_s',
+ 'dev_lat_med', 'dev_lat_max', 'dev_failed', 'heap_min',
+ 'largest_min', 'usb_drops')}))
+ srv.stop()
+ return rows
+
+
+def main():
+ seconds = int(sys.argv[1]) if len(sys.argv) > 1 else 90
+ kill_stale()
+ t = Target(os.path.join(C.OUT, 'serial_perf.log'))
+ time.sleep(1)
+ out = {'started': time.time(), 'seconds_per_run': seconds, 'rows': []}
+
+ C.log('=== HTTP plain ===')
+ out['rows'] += http_matrix(t, False, C.PORT_HTTP, [1, 10, 25, 50], seconds, 'perf')
+ C.save('phase_perf_http.json', out)
+
+ C.log('=== HTTPS ===')
+ out['rows'] += http_matrix(t, True, C.PORT_HTTPS, [1, 10, 25, 50], seconds, 'perf')
+ C.save('phase_perf_http.json', out)
+
+ t.close()
+ C.log('done')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/phase_survive.py b/tools/telemetry_bench/phase_survive.py
new file mode 100644
index 0000000..7ec4612
--- /dev/null
+++ b/tools/telemetry_bench/phase_survive.py
@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""Phase C — survival against broken servers.
+
+Each fault gets the same three-act structure, because "the device did not
+reboot" is only half the question. The other half is whether it silently threw
+data away.
+
+ act 1 good sink, `tel reset` + `tel sync` → note the last epoch accepted (E1)
+ act 2 swap in the fault, run it for the full window, watch for death
+ act 3 good sink again, `tel sync` → note the first epoch accepted (E2)
+
+If E2 is more than one sampling interval past E1, the device advanced its cursor
+over records no server ever acknowledged: data loss. That check is the point of
+acts 1 and 3 — without them a fault that quietly eats history looks identical to
+one the device shrugged off.
+"""
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import campaign as C # noqa: E402
+from bench import Target, Server, kill_stale # noqa: E402
+
+HIST_INTERVAL_S = 60 # h_int = 1 minute
+
+
+def good_server(tag, port, tls):
+ return Server('http', tag, C.OUT, port=port, mode='ok', tls=tls,
+ cert='certs/cert.pem', key='certs/key.pem')
+
+
+def act_baseline(t, tag, port, tls, batch=10):
+ """Drain a couple of batches through a healthy sink and report where the
+ cursor got to."""
+ kill_stale()
+ srv = good_server(f'{tag}_pre', port, tls)
+ C.cfg_http(t, C.HOST_IP, port, crypto=tls, batch=batch, interval=1000,
+ mode='json', path='/ingest')
+ C.tel_reset(t)
+ C.tel_sync(t)
+ time.sleep(12)
+ st = srv.stats()
+ srv.stop()
+ return {
+ 'requests': st.get('requests'), 'records': st.get('records'),
+ 'epoch_min': st.get('epoch_min'), 'epoch_max': st.get('epoch_max'),
+ }
+
+
+def act_recover(t, tag, port, tls, batch=10, settle=25):
+ kill_stale()
+ srv = good_server(f'{tag}_post', port, tls)
+ C.tel_sync(t)
+ time.sleep(settle)
+ st = srv.stats()
+ srv.stop()
+ return {
+ 'requests': st.get('requests'), 'records': st.get('records'),
+ 'epoch_min': st.get('epoch_min'), 'epoch_max': st.get('epoch_max'),
+ }
+
+
+def run_fault(t, name, port, tls, seconds, server_kw=None, no_server=False,
+ batch=10, host=None):
+ C.log(f'=== fault: {name} ({seconds}s)')
+ pre = act_baseline(t, name, port, tls, batch=batch)
+ C.log(f' baseline: {pre}')
+
+ kill_stale()
+ srv = None
+ if not no_server:
+ kw = dict(port=port, tls=tls, cert='certs/cert.pem', key='certs/key.pem')
+ kw.update(server_kw or {})
+ srv = Server('http', f'{name}_fault', C.OUT, **kw)
+
+ if host:
+ # Faults that live in the address itself: a name that does not resolve,
+ # and an address that swallows the SYN instead of refusing it. Neither
+ # can be produced by a listening socket.
+ C.cfg_http(t, host, port, crypto=tls, batch=batch, interval=1000,
+ mode='json', path='/ingest')
+
+ # Clear the backoff so the window is spent attacking the fault rather than
+ # waiting out a timer inherited from the baseline act.
+ C.tel_sync(t, wait=30)
+
+ boots0, fatal0 = t.port_drops, len(t.fatal_lines)
+ samples = []
+ t0 = time.time()
+ web_fail_streak = 0
+ worst_streak = 0
+ while time.time() - t0 < seconds:
+ s = C.status()
+ s['_t'] = round(time.time() - t0, 1)
+ samples.append(s)
+ if '_err' in s or '_http' in s:
+ web_fail_streak += 1
+ worst_streak = max(worst_streak, web_fail_streak)
+ else:
+ web_fail_streak = 0
+ time.sleep(2)
+
+ st = srv.stats() if srv else {}
+ if srv:
+ srv.stop()
+
+ up = [s.get('sys', {}).get('uptime') for s in samples if s.get('sys')]
+ heap = [s.get('sys', {}).get('heap_f') for s in samples if s.get('sys')]
+ lb = [s.get('metr', {}).get('lb') for s in samples if s.get('metr')]
+ tf = [s.get('metr', {}).get('tf') for s in samples if s.get('metr')]
+ ts_ = [s.get('metr', {}).get('ts') for s in samples if s.get('metr')]
+ resets = sum(1 for a, b in zip(up, up[1:])
+ if a is not None and b is not None and b < a)
+
+ if host:
+ C.cfg_http(t, C.HOST_IP, port, crypto=tls, batch=batch, interval=1000,
+ mode='json', path='/ingest')
+ post = act_recover(t, name, port, tls, batch=batch)
+ C.log(f' recovery: {post}')
+
+ gap = None
+ lost = None
+ if pre.get('epoch_max') and post.get('epoch_min'):
+ gap = post['epoch_min'] - pre['epoch_max']
+ lost = max(0, (gap // HIST_INTERVAL_S) - 1)
+
+ row = {
+ 'fault': name, 'seconds': seconds, 'tls': tls,
+ 'server_kw': server_kw, 'no_server': no_server,
+ 'pre': pre, 'post': post,
+ 'cursor_gap_s': gap,
+ 'records_skipped': lost,
+ 'uptime_resets': resets,
+ 'usb_drops': t.port_drops - boots0,
+ 'fatal_lines': len(t.fatal_lines) - fatal0,
+ 'fatal_text': [l for _, l in t.fatal_lines[fatal0:]][:6],
+ 'web_unreachable_polls': sum(1 for s in samples if '_err' in s or '_http' in s),
+ 'web_worst_streak_polls': worst_streak,
+ 'heap_min': min([h for h in heap if h], default=None),
+ 'heap_max': max([h for h in heap if h], default=None),
+ 'largest_min': min([x for x in lb if x], default=None),
+ 'dev_failed_delta': (tf[-1] - tf[0]) if len(tf) > 1 and None not in (tf[0], tf[-1]) else None,
+ 'dev_sent_delta': (ts_[-1] - ts_[0]) if len(ts_) > 1 and None not in (ts_[0], ts_[-1]) else None,
+ 'srv_conns': st.get('conns'), 'srv_requests': st.get('requests'),
+ 'srv_records': st.get('records'),
+ 'srv_tls_ok': st.get('tls_ok'), 'srv_tls_fail': st.get('tls_failures'),
+ 'uptime_first': up[0] if up else None,
+ 'uptime_last': up[-1] if up else None,
+ }
+ verdict = []
+ if row['usb_drops'] or resets:
+ verdict.append('REBOOT')
+ if row['fatal_lines']:
+ verdict.append('FATAL')
+ if lost:
+ verdict.append(f'DATA-LOSS({lost} rec)')
+ if worst_streak >= 3:
+ verdict.append(f'WEB-STALL({worst_streak * 2}s)')
+ if not post.get('records'):
+ verdict.append('NO-RECOVERY')
+ row['verdict'] = ' '.join(verdict) if verdict else 'SURVIVED'
+ C.log(f' -> {row["verdict"]} '
+ f'(reboots={row["usb_drops"]}/{resets} devFail+{row["dev_failed_delta"]} '
+ f'heapMin={row["heap_min"]} srvConns={row["srv_conns"]})')
+ return row
+
+
+HTTP_FAULTS = [
+ ('refused', dict(no_server=True)),
+ ('blackhole', dict(server_kw={'mode': 'blackhole', 'delay': 120})),
+ ('slow20', dict(server_kw={'mode': 'slow', 'delay': 20})),
+ ('half', dict(server_kw={'mode': 'half'})),
+ ('rst', dict(server_kw={'mode': 'rst'})),
+ ('rst_mid', dict(server_kw={'mode': 'rst_mid'})),
+ ('garbage', dict(server_kw={'mode': 'garbage'})),
+ ('huge1mb', dict(server_kw={'mode': 'huge', 'huge_bytes': 1048576})),
+ ('drip', dict(server_kw={'mode': 'drip', 'drip_ms': 400})),
+ ('error500', dict(server_kw={'mode': 'error500'})),
+ ('error401', dict(server_kw={'mode': 'error401'})),
+ ('close_early', dict(server_kw={'mode': 'close_early'})),
+ # RFC 5737 TEST-NET-1: routed nowhere, so the SYN is swallowed rather than
+ # refused — the connect() blocks instead of failing fast.
+ ('syn_blackhole', dict(no_server=True, host='192.0.2.1')),
+ ('dns_fail', dict(no_server=True, host='nao-existe.invalid')),
+]
+
+TLS_FAULTS = [
+ ('tls_blackhole', dict(server_kw={'mode': 'ok', 'tls_fault': 'blackhole', 'delay': 120})),
+ ('tls_garbage', dict(server_kw={'mode': 'ok', 'tls_fault': 'garbage'})),
+ ('tls_rst', dict(server_kw={'mode': 'ok', 'tls_fault': 'rst'})),
+ ('tls_slow20', dict(server_kw={'mode': 'ok', 'tls_fault': 'slow', 'delay': 20})),
+ ('tls_refused', dict(no_server=True)),
+ ('tls_error500', dict(server_kw={'mode': 'error500'})),
+ ('tls_blackhole_http', dict(server_kw={'mode': 'blackhole', 'delay': 120})),
+]
+
+
+def main():
+ seconds = int(sys.argv[1]) if len(sys.argv) > 1 else 120
+ which = sys.argv[2] if len(sys.argv) > 2 else 'all'
+ kill_stale()
+ t = Target(os.path.join(C.OUT, 'serial_survive.log'))
+ time.sleep(1)
+ rows = []
+ outname = f'phase_survive_{which}.json'
+
+ if which in ('all', 'http'):
+ for name, kw in HTTP_FAULTS:
+ rows.append(run_fault(t, name, C.PORT_HTTP, False, seconds, **kw))
+ C.save(outname, {'rows': rows})
+
+ if which in ('all', 'tls'):
+ for name, kw in TLS_FAULTS:
+ rows.append(run_fault(t, name, C.PORT_HTTPS, True, seconds, **kw))
+ C.save(outname, {'rows': rows})
+
+ C.save(outname, {'rows': rows})
+ t.close()
+ for r in rows:
+ C.log(f'{r["fault"]:22s} {r["verdict"]}')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/rescore.py b/tools/telemetry_bench/rescore.py
new file mode 100644
index 0000000..97dd20d
--- /dev/null
+++ b/tools/telemetry_bench/rescore.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Re-score the data-loss check, counting what the FAULT server received.
+
+The original check compared only the baseline server's last accepted epoch
+against the recovery server's first one. That is right for a fault that accepts
+nothing — which is most of them — and wrong for any fault that is a working
+server in every respect but the one being tested.
+
+`mq_no_pingresp` is exactly that: a broker that answers CONNECT and PUBLISH
+normally and only ignores PINGREQ. It took 780 records during its window, the
+cursor advanced over them legitimately, and the naive check called the whole
+span lost. Scored properly the answer is zero.
+
+ skipped = (post.epoch_min - max(pre.epoch_max, fault.epoch_max)) / 60 - 1
+
+Rows where the fault server reported no epochs are unchanged, so this cannot
+launder a real loss into a pass — it only removes the false ones.
+"""
+import json
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+OUT = os.path.join(HERE, 'results')
+INTERVAL = 60
+
+
+def rescore(row):
+ pre = row.get('pre') or {}
+ post = row.get('post') or {}
+ fault_max = row.get('srv_epoch_max')
+ pre_max = pre.get('epoch_max')
+ post_min = post.get('epoch_min')
+ if not pre_max or not post_min:
+ return None, None, None
+ delivered = max([e for e in (pre_max, fault_max) if e] or [pre_max])
+ gap = post_min - delivered
+ skipped = max(0, (gap // INTERVAL) - 1) if gap > 0 else 0
+ return gap, skipped, delivered
+
+
+def main():
+ files = [
+ ('phase_survive_http.json', 'rows', 'HTTP'),
+ ('phase_survive_tls.json', 'rows', 'HTTPS'),
+ ('phase_mqtt_plain.json', 'faults', 'MQTT'),
+ ('phase_mqtt_tls.json', 'faults', 'MQTTS'),
+ ]
+ print(f"{'transporte':<8}{'falha':<24}{'skip antigo':>12}{'skip corrigido':>16}"
+ f"{'gap corrigido':>15}{'aceitou na falha':>18}")
+ total_old = total_new = 0
+ out = []
+ for fn, key, label in files:
+ p = os.path.join(OUT, fn)
+ if not os.path.exists(p):
+ continue
+ with open(p) as fh:
+ d = json.load(fh)
+ for r in d.get(key, []):
+ gap, skipped, delivered = rescore(r)
+ old = r.get('records_skipped')
+ got = r.get('srv_records')
+ if skipped is None:
+ continue
+ total_old += old or 0
+ total_new += skipped
+ flag = ' <-- corrigido' if (old or 0) != skipped else ''
+ print(f"{label:<8}{r.get('fault',''):<24}{str(old):>12}{skipped:>16}"
+ f"{str(gap):>15}{str(got):>18}{flag}")
+ out.append({'transport': label, 'fault': r.get('fault'),
+ 'skipped_naive': old, 'skipped_corrected': skipped,
+ 'gap_corrected': gap, 'fault_server_records': got,
+ 'verdict_naive': r.get('verdict')})
+ print()
+ print(f'TOTAL registros "perdidos": ingênuo={total_old} corrigido={total_new}')
+ with open(os.path.join(OUT, 'rescore.json'), 'w') as fh:
+ json.dump({'rows': out, 'total_naive': total_old,
+ 'total_corrected': total_new}, fh, indent=1)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/revalidate.py b/tools/telemetry_bench/revalidate.py
new file mode 100644
index 0000000..579a63d
--- /dev/null
+++ b/tools/telemetry_bench/revalidate.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+"""Re-run, against the fixed firmware, every test that failed or exposed a defect.
+
+A fix is only a fix if the thing that caught it now passes and nothing else
+moved. So this runs three groups:
+
+ regressions the faults that killed or misreported (huge1mb, drip, error500,
+ error401) — same servers, same windows, compared to the numbers
+ the broken build produced
+ fixes the defects proved by inspection rather than by a crash
+ (CSV header, /api/ls completeness, MQTT credentials)
+ no-regression a perf spot-check at the two batch sizes that matter, to show
+ the bounded read loops did not cost throughput
+"""
+import datetime as dt
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import campaign as C # noqa: E402
+import phase_survive as PS # noqa: E402
+from bench import Target, Server, kill_stale # noqa: E402
+
+OUT = C.OUT
+
+
+def reboot_and_wait(t, why=''):
+ """Give the run a clean network stack.
+
+ D14: a burst of 1 MB responses leaks the 12-entry lwIP pbuf pool and the web
+ server goes silent until reboot — with the device otherwise alive. Since
+ /api/status is one of the three instruments, a run that starts on an
+ exhausted pool measures the leak instead of the fix. Reboot between groups
+ and let the ordering keep the leaky test last.
+ """
+ C.log(f'--- reboot {why}')
+ t.send('enable', wait=1.5)
+ t.send('reload confirm', wait=3.0)
+ time.sleep(8)
+ back = C.wait_web(C.DEV, timeout=120)
+ C.web_session(force=True)
+ C.log(f' web back after {back}s')
+ return back
+
+
+def group_regressions(t, seconds=120):
+ rows = []
+ # huge1mb LAST on purpose: it is the one that leaks the pbuf pool, so
+ # anything after it would be measuring D14 rather than its own fault.
+ for name, kw in [
+ ('error500', dict(server_kw={'mode': 'error500'})),
+ ('error401', dict(server_kw={'mode': 'error401'})),
+ ('drip', dict(server_kw={'mode': 'drip', 'drip_ms': 400})),
+ ]:
+ rows.append(PS.run_fault(t, name, C.PORT_HTTP, False, seconds, **kw))
+ reboot_and_wait(t, f'after {name}')
+ # The TLS kill is reached through a different door — WiFiClientSecure's
+ # available() drives the BearSSL engine, so the same unbounded drain in
+ # HTTPClient::disconnect() blocks there too — but it should fall to the
+ # same fix. Verify rather than assume.
+ rows.append(PS.run_fault(t, 'tls_slow20', C.PORT_HTTPS, True, seconds,
+ server_kw={'mode': 'ok', 'tls_fault': 'slow',
+ 'delay': 20}))
+ reboot_and_wait(t, 'after tls_slow20')
+ rows.append(PS.run_fault(t, 'huge1mb', C.PORT_HTTP, False, seconds,
+ server_kw={'mode': 'huge', 'huge_bytes': 1048576}))
+ return rows
+
+
+def check_csv_header(t, seconds=45):
+ kill_stale()
+ srv = Server('http', 'reval_csv', OUT, port=C.PORT_HTTP, mode='ok',
+ raw_dump=6)
+ C.cfg_http(t, C.HOST_IP, C.PORT_HTTP, crypto=False, batch=5,
+ interval=2000, mode='csv', path='/ingest')
+ C.tel_reset(t)
+ C.tel_sync(t, wait=30)
+ time.sleep(seconds)
+ st = srv.stats()
+ srv.stop()
+ body = None
+ for b in st.get('raw_bodies', []):
+ if b.startswith('timestamp;'):
+ body = b
+ break
+ res = {'found_csv_body': body is not None}
+ if body:
+ lines = [l for l in body.split('\n') if l]
+ hdr = lines[0].split(';')
+ widths = {len(l.split(';')) for l in lines[1:]}
+ res.update({
+ 'header_cols': len(hdr),
+ 'row_cols': sorted(widths),
+ 'match': len(widths) == 1 and len(hdr) == list(widths)[0],
+ 'header': lines[0][:400],
+ 'first_row': lines[1][:200] if len(lines) > 1 else None,
+ })
+ # Put the transport back the way the other checks expect it.
+ C.cfg_http(t, C.HOST_IP, C.PORT_HTTP, crypto=False, batch=10,
+ interval=1000, mode='json', path='/ingest')
+ return res
+
+
+def check_ls_complete():
+ """Does /api/ls now report everything — and say so when it cannot?"""
+ w = C.web_session(force=True)
+ listings = []
+ for _ in range(4):
+ try:
+ j = w.get('/api/ls?dir=/history').json()
+ listings.append({'n': len(j.get('entries', [])),
+ 'truncated': j.get('truncated', False),
+ 'names': sorted(e['n'] for e in j.get('entries', []))})
+ except Exception as e:
+ listings.append({'err': str(e)[:80]})
+ w = C.web_session(force=True)
+ time.sleep(1.5)
+ ok = [l for l in listings if 'names' in l]
+ stable = len({tuple(l['names']) for l in ok}) <= 1 if ok else False
+ inv = os.path.join(OUT, 'history_inventory.json')
+ truth = None
+ if os.path.exists(inv):
+ with open(inv) as fh:
+ truth = json.load(fh)
+ res = {
+ 'listings': [{k: v for k, v in l.items() if k != 'names'} for l in listings],
+ 'stable_across_calls': stable,
+ 'counts': [l.get('n') for l in listings],
+ }
+ if truth and ok:
+ real = {p['file'] for p in truth['per_file']}
+ got = set(ok[-1]['names'])
+ res.update({
+ 'files_on_flash': len(real),
+ 'files_listed': len(got),
+ 'missing_from_listing': sorted(real - got),
+ 'complete': real.issubset(got),
+ })
+ return res
+
+
+def check_mqtt_credentials(t, seconds=70):
+ """D1: does the password typed on the config page reach the broker?"""
+ import phase_mqtt as PM
+ res = PM.config_fidelity(t, C.web_session(), C.PORT_MQTT, seconds=seconds)
+ # Put the transport back on HTTP before anything else runs. telTransport is
+ # read once, in TelemetryManager::begin( ), so only a commit_all + reboot
+ # moves it — a CLI `tel server`/`tel port` cannot. Skipping this left the
+ # device speaking MQTT at an HTTP sink and the next perf run measured
+ # 0 records/s, which reads exactly like a throughput regression and is not.
+ C.log('--- restoring HTTP transport after the MQTT check')
+ C.commit(C.web_session(), {
+ 't_transport': '0', 't_sec': '0', 't_srv': C.HOST_IP,
+ 't_port': str(C.PORT_HTTP), 't_path': '/ingest',
+ 't_int': '1000', 't_bat': '50', 't_mode': '0',
+ 'm_topic': 'simut/data', 'm_cid': '', 'm_user': '',
+ 'm_qos': '0', 'm_retain': '0', 'm_ka': '60'})
+ C.web_session(force=True)
+ time.sleep(5)
+ return res
+
+
+def perf_spotcheck(t, seconds=90):
+ import phase_perf as PP
+ rows = []
+ rows += PP.http_matrix(t, False, C.PORT_HTTP, [50], seconds, 'reval')
+ rows += PP.http_matrix(t, True, C.PORT_HTTPS, [50], seconds, 'reval')
+ return rows
+
+
+def main():
+ which = sys.argv[1] if len(sys.argv) > 1 else 'all'
+ kill_stale()
+ t = Target(os.path.join(OUT, 'serial_reval.log'))
+ time.sleep(1)
+ out = {'started': dt.datetime.now().isoformat()}
+
+ if which in ('all', 'fixes'):
+ C.log('=== CSV header')
+ out['csv'] = check_csv_header(t)
+ C.log(json.dumps({k: v for k, v in out['csv'].items() if k != 'header'}))
+ C.log(' header: ' + str(out['csv'].get('header'))[:300])
+ C.save('revalidate.json', out)
+
+ C.log('=== /api/ls completeness')
+ out['api_ls'] = check_ls_complete()
+ C.log(json.dumps({k: v for k, v in out['api_ls'].items()
+ if k != 'listings'}))
+ C.save('revalidate.json', out)
+
+ C.log('=== MQTT credentials on the wire')
+ out['mqtt_fidelity'] = check_mqtt_credentials(t)
+ C.log(json.dumps(out['mqtt_fidelity']['checks']))
+ C.save('revalidate.json', out)
+
+ if which in ('all', 'perf'):
+ C.log('=== perf spot-check (no-regression)')
+ out['perf'] = perf_spotcheck(t)
+ C.save('revalidate.json', out)
+
+ # Last, because huge1mb leaks the pbuf pool and takes the web server with
+ # it (D14). Everything that needs /api/status has already run by now.
+ if which in ('all', 'regress'):
+ C.log('=== regressions: the faults that failed on the broken build')
+ out['regressions'] = group_regressions(t)
+ for r in out['regressions']:
+ C.log(f' {r["fault"]:10s} {r["verdict"]:40s} '
+ f'reboots={r["usb_drops"]} devFail+{r["dev_failed_delta"]}')
+ C.save('revalidate.json', out)
+
+ t.close()
+ C.save('revalidate.json', out)
+ C.log('revalidation done')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/run_rest.sh b/tools/telemetry_bench/run_rest.sh
new file mode 100755
index 0000000..d540b41
--- /dev/null
+++ b/tools/telemetry_bench/run_rest.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Runs the remaining phases back to back. Only one may hold the serial port at
+# a time, so this is strictly sequential by design.
+set -u
+cd "$(dirname "$0")"
+exec >> results/run_rest.out 2>&1
+
+stamp() { echo "=== $(date +%H:%M:%S) $*"; }
+
+stamp "waiting for phase_perf to finish"
+while pgrep -f "[p]hase_perf.py" > /dev/null; do sleep 5; done
+
+stamp "phase_payload"
+python3 phase_payload.py
+
+stamp "phase_drain"
+python3 phase_drain.py 2400
+
+stamp "phase_survive http"
+python3 phase_survive.py 120 http
+
+stamp "phase_survive tls"
+python3 phase_survive.py 120 tls
+
+stamp "phase_mqtt plain"
+python3 phase_mqtt.py 90 plain
+
+stamp "phase_mqtt tls"
+python3 phase_mqtt.py 90 tls
+
+stamp "ALL PHASES DONE"
diff --git a/tools/telemetry_bench/run_rest2.sh b/tools/telemetry_bench/run_rest2.sh
new file mode 100755
index 0000000..32e769b
--- /dev/null
+++ b/tools/telemetry_bench/run_rest2.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Second leg: the two runs that need the MQTT phases out of the way first.
+set -u
+cd "$(dirname "$0")"
+exec >> results/run_rest2.out 2>&1
+
+stamp() { echo "=== $(date +%H:%M:%S) $*"; }
+
+stamp "waiting for leg 1"
+while pgrep -f "[r]un_rest.sh" > /dev/null; do sleep 10; done
+
+stamp "phase_mqtt_oversize"
+python3 phase_mqtt_oversize.py
+
+stamp "restoring HTTP transport for the full drain (reboot)"
+python3 - <<'PY'
+import sys, time
+sys.path.insert(0, '.')
+import campaign as C
+w = C.web_session()
+C.commit(w, {'t_transport': '0', 't_sec': '0', 't_srv': C.HOST_IP,
+ 't_port': str(C.PORT_HTTP), 't_path': '/ingest',
+ 't_int': '1000', 't_bat': '50', 't_mode': '0',
+ 't_glob': '{"dev":"{DEV}","mac":"{MAC}","data":[{DATA}]}',
+ 't_line': '{"ts":{TS},"t0_ID":{t0},"u0_ID":{u0}}',
+ 't_sep': ','})
+C.web_session(force=True)
+time.sleep(5)
+print('transport back on HTTP')
+PY
+
+stamp "phase_drain_full"
+python3 phase_drain_full.py 4200
+
+stamp "LEG 2 DONE"
diff --git a/tools/telemetry_bench/run_rest3.sh b/tools/telemetry_bench/run_rest3.sh
new file mode 100755
index 0000000..7f6f1ef
--- /dev/null
+++ b/tools/telemetry_bench/run_rest3.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Third leg: true history inventory, once nothing else is loading the device.
+set -u
+cd "$(dirname "$0")"
+exec >> results/run_rest3.out 2>&1
+
+stamp() { echo "=== $(date +%H:%M:%S) $*"; }
+
+stamp "waiting for leg 2"
+while pgrep -f "[r]un_rest2.sh" > /dev/null; do sleep 15; done
+
+stamp "enumerate_history (by date, not by /api/ls)"
+python3 enumerate_history.py
+
+stamp "LEG 3 DONE"
diff --git a/tools/telemetry_bench/server_http.py b/tools/telemetry_bench/server_http.py
new file mode 100644
index 0000000..de2b800
--- /dev/null
+++ b/tools/telemetry_bench/server_http.py
@@ -0,0 +1,393 @@
+#!/usr/bin/env python3
+"""Instrumented HTTP/HTTPS telemetry sink with fault injection.
+
+One process, one port, one mode. The device points its telemetry at it and the
+server records every byte, the wall-clock of every request, and every record it
+could parse out of the payload. Modes other than `ok` are deliberate failures —
+each one models something a real server does when it goes wrong, so the device's
+survival can be measured against a named fault instead of "the network was bad".
+
+Metrics land in a JSON file (--stats) that the orchestrator reads; the raw
+records land in an NDJSON file (--records) so the drained history can be
+compared against what is actually on the device.
+
+Modes
+-----
+ok 200 with the same body shape as the user's real server.
+error500 valid HTTP 500 — device must NOT advance its cursor.
+error401 valid HTTP 401.
+blackhole accept the socket, read the request, answer nothing, hold open.
+slow N accept, then answer only after --delay seconds.
+half send a partial status line and close.
+rst accept then RST (SO_LINGER 0) before reading.
+rst_mid read the request, send half the headers, then RST.
+garbage answer with non-HTTP bytes.
+huge answer 200 with a --huge-bytes body.
+drip answer one byte every --drip-ms ms (slowloris, server side).
+close_early read only the headers, close before the body arrives.
+"""
+import argparse
+import json
+import os
+import re
+import socket
+import ssl
+import struct
+import sys
+import threading
+import time
+
+STATS_LOCK = threading.Lock()
+
+
+class Stats:
+ def __init__(self, path, records_path):
+ self.path = path
+ self.records_path = records_path
+ self.started = time.time()
+ self.conns = 0
+ self.requests = 0
+ self.bytes_in = 0
+ self.records = 0
+ self.batches = [] # per-request: {t, bytes, n, ms, first_epoch, last_epoch}
+ self.tls_failures = 0
+ self.tls_ok = 0
+ self.errors = []
+ self.raw_bodies = []
+ self.epochs_seen = set()
+ self._rf = open(records_path, 'a') if records_path else None
+
+ def add_batch(self, entry, recs):
+ with STATS_LOCK:
+ self.requests += 1
+ self.batches.append(entry)
+ self.records += len(recs)
+ if self._rf:
+ for r in recs:
+ self._rf.write(json.dumps(r, separators=(',', ':')) + '\n')
+ self._rf.flush()
+ for r in recs:
+ ts = r.get('ts')
+ if ts is not None:
+ self.epochs_seen.add(ts)
+
+ def dump(self):
+ with STATS_LOCK:
+ lat = [b['ms'] for b in self.batches if b.get('ms') is not None]
+ lat_sorted = sorted(lat)
+
+ def pct(p):
+ if not lat_sorted:
+ return None
+ k = min(len(lat_sorted) - 1, int(round((p / 100.0) * (len(lat_sorted) - 1))))
+ return lat_sorted[k]
+
+ d = {
+ 'started': self.started,
+ 'now': time.time(),
+ 'elapsed_s': round(time.time() - self.started, 1),
+ 'conns': self.conns,
+ 'requests': self.requests,
+ 'bytes_in': self.bytes_in,
+ 'records': self.records,
+ 'unique_epochs': len(self.epochs_seen),
+ 'epoch_min': min(self.epochs_seen) if self.epochs_seen else None,
+ 'epoch_max': max(self.epochs_seen) if self.epochs_seen else None,
+ 'tls_ok': self.tls_ok,
+ 'tls_failures': self.tls_failures,
+ 'errors': self.errors[-40:],
+ 'raw_bodies': self.raw_bodies,
+ 'server_ms_min': min(lat) if lat else None,
+ 'server_ms_p50': pct(50),
+ 'server_ms_p90': pct(90),
+ 'server_ms_max': max(lat) if lat else None,
+ 'batches': self.batches[-400:],
+ }
+ tmp = self.path + '.tmp'
+ with open(tmp, 'w') as fh:
+ json.dump(d, fh, indent=1)
+ os.replace(tmp, self.path)
+ return d
+
+
+def parse_records(body):
+ """Pull {"ts":...} objects out of the JSON payload.
+
+ Tolerant on purpose: the point is to count what arrived even when the
+ device truncates, so a strict json.loads that throws would hide the very
+ failure being measured.
+ """
+ recs = []
+ try:
+ obj = json.loads(body)
+ if isinstance(obj, list):
+ return [r for r in obj if isinstance(r, dict)]
+ if isinstance(obj, dict):
+ for k in ('data', 'records', 'points'):
+ if isinstance(obj.get(k), list):
+ return [r for r in obj[k] if isinstance(r, dict)]
+ return [obj]
+ except Exception:
+ pass
+ # Fallback: brace-matched scan.
+ depth = 0
+ start = None
+ for i, c in enumerate(body):
+ if c == '{':
+ if depth == 0:
+ start = i
+ depth += 1
+ elif c == '}':
+ depth -= 1
+ if depth == 0 and start is not None:
+ try:
+ recs.append(json.loads(body[start:i + 1]))
+ except Exception:
+ pass
+ start = None
+ return recs
+
+
+def read_request(conn, stats, timeout):
+ conn.settimeout(timeout)
+ buf = b''
+ # headers
+ while b'\r\n\r\n' not in buf:
+ chunk = conn.recv(4096)
+ if not chunk:
+ return None, buf
+ buf += chunk
+ if len(buf) > 1 << 20:
+ break
+ head, _, rest = buf.partition(b'\r\n\r\n')
+ m = re.search(rb'Content-Length:\s*(\d+)', head, re.I)
+ want = int(m.group(1)) if m else 0
+ body = rest
+ while len(body) < want:
+ chunk = conn.recv(min(65536, want - len(body)))
+ if not chunk:
+ break
+ body += chunk
+ return head.decode('latin-1', 'replace'), body
+
+
+def handle(conn, addr, args, stats):
+ t0 = time.time()
+ with STATS_LOCK:
+ stats.conns += 1
+ mode = args.mode
+ try:
+ if mode == 'rst':
+ conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
+ struct.pack('ii', 1, 0))
+ conn.close()
+ return
+
+ head, body = read_request(conn, stats, args.read_timeout)
+ if head is None:
+ return
+ with STATS_LOCK:
+ stats.bytes_in += len(body)
+
+ text = body.decode('utf-8', 'replace') if body else ''
+ if args.raw_dump:
+ with STATS_LOCK:
+ if len(stats.raw_bodies) < args.raw_dump:
+ stats.raw_bodies.append(text[:4096])
+ recs = parse_records(text) if body else []
+ epochs = [r.get('ts') for r in recs if isinstance(r.get('ts'), int)]
+ entry = {
+ 't': round(t0 - stats.started, 3),
+ 'bytes': len(body),
+ 'n': len(recs),
+ 'first_epoch': min(epochs) if epochs else None,
+ 'last_epoch': max(epochs) if epochs else None,
+ 'ms': None,
+ }
+
+ if mode == 'ok':
+ resp = json.dumps({'status': 'ok', 'msg': 'Salvo', 'pts': len(recs)}).encode()
+ conn.sendall(b'HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n'
+ b'Content-Length: ' + str(len(resp)).encode() +
+ b'\r\nConnection: close\r\n\r\n' + resp)
+ elif mode == 'slow':
+ time.sleep(args.delay)
+ resp = json.dumps({'status': 'ok', 'pts': len(recs)}).encode()
+ conn.sendall(b'HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n'
+ b'Content-Length: ' + str(len(resp)).encode() +
+ b'\r\nConnection: close\r\n\r\n' + resp)
+ elif mode == 'error500':
+ resp = b'{"status":"error","msg":"internal"}'
+ conn.sendall(b'HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n'
+ b'Content-Length: ' + str(len(resp)).encode() +
+ b'\r\nConnection: close\r\n\r\n' + resp)
+ elif mode == 'error401':
+ resp = b'{"error":"unauthorized"}'
+ conn.sendall(b'HTTP/1.1 401 Unauthorized\r\nContent-Type: application/json\r\n'
+ b'Content-Length: ' + str(len(resp)).encode() +
+ b'\r\nConnection: close\r\n\r\n' + resp)
+ elif mode == 'blackhole':
+ # Answer nothing. Hold the socket open past anything the device
+ # could reasonably wait for, then drop it.
+ time.sleep(args.delay)
+ elif mode == 'half':
+ conn.sendall(b'HTTP/1.1 200 OK\r\nContent-Len')
+ conn.close()
+ return
+ elif mode == 'rst_mid':
+ conn.sendall(b'HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n')
+ time.sleep(0.05)
+ conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
+ struct.pack('ii', 1, 0))
+ conn.close()
+ return
+ elif mode == 'garbage':
+ conn.sendall(os.urandom(512))
+ conn.close()
+ return
+ elif mode == 'huge':
+ n = args.huge_bytes
+ conn.sendall(b'HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n'
+ b'Content-Length: ' + str(n).encode() + b'\r\n\r\n')
+ blob = b'A' * 4096
+ sent = 0
+ while sent < n:
+ k = min(len(blob), n - sent)
+ conn.sendall(blob[:k])
+ sent += k
+ elif mode == 'drip':
+ resp = (b'HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n'
+ b'Content-Length: 2\r\n\r\n{}')
+ for b in resp:
+ conn.sendall(bytes([b]))
+ time.sleep(args.drip_ms / 1000.0)
+ elif mode == 'close_early':
+ conn.close()
+ return
+ else:
+ raise SystemExit(f'unknown mode {mode}')
+
+ entry['ms'] = int((time.time() - t0) * 1000)
+ stats.add_batch(entry, recs)
+ except Exception as e:
+ with STATS_LOCK:
+ stats.errors.append(f'{type(e).__name__}: {e}')
+ finally:
+ try:
+ conn.close()
+ except Exception:
+ pass
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('--port', type=int, required=True)
+ ap.add_argument('--bind', default='0.0.0.0')
+ ap.add_argument('--tls', action='store_true')
+ ap.add_argument('--cert', default='certs/cert.pem')
+ ap.add_argument('--key', default='certs/key.pem')
+ ap.add_argument('--mode', default='ok')
+ ap.add_argument('--delay', type=float, default=30.0)
+ ap.add_argument('--drip-ms', type=float, default=500.0)
+ ap.add_argument('--huge-bytes', type=int, default=1 << 20)
+ ap.add_argument('--read-timeout', type=float, default=30.0)
+ ap.add_argument('--stats', default='stats.json')
+ ap.add_argument('--records', default='')
+ ap.add_argument('--raw-dump', type=int, default=0,
+ help='keep the first N request bodies verbatim, for '
+ 'checking the payload builders (json/csv/custom)')
+ # TLS fault modes act before any HTTP is spoken.
+ ap.add_argument('--tls-fault', default='',
+ choices=['', 'blackhole', 'garbage', 'rst', 'slow'],
+ help='blackhole: accept and never handshake; garbage: junk '
+ 'bytes instead of ServerHello; rst: RST at accept; '
+ 'slow: sleep --delay then handshake')
+ args = ap.parse_args()
+
+ stats = Stats(args.stats, args.records)
+
+ srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ srv.bind((args.bind, args.port))
+ srv.listen(16)
+ ctx = None
+ if args.tls:
+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ ctx.load_cert_chain(args.cert, args.key)
+ # BearSSL on the RP2040 speaks TLS 1.2 with ECDHE-RSA-AES-GCM. Pinning
+ # to 1.2 keeps the negotiation identical to the user's real server.
+ ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+ ctx.maximum_version = ssl.TLSVersion.TLSv1_2
+
+ def dumper():
+ while True:
+ time.sleep(2)
+ try:
+ stats.dump()
+ except Exception:
+ pass
+
+ threading.Thread(target=dumper, daemon=True).start()
+
+ label = ('https' if args.tls else 'http')
+ print(f'[{label}] listening on {args.bind}:{args.port} mode={args.mode} '
+ f'tls_fault={args.tls_fault or "-"}', flush=True)
+
+ while True:
+ try:
+ conn, addr = srv.accept()
+ except OSError:
+ break
+ conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+ def wrapped(conn=conn, addr=addr):
+ if args.tls:
+ f = args.tls_fault
+ if f == 'rst':
+ conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
+ struct.pack('ii', 1, 0))
+ with STATS_LOCK:
+ stats.conns += 1
+ conn.close()
+ return
+ if f == 'blackhole':
+ with STATS_LOCK:
+ stats.conns += 1
+ time.sleep(args.delay)
+ conn.close()
+ return
+ if f == 'garbage':
+ with STATS_LOCK:
+ stats.conns += 1
+ try:
+ conn.recv(4096)
+ conn.sendall(os.urandom(2048))
+ except Exception:
+ pass
+ conn.close()
+ return
+ if f == 'slow':
+ time.sleep(args.delay)
+ try:
+ conn.settimeout(args.read_timeout)
+ tconn = ctx.wrap_socket(conn, server_side=True)
+ with STATS_LOCK:
+ stats.tls_ok += 1
+ except Exception as e:
+ with STATS_LOCK:
+ stats.tls_failures += 1
+ stats.errors.append(f'TLS: {type(e).__name__}: {e}')
+ try:
+ conn.close()
+ except Exception:
+ pass
+ return
+ handle(tconn, addr, args, stats)
+ else:
+ handle(conn, addr, args, stats)
+
+ threading.Thread(target=wrapped, daemon=True).start()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/server_mqtt.py b/tools/telemetry_bench/server_mqtt.py
new file mode 100644
index 0000000..4fd89b8
--- /dev/null
+++ b/tools/telemetry_bench/server_mqtt.py
@@ -0,0 +1,475 @@
+#!/usr/bin/env python3
+"""Instrumented MQTT 3.1.1 broker with fault injection (plain and TLS).
+
+Written from the wire format rather than wrapping mosquitto, because the point
+is to misbehave on purpose: refuse the CONNECT, answer half a CONNACK, drop the
+socket exactly when the first PUBLISH lands. A real broker has no switch for
+any of that.
+
+Speaks enough of the protocol for the device's PubSubClient: CONNECT/CONNACK,
+PUBLISH (QoS 0 and 1), PUBACK, SUBSCRIBE/SUBACK, PINGREQ/PINGRESP, DISCONNECT.
+Every published message is timestamped and written to --records as NDJSON.
+
+Modes
+-----
+ok full broker.
+rst RST at accept, before CONNECT.
+no_connack read CONNECT, answer nothing, hold the socket.
+slow_connack answer CONNACK after --delay seconds.
+half_connack send 2 of the 4 CONNACK bytes and stall.
+connack_refuse CONNACK rc=5 (not authorized).
+connack_badproto CONNACK rc=1 (unacceptable protocol version).
+connack_badid CONNACK rc=2 (identifier rejected).
+connack_unavail CONNACK rc=3 (server unavailable).
+drop_after_connack CONNACK ok, then close immediately.
+drop_on_publish CONNACK ok, close when the first PUBLISH arrives.
+rst_on_publish CONNACK ok, RST when the first PUBLISH arrives.
+garbage send junk instead of CONNACK.
+no_pingresp full broker except PINGREQ is ignored (keepalive death).
+"""
+import argparse
+import json
+import os
+import socket
+import ssl
+import struct
+import threading
+import time
+
+LOCK = threading.Lock()
+
+CONNECT, CONNACK, PUBLISH, PUBACK = 1, 2, 3, 4
+SUBSCRIBE, SUBACK, PINGREQ, PINGRESP, DISCONNECT = 8, 9, 12, 13, 14
+
+
+class Stats:
+ def __init__(self, path, records_path):
+ self.path = path
+ self.started = time.time()
+ self.conns = 0
+ self.connects = 0
+ self.connacks = 0
+ self.publishes = 0
+ self.bytes_in = 0
+ self.records = 0
+ self.pings = 0
+ self.tls_ok = 0
+ self.tls_failures = 0
+ self.errors = []
+ self.msgs = [] # per-publish: {t, topic, bytes, n, first_epoch, last_epoch}
+ self.epochs_seen = set()
+ self.client_ids = []
+ self.qos_seen = {}
+ self.retain_seen = {}
+ self.will_topics = []
+ self.status_msgs = []
+ self.connect_frames = [] # full CONNECT fields, for config-fidelity checks
+ self.connect_intervals = []
+ self._last_connect = None
+ self._rf = open(records_path, 'a') if records_path else None
+
+ def note_connect(self, cid):
+ with LOCK:
+ self.connects += 1
+ if cid not in self.client_ids:
+ self.client_ids.append(cid)
+ now = time.time()
+ if self._last_connect is not None:
+ self.connect_intervals.append(round(now - self._last_connect, 2))
+ self._last_connect = now
+
+ def note_publish(self, topic, payload, qos=0, retain=False, dup=False):
+ recs = []
+ txt = payload.decode('utf-8', 'replace')
+ try:
+ obj = json.loads(txt)
+ recs = obj if isinstance(obj, list) else [obj]
+ except Exception:
+ depth, start = 0, None
+ for i, c in enumerate(txt):
+ if c == '{':
+ if depth == 0:
+ start = i
+ depth += 1
+ elif c == '}':
+ depth -= 1
+ if depth == 0 and start is not None:
+ try:
+ recs.append(json.loads(txt[start:i + 1]))
+ except Exception:
+ pass
+ start = None
+ recs = [r for r in recs if isinstance(r, dict)]
+ data = [r for r in recs if 'ts' in r]
+ epochs = [r['ts'] for r in data if isinstance(r.get('ts'), int)]
+ with LOCK:
+ self.publishes += 1
+ self.bytes_in += len(payload)
+ self.records += len(data)
+ self.msgs.append({
+ 't': round(time.time() - self.started, 3),
+ 'topic': topic,
+ 'bytes': len(payload),
+ 'n': len(data),
+ 'qos': qos, 'retain': retain, 'dup': dup,
+ 'first_epoch': min(epochs) if epochs else None,
+ 'last_epoch': max(epochs) if epochs else None,
+ })
+ self.qos_seen[qos] = self.qos_seen.get(qos, 0) + 1
+ self.retain_seen[bool(retain)] = self.retain_seen.get(bool(retain), 0) + 1
+ for e in epochs:
+ self.epochs_seen.add(e)
+ if self._rf:
+ for r in data:
+ self._rf.write(json.dumps(r, separators=(',', ':')) + '\n')
+ self._rf.flush()
+
+ def dump(self):
+ with LOCK:
+ d = {
+ 'started': self.started,
+ 'elapsed_s': round(time.time() - self.started, 1),
+ 'conns': self.conns,
+ 'connects': self.connects,
+ 'connacks': self.connacks,
+ 'publishes': self.publishes,
+ 'records': self.records,
+ 'bytes_in': self.bytes_in,
+ 'pings': self.pings,
+ 'unique_epochs': len(self.epochs_seen),
+ 'epoch_min': min(self.epochs_seen) if self.epochs_seen else None,
+ 'epoch_max': max(self.epochs_seen) if self.epochs_seen else None,
+ 'tls_ok': self.tls_ok,
+ 'tls_failures': self.tls_failures,
+ 'client_ids': self.client_ids[:10],
+ 'qos_seen': self.qos_seen,
+ 'retain_seen': {str(k): v for k, v in self.retain_seen.items()},
+ 'will_topics': self.will_topics[:5],
+ 'connect_frames': self.connect_frames[:5],
+ 'status_msgs': self.status_msgs[:10],
+ 'connect_intervals': self.connect_intervals[-60:],
+ 'errors': self.errors[-40:],
+ 'msgs': self.msgs[-400:],
+ }
+ tmp = self.path + '.tmp'
+ with open(tmp, 'w') as fh:
+ json.dump(d, fh, indent=1)
+ os.replace(tmp, self.path)
+ return d
+
+
+def read_exact(sock, n):
+ buf = b''
+ while len(buf) < n:
+ c = sock.recv(n - len(buf))
+ if not c:
+ return None
+ buf += c
+ return buf
+
+
+def read_packet(sock):
+ """Return (ptype, flags, payload) or None on clean EOF."""
+ b0 = read_exact(sock, 1)
+ if b0 is None:
+ return None
+ ptype = b0[0] >> 4
+ flags = b0[0] & 0x0F
+ mult, length = 1, 0
+ while True:
+ b = read_exact(sock, 1)
+ if b is None:
+ return None
+ length += (b[0] & 0x7F) * mult
+ if not (b[0] & 0x80):
+ break
+ mult *= 128
+ if mult > 128 ** 4:
+ raise ValueError('malformed remaining length')
+ body = read_exact(sock, length) if length else b''
+ if body is None:
+ return None
+ return ptype, flags, body
+
+
+def enc_len(n):
+ out = b''
+ while True:
+ d = n % 128
+ n //= 128
+ if n:
+ d |= 0x80
+ out += bytes([d])
+ if not n:
+ return out
+
+
+def parse_connect(body):
+ """Return dict with protocol name/level, client id, will, user, keepalive."""
+ i = 0
+ nlen = struct.unpack('>H', body[i:i + 2])[0]
+ i += 2
+ name = body[i:i + nlen].decode('latin-1')
+ i += nlen
+ level = body[i]
+ i += 1
+ cflags = body[i]
+ i += 1
+ keepalive = struct.unpack('>H', body[i:i + 2])[0]
+ i += 2
+
+ def rd_str():
+ nonlocal i
+ ln = struct.unpack('>H', body[i:i + 2])[0]
+ i += 2
+ s = body[i:i + ln]
+ i += ln
+ return s
+
+ cid = rd_str().decode('utf-8', 'replace')
+ will_topic = will_msg = None
+ if cflags & 0x04:
+ will_topic = rd_str().decode('utf-8', 'replace')
+ will_msg = rd_str().decode('utf-8', 'replace')
+ user = passwd = None
+ if cflags & 0x80:
+ user = rd_str().decode('utf-8', 'replace')
+ if cflags & 0x40:
+ passwd = rd_str().decode('utf-8', 'replace')
+ return {
+ 'proto': name, 'level': level, 'clientId': cid, 'keepalive': keepalive,
+ 'clean': bool(cflags & 0x02), 'willTopic': will_topic,
+ 'willMsg': will_msg, 'willRetain': bool(cflags & 0x20),
+ 'willQos': (cflags >> 3) & 0x03, 'user': user, 'pass': passwd,
+ }
+
+
+def parse_publish(flags, body):
+ qos = (flags >> 1) & 0x03
+ tlen = struct.unpack('>H', body[0:2])[0]
+ topic = body[2:2 + tlen].decode('utf-8', 'replace')
+ i = 2 + tlen
+ pid = None
+ if qos > 0:
+ pid = struct.unpack('>H', body[i:i + 2])[0]
+ i += 2
+ return topic, qos, pid, body[i:]
+
+
+def serve(conn, args, stats):
+ mode = args.mode
+ with LOCK:
+ stats.conns += 1
+ try:
+ if mode == 'rst':
+ conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
+ struct.pack('ii', 1, 0))
+ conn.close()
+ return
+
+ conn.settimeout(args.read_timeout)
+ pkt = read_packet(conn)
+ if pkt is None:
+ return
+ ptype, flags, body = pkt
+ if ptype != CONNECT:
+ with LOCK:
+ stats.errors.append(f'first packet type {ptype}, expected CONNECT')
+ return
+ info = parse_connect(body)
+ stats.note_connect(info['clientId'])
+ with LOCK:
+ if info.get('willTopic') and info['willTopic'] not in stats.will_topics:
+ stats.will_topics.append(info['willTopic'])
+ if len(stats.connect_frames) < 5:
+ stats.connect_frames.append(dict(info))
+ with LOCK:
+ stats.errors.append(
+ 'CONNECT ' + json.dumps({k: info[k] for k in
+ ('proto', 'level', 'clientId', 'keepalive',
+ 'user', 'willTopic')}))
+
+ if mode == 'no_connack':
+ time.sleep(args.delay)
+ return
+ if mode == 'garbage':
+ conn.sendall(os.urandom(64))
+ return
+ if mode == 'half_connack':
+ conn.sendall(b'\x20\x02')
+ time.sleep(args.delay)
+ return
+ if mode == 'slow_connack':
+ time.sleep(args.delay)
+
+ rc = {'connack_refuse': 5, 'connack_badproto': 1,
+ 'connack_badid': 2, 'connack_unavail': 3}.get(mode, 0)
+ conn.sendall(bytes([CONNACK << 4, 2, 0, rc]))
+ with LOCK:
+ stats.connacks += 1
+ if rc != 0:
+ return
+ if mode == 'drop_after_connack':
+ conn.close()
+ return
+
+ # Steady state.
+ deadline_ka = info['keepalive'] * 2 if info['keepalive'] else 0
+ while True:
+ conn.settimeout(args.read_timeout)
+ pkt = read_packet(conn)
+ if pkt is None:
+ return
+ ptype, flags, body = pkt
+ if ptype == PUBLISH:
+ topic, qos, pid, payload = parse_publish(flags, body)
+ if mode == 'drop_on_publish':
+ with LOCK:
+ stats.errors.append(f'dropping on PUBLISH to {topic}')
+ conn.close()
+ return
+ if mode == 'rst_on_publish':
+ with LOCK:
+ stats.errors.append(f'RST on PUBLISH to {topic}')
+ conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
+ struct.pack('ii', 1, 0))
+ conn.close()
+ return
+ retain = bool(flags & 0x01)
+ dup = bool(flags & 0x08)
+ stats.note_publish(topic, payload, qos=qos, retain=retain, dup=dup)
+ if topic.endswith('/status'):
+ with LOCK:
+ stats.status_msgs.append(
+ {'topic': topic, 'retain': retain,
+ 'payload': payload.decode('utf-8', 'replace')[:200]})
+ if qos == 1 and pid is not None:
+ conn.sendall(bytes([PUBACK << 4, 2]) + struct.pack('>H', pid))
+ elif ptype == PINGREQ:
+ with LOCK:
+ stats.pings += 1
+ if mode != 'no_pingresp':
+ conn.sendall(bytes([PINGRESP << 4, 0]))
+ elif ptype == SUBSCRIBE:
+ pid = struct.unpack('>H', body[0:2])[0]
+ # One granted QoS byte per requested filter.
+ i, granted = 2, b''
+ while i < len(body):
+ ln = struct.unpack('>H', body[i:i + 2])[0]
+ i += 2 + ln
+ granted += bytes([body[i]])
+ i += 1
+ conn.sendall(bytes([SUBACK << 4]) + enc_len(2 + len(granted)) +
+ struct.pack('>H', pid) + granted)
+ elif ptype == DISCONNECT:
+ return
+ except Exception as e:
+ with LOCK:
+ stats.errors.append(f'{type(e).__name__}: {e}')
+ finally:
+ try:
+ conn.close()
+ except Exception:
+ pass
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('--port', type=int, required=True)
+ ap.add_argument('--bind', default='0.0.0.0')
+ ap.add_argument('--tls', action='store_true')
+ ap.add_argument('--cert', default='certs/cert.pem')
+ ap.add_argument('--key', default='certs/key.pem')
+ ap.add_argument('--mode', default='ok')
+ ap.add_argument('--delay', type=float, default=30.0)
+ ap.add_argument('--read-timeout', type=float, default=120.0)
+ ap.add_argument('--stats', default='mqtt_stats.json')
+ ap.add_argument('--records', default='')
+ ap.add_argument('--tls-fault', default='',
+ choices=['', 'blackhole', 'garbage', 'rst', 'slow'])
+ args = ap.parse_args()
+
+ stats = Stats(args.stats, args.records)
+
+ srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ srv.bind((args.bind, args.port))
+ srv.listen(16)
+
+ ctx = None
+ if args.tls:
+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ ctx.load_cert_chain(args.cert, args.key)
+ ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+ ctx.maximum_version = ssl.TLSVersion.TLSv1_2
+
+ def dumper():
+ while True:
+ time.sleep(2)
+ try:
+ stats.dump()
+ except Exception:
+ pass
+
+ threading.Thread(target=dumper, daemon=True).start()
+ print(f'[{"mqtts" if args.tls else "mqtt"}] listening on {args.bind}:{args.port} '
+ f'mode={args.mode} tls_fault={args.tls_fault or "-"}', flush=True)
+
+ while True:
+ try:
+ conn, _ = srv.accept()
+ except OSError:
+ break
+ conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+ def wrapped(conn=conn):
+ if args.tls:
+ f = args.tls_fault
+ if f == 'rst':
+ with LOCK:
+ stats.conns += 1
+ conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
+ struct.pack('ii', 1, 0))
+ conn.close()
+ return
+ if f == 'blackhole':
+ with LOCK:
+ stats.conns += 1
+ time.sleep(args.delay)
+ conn.close()
+ return
+ if f == 'garbage':
+ with LOCK:
+ stats.conns += 1
+ try:
+ conn.recv(4096)
+ conn.sendall(os.urandom(2048))
+ except Exception:
+ pass
+ conn.close()
+ return
+ if f == 'slow':
+ time.sleep(args.delay)
+ try:
+ conn.settimeout(args.read_timeout)
+ tconn = ctx.wrap_socket(conn, server_side=True)
+ with LOCK:
+ stats.tls_ok += 1
+ except Exception as e:
+ with LOCK:
+ stats.tls_failures += 1
+ stats.errors.append(f'TLS: {type(e).__name__}: {e}')
+ try:
+ conn.close()
+ except Exception:
+ pass
+ return
+ serve(tconn, args, stats)
+ else:
+ serve(conn, args, stats)
+
+ threading.Thread(target=wrapped, daemon=True).start()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/soak24.py b/tools/telemetry_bench/soak24.py
new file mode 100644
index 0000000..68f0546
--- /dev/null
+++ b/tools/telemetry_bench/soak24.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Soak de 24 h com a configuração real do usuário.
+
+Serve a dois propósitos: é o teste de aceitação das 11 correções desta campanha,
+e cobre a classe de problema que as rajadas de 90–150 s não tocam — deriva de
+cursor, fragmentação de heap ao longo de horas, interação com o GC do histórico,
+NTP re-sincronizando, e a detecção de RSSI implausível (D15) que ainda não foi
+vista disparar.
+
+Monitoramento é só pela serial, de propósito: `/api/status` exigiria deixar um
+usuário web admin descartável vivo por 24 h, e a CLI já entrega tudo que
+importa. Nada de servidor de teste — o alvo é o servidor real do usuário.
+
+Amostra a cada 5 min (288 amostras). Cada linha anômala vai para o log com
+prefixo ANOMALIA, para `grep` valer como triagem.
+
+Parar antes da hora: pkill -f soak24.py
+"""
+import json
+import os
+import re
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import campaign as C # noqa: E402
+from bench import Target # noqa: E402
+
+DURACAO_S = 24 * 3600
+PERIODO_S = 300
+OUT = os.path.join(C.OUT, 'soak24.ndjson')
+
+
+def amostra(t):
+ d = {'wall': time.time()}
+ m = t.metrics(wait=4.0)
+ for k in ('uptime', 'heap', 'heap_min', 'largest', 'largest_min',
+ 'tel_sent', 'tel_failed', 'tel_retries', 'tel_bytes', 'tel_lat',
+ 'reads_ok', 'reads_err', 'flash_ops', 'core1_exposed',
+ 'wifi_conns', 'mqtt_conns'):
+ d[k] = m.get(k)
+ net = t.send('show net status', wait=4.0)
+ r = re.search(r'RSSI:\s*(-?\d+)', net)
+ p = re.search(r'PBUF pool:\s*(\d+) em uso / pico (\d+) / (\d+) total, (\d+) falhas', net)
+ ip = re.search(r'IP:\s*([\d.]+)', net)
+ d['rssi'] = int(r.group(1)) if r else None
+ d['ip'] = ip.group(1) if ip else None
+ if p:
+ d['pbuf_uso'], d['pbuf_pico'], d['pbuf_total'], d['pbuf_falhas'] = \
+ (int(x) for x in p.groups())
+ return d
+
+
+def main():
+ t = Target(os.path.join(C.OUT, 'serial_soak24.log'))
+ time.sleep(1)
+ t0 = time.time()
+ base = amostra(t)
+ boots0, fatal0 = t.port_drops, len(t.fatal_lines)
+ C.log(f'SOAK 24h iniciado. base={json.dumps(base)}')
+ fh = open(OUT, 'a', buffering=1)
+ fh.write(json.dumps({'evento': 'inicio', **base}) + '\n')
+
+ prev = base
+ n = 0
+ while time.time() - t0 < DURACAO_S:
+ time.sleep(PERIODO_S)
+ n += 1
+ d = amostra(t)
+ d['t_h'] = round((time.time() - t0) / 3600, 2)
+ d['reboots_usb'] = t.port_drops - boots0
+ d['ftl'] = len(t.fatal_lines) - fatal0
+ fh.write(json.dumps(d) + '\n')
+
+ alerta = []
+ # 1. Reboot ou pânico — critério de parada imediata.
+ if d['reboots_usb'] or d['ftl']:
+ alerta.append(f"REBOOT/FTL usb={d['reboots_usb']} ftl={d['ftl']}")
+ if (prev.get('uptime') and d.get('uptime')
+ and d['uptime'] < prev['uptime']):
+ alerta.append(f"uptime regrediu {prev['uptime']}->{d['uptime']}")
+ # 2. PBUF falhando fora de rajada: com o pool em 24 e sem carga de
+ # teste, qualquer falha aqui é sinal, não ruído.
+ if d.get('pbuf_falhas'):
+ alerta.append(f"PBUF falhas={d['pbuf_falhas']}")
+ # 3. RSSI implausível — é o gatilho do D15. Se aparecer, o log serial
+ # deve trazer "Implausible RSSI twice" logo em seguida.
+ if d.get('rssi') is not None and (d['rssi'] >= 0 or d['rssi'] < -120):
+ alerta.append(f"RSSI implausivel={d['rssi']}")
+ # 4. Heap: deriva > 5% contra a linha de base é o critério de sucesso.
+ if base.get('heap') and d.get('heap'):
+ drift = abs(d['heap'] - base['heap']) / base['heap']
+ if drift > 0.05:
+ alerta.append(f"heap {base['heap']}->{d['heap']} ({drift:.1%})")
+ if d.get('ip') != base.get('ip'):
+ alerta.append(f"IP mudou {base.get('ip')}->{d.get('ip')}")
+
+ if alerta:
+ C.log(f"ANOMALIA t={d['t_h']}h :: " + ' | '.join(alerta))
+ elif n % 12 == 0: # resumo de hora em hora
+ C.log(f"ok t={d['t_h']}h up={d['uptime']}s heap={d['heap']} "
+ f"lb={d['largest']} rssi={d['rssi']} "
+ f"pbuf={d.get('pbuf_uso')}/{d.get('pbuf_total')}({d.get('pbuf_falhas')}) "
+ f"sent={d['tel_sent']} fail={d['tel_failed']}")
+ prev = d
+
+ fim = amostra(t)
+ fim['evento'] = 'fim'
+ fim['reboots_usb'] = t.port_drops - boots0
+ fim['ftl'] = len(t.fatal_lines) - fatal0
+ fh.write(json.dumps(fim) + '\n')
+ C.log(f'SOAK 24h concluido. {json.dumps(fim)}')
+ fh.close()
+ t.close()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/telemetry_bench/summarize.py b/tools/telemetry_bench/summarize.py
new file mode 100644
index 0000000..a6c990b
--- /dev/null
+++ b/tools/telemetry_bench/summarize.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""Turn the campaign's JSON output into the tables that go in the report."""
+import datetime as dt
+import json
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+OUT = os.path.join(HERE, 'results')
+
+
+def load(name):
+ p = os.path.join(OUT, name)
+ if not os.path.exists(p):
+ return None
+ with open(p) as fh:
+ return json.load(fh)
+
+
+def ts(e):
+ if not e:
+ return '—'
+ return dt.datetime.fromtimestamp(e).strftime('%Y-%m-%d %H:%M')
+
+
+def md_table(rows, cols, headers=None):
+ headers = headers or cols
+ out = ['| ' + ' | '.join(headers) + ' |',
+ '|' + '|'.join(['---'] * len(cols)) + '|']
+ for r in rows:
+ out.append('| ' + ' | '.join(str(r.get(c, '—')) for c in cols) + ' |')
+ return '\n'.join(out)
+
+
+def perf_table():
+ d = load('phase_perf_http.json')
+ if not d:
+ return '_(sem dados)_'
+ rows = []
+ for r in d['rows']:
+ rows.append({
+ 'transporte': 'HTTPS' if r.get('tls') else 'HTTP',
+ 'lote': r.get('batch'),
+ 'envios': r.get('srv_requests'),
+ 'registros': r.get('srv_records'),
+ 'reg/s': r.get('records_per_s'),
+ 'B/s': r.get('bytes_per_s'),
+ 'lat med (ms)': r.get('dev_lat_med'),
+ 'lat max (ms)': r.get('dev_lat_max'),
+ 'falhas': r.get('dev_failed'),
+ 'heap min': r.get('heap_min'),
+ 'maior bloco min': r.get('largest_min'),
+ 'reboots': (r.get('usb_drops') or 0) + (r.get('uptime_resets') or 0),
+ })
+ return md_table(rows, list(rows[0].keys()))
+
+
+def mqtt_perf_table():
+ out = []
+ for f, lbl in (('phase_mqtt_plain.json', 'MQTT'), ('phase_mqtt_tls.json', 'MQTTS')):
+ d = load(f)
+ if not d:
+ continue
+ rows = []
+ for r in d.get('perf', []):
+ rows.append({
+ 'transporte': lbl,
+ 'lote': r.get('batch'),
+ 'publishes': r.get('srv_publishes'),
+ 'registros': r.get('srv_records'),
+ 'reg/s': r.get('records_per_s'),
+ 'lat med (ms)': r.get('dev_lat_med'),
+ 'falhas': r.get('dev_failed'),
+ 'conexoes': r.get('srv_connects'),
+ 'qos visto': json.dumps(r.get('srv_qos_seen') or {}),
+ 'heap min': r.get('heap_min'),
+ 'reboots': (r.get('usb_drops') or 0) + (r.get('uptime_resets') or 0),
+ })
+ if rows:
+ out.append(md_table(rows, list(rows[0].keys())))
+ return '\n\n'.join(out) if out else '_(sem dados)_'
+
+
+def survive_table():
+ rows = []
+ for f in ('phase_survive_all.json', 'phase_survive_http.json',
+ 'phase_survive_tls.json'):
+ d = load(f)
+ if not d:
+ continue
+ for r in d['rows']:
+ rows.append({
+ 'falha': r.get('fault'),
+ 'transporte': 'HTTPS' if r.get('tls') else 'HTTP',
+ 'janela (s)': r.get('seconds'),
+ 'reboots': (r.get('usb_drops') or 0) + (r.get('uptime_resets') or 0),
+ 'FTL': r.get('fatal_lines'),
+ 'falhas tel': r.get('dev_failed_delta'),
+ 'envios tel': r.get('dev_sent_delta'),
+ 'heap min': r.get('heap_min'),
+ 'web mudo (polls)': r.get('web_worst_streak_polls'),
+ 'reg perdidos': r.get('records_skipped'),
+ 'recuperou': 'sim' if (r.get('post') or {}).get('records') else 'NAO',
+ 'veredito': r.get('verdict'),
+ })
+ for f, lbl in (('phase_mqtt_plain.json', 'MQTT'), ('phase_mqtt_tls.json', 'MQTTS')):
+ d = load(f)
+ if not d:
+ continue
+ for r in d.get('faults', []):
+ rows.append({
+ 'falha': r.get('fault'),
+ 'transporte': lbl,
+ 'janela (s)': r.get('seconds'),
+ 'reboots': (r.get('usb_drops') or 0) + (r.get('uptime_resets') or 0),
+ 'FTL': r.get('fatal_lines'),
+ 'falhas tel': r.get('dev_failed'),
+ 'envios tel': r.get('dev_sent'),
+ 'heap min': r.get('heap_min'),
+ 'web mudo (polls)': r.get('unreachable_polls'),
+ 'reg perdidos': r.get('records_skipped'),
+ 'recuperou': 'sim' if (r.get('post') or {}).get('records') else 'NAO',
+ 'veredito': r.get('verdict'),
+ })
+ if not rows:
+ return '_(sem dados)_'
+ return md_table(rows, list(rows[0].keys()))
+
+
+def drain_block():
+ d = load('phase_drain.json')
+ if not d:
+ return '_(sem dados)_'
+ t, g, c = d['telemetry'], d['ground_truth'], d['coverage']
+ return f"""**Via telemetria** (lote {t['batch']}, intervalo {t['interval']} ms, HTTP puro)
+
+| métrica | valor |
+|---|---|
+| duração | {t['wall_s']} s |
+| envios HTTP | {t['requests']} |
+| registros aceitos | {t['records']} |
+| epochs únicos | {t['unique_epochs']} |
+| duplicados | {t['duplicates']} |
+| bytes recebidos | {t['bytes_in']} |
+| primeiro registro | {ts(t['epoch_min'])} |
+| último registro | {ts(t['epoch_max'])} |
+| reboots | {t['usb_drops']} |
+
+**Verdade de solo** (todos os `.h5` baixados e decodificados pelo codec de referência)
+
+| métrica | valor |
+|---|---|
+| arquivos listados | {g['files_listed']} |
+| arquivos baixados | {g['files_downloaded']} |
+| falhas de download | {len(g['download_failures'])} |
+| bytes | {g['total_bytes']} |
+| registros no disco | {g['total_records']} |
+| epochs únicos | {g['unique_epochs']} |
+| primeiro | {ts(g['epoch_min'])} |
+| último | {ts(g['epoch_max'])} |
+| erros de decodificação | {len(g['decode_errors'])} |
+
+**Cobertura**: {c['via_telemetry']} de {c['on_disk']} epochs = **{c['pct']}%**.
+Faltaram {c['missing_count']} registros, de {ts(c['missing_first'])} a {ts(c['missing_last'])}.
+"""
+
+
+def fidelity_block():
+ d = load('phase_mqtt_plain.json')
+ if not d or 'fidelity' not in d:
+ return '_(sem dados)_'
+ f = d['fidelity']
+ rows = []
+ for k, v in f['checks'].items():
+ rows.append({'verificação': k, 'resultado': 'OK' if v else 'FALHOU'})
+ return (md_table(rows, ['verificação', 'resultado']) +
+ '\n\nEnviado: `' + json.dumps(f['wanted']) + '`\n\n' +
+ 'Recebido no broker: `' + json.dumps(f['got']) + '`')
+
+
+if __name__ == '__main__':
+ print('## Desempenho HTTP/HTTPS\n')
+ print(perf_table())
+ print('\n## Desempenho MQTT/MQTTS\n')
+ print(mqtt_perf_table())
+ print('\n## Fidelidade da config MQTT\n')
+ print(fidelity_block())
+ print('\n## Sobrevivência\n')
+ print(survive_table())
+ print('\n## Descarga do histórico\n')
+ print(drain_block())