From 706115ad053b9785a3a9e5ee586d796905520c8a Mon Sep 17 00:00:00 2001 From: Swanny Date: Thu, 16 Apr 2026 17:07:43 -0400 Subject: [PATCH 1/2] fix: handle nil block result in EmptyBlocksSanitizer A spec-compliant JSON-RPC server returns `result: null` for blocks it cannot find (e.g. pruned or reorged out). The previous implementation of `classify_blocks_from_result/1` assumed every batch response carried a block map, so a `nil` result crashed the GenServer with `BadMapError` at `Map.get(nil, "transactions", nil)`. Because the fetcher uses `restart: :permanent`, repeated crashes eventually exhausted the supervisor's restart intensity and brought down the `indexer` application, taking the whole BEAM VM with it. Add a match clause for `%{id: _, result: nil}` that logs a warning and skips the block. Include a regression test that mocks a nil batch response and asserts the sanitizer does not mutate the DB record. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../indexer/fetcher/empty_blocks_sanitizer.ex | 23 +++++++++---- .../fetcher/empty_blocks_sanitizer_test.exs | 34 +++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex b/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex index 32d82fc48766..4563bb87ff2e 100644 --- a/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex +++ b/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex @@ -114,13 +114,22 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizer do end defp classify_blocks_from_result(result) do - result - |> Enum.reduce({[], []}, fn %{id: _id, result: block}, {non_empty_blocks, empty_blocks} -> - if Enum.empty?(block["transactions"]) do - {non_empty_blocks, [block_fields(block) | empty_blocks]} - else - {[block_fields(block) | non_empty_blocks], empty_blocks} - end + Enum.reduce(result, {[], []}, fn + %{id: _id, result: nil}, acc -> + # A spec-compliant JSON-RPC server returns `result: null` for blocks it + # cannot find (e.g. pruned or reorged). Skip without crashing. + Logger.warning("Received nil block from RPC while sanitizing empty blocks; skipping", + fetcher: :empty_blocks_to_refetch + ) + + acc + + %{id: _id, result: block}, {non_empty_blocks, empty_blocks} -> + if Enum.empty?(block["transactions"]) do + {non_empty_blocks, [block_fields(block) | empty_blocks]} + else + {[block_fields(block) | non_empty_blocks], empty_blocks} + end end) end diff --git a/apps/indexer/test/indexer/fetcher/empty_blocks_sanitizer_test.exs b/apps/indexer/test/indexer/fetcher/empty_blocks_sanitizer_test.exs index 6b575c2896d2..2bba503b521e 100644 --- a/apps/indexer/test/indexer/fetcher/empty_blocks_sanitizer_test.exs +++ b/apps/indexer/test/indexer/fetcher/empty_blocks_sanitizer_test.exs @@ -140,6 +140,40 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizerTest do assert processed_block.refetch_needed == true, "invalid `refetch_needed` value set for processed block" end + test "skips blocks for which JSON-RPC returns nil result", %{json_rpc_named_arguments: json_rpc_named_arguments} do + # Setup + block_to_process = insert(:block, is_empty: nil) + populate_database_with_dummy_blocks() + assert Repo.get!(Block, block_to_process.hash).is_empty == nil, "precondition to check setup correctness" + + encoded_expected_block_number = "0x" <> Integer.to_string(block_to_process.number, 16) + + if json_rpc_named_arguments[:transport] == EthereumJSONRPC.Mox do + EthereumJSONRPC.Mox + |> stub( + :json_rpc, + fn [ + %{ + id: id, + method: "eth_getBlockByNumber", + params: [^encoded_expected_block_number, false] + } + ], + _options -> + {:ok, [%{id: id, result: nil}]} + end + ) + end + + EmptyBlocksSanitizer.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) + + # Give the sanitizer a moment to process the nil response. + Process.sleep(500) + + # The block should remain untouched (sanitizer skipped it without crashing). + assert Repo.get!(Block, block_to_process.hash).is_empty == nil + end + test "only old enough blocks are sanitized", %{json_rpc_named_arguments: json_rpc_named_arguments} do # Setup block_to_process = insert(:block, is_empty: nil) From 0260addcbe382ee28651864af1ad59ffe8234f71 Mon Sep 17 00:00:00 2001 From: Swanny Date: Thu, 16 Apr 2026 17:17:33 -0400 Subject: [PATCH 2/2] fix(empty_blocks_sanitizer): flag nil-result blocks as refetch_needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The initial nil-handling commit silently skipped blocks for which the JSON-RPC returned `result: null`, which left those blocks with `is_empty: nil, refetch_needed: false` — the exact predicate that `consensus_blocks_with_nil_is_empty_query/1` selects for. The sanitizer would then re-query the same blocks every cycle (every 10s), producing perpetual RPC traffic and log spam. Reconcile requested vs returned block numbers in the caller and call `Block.set_refetch_needed/1` on the difference. The refetch_needed predicate then excludes them from subsequent sanitizer cycles and hands them off to the regular refetch path — consistent with how the fetcher already handles blocks whose RPC view disagrees with the DB. Also strengthen the regression test so it actually discriminates the fixed code from the buggy code: - Replace `Process.sleep/1` with `wait_for_results/1`, which polls for `refetch_needed == true` and fails the test if the sanitizer never updates the row (as happens on the pre-fix `BadMapError` crash path). - Assert both `is_empty == nil` (unchanged) and `refetch_needed == true` (positive signal only the fix can produce). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../indexer/fetcher/empty_blocks_sanitizer.ex | 36 +++++++++++++++---- .../fetcher/empty_blocks_sanitizer_test.exs | 23 +++++++++--- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex b/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex index 4563bb87ff2e..3f49ad01330d 100644 --- a/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex +++ b/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex @@ -99,6 +99,7 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizer do {non_empty_blocks, empty_blocks} = classify_blocks_from_result(result) process_non_empty_blocks(non_empty_blocks) process_empty_blocks(empty_blocks) + process_missing_blocks(unprocessed_empty_blocks_list, non_empty_blocks, empty_blocks) Logger.info("Batch of empty blocks is sanitized", fetcher: :empty_blocks_to_refetch @@ -114,14 +115,11 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizer do end defp classify_blocks_from_result(result) do + # A spec-compliant JSON-RPC server returns `result: null` for blocks it + # cannot find (e.g. pruned or reorged). Skip without crashing — the caller + # reconciles which requested blocks are missing and flags them for refetch. Enum.reduce(result, {[], []}, fn %{id: _id, result: nil}, acc -> - # A spec-compliant JSON-RPC server returns `result: null` for blocks it - # cannot find (e.g. pruned or reorged). Skip without crashing. - Logger.warning("Received nil block from RPC while sanitizing empty blocks; skipping", - fetcher: :empty_blocks_to_refetch - ) - acc %{id: _id, result: block}, {non_empty_blocks, empty_blocks} -> @@ -133,6 +131,32 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizer do end) end + # Blocks the RPC returned nil for stay in `is_empty: nil, refetch_needed: false`, + # so without intervention the sanitizer's query would re-select them every cycle. + # Flag them `refetch_needed: true` to remove them from the query set and let the + # regular refetch path handle them. + defp process_missing_blocks(requested, non_empty_blocks, empty_blocks) do + returned = MapSet.new(non_empty_blocks ++ empty_blocks, & &1.number) + + missing = + requested + |> Enum.map(& &1.number) + |> Enum.reject(&MapSet.member?(returned, &1)) + + case missing do + [] -> + :ok + + numbers -> + Logger.warning( + "JSON-RPC returned nil for block numbers #{inspect(numbers)}; marking as refetch_needed", + fetcher: :empty_blocks_to_refetch + ) + + Block.set_refetch_needed(numbers) + end + end + defp block_fields(block) do %{ number: quantity_to_integer(block["number"]), diff --git a/apps/indexer/test/indexer/fetcher/empty_blocks_sanitizer_test.exs b/apps/indexer/test/indexer/fetcher/empty_blocks_sanitizer_test.exs index 2bba503b521e..63928e1972e2 100644 --- a/apps/indexer/test/indexer/fetcher/empty_blocks_sanitizer_test.exs +++ b/apps/indexer/test/indexer/fetcher/empty_blocks_sanitizer_test.exs @@ -140,11 +140,13 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizerTest do assert processed_block.refetch_needed == true, "invalid `refetch_needed` value set for processed block" end - test "skips blocks for which JSON-RPC returns nil result", %{json_rpc_named_arguments: json_rpc_named_arguments} do + test "marks block as refetch_needed when JSON-RPC returns nil result", + %{json_rpc_named_arguments: json_rpc_named_arguments} do # Setup block_to_process = insert(:block, is_empty: nil) populate_database_with_dummy_blocks() assert Repo.get!(Block, block_to_process.hash).is_empty == nil, "precondition to check setup correctness" + assert Repo.get!(Block, block_to_process.hash).refetch_needed == false, "precondition to check setup correctness" encoded_expected_block_number = "0x" <> Integer.to_string(block_to_process.number, 16) @@ -167,11 +169,22 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizerTest do EmptyBlocksSanitizer.Supervisor.Case.start_supervised!(json_rpc_named_arguments: json_rpc_named_arguments) - # Give the sanitizer a moment to process the nil response. - Process.sleep(500) + # Wait for the sanitizer to flag the nil-result block as refetch_needed. + # On the un-fixed code this never happens (the GenServer crashes on + # BadMapError), so `wait_for_results` would time out and fail the test. + processed_block = + wait_for_results(fn -> + Repo.one!( + from(block in Block, + where: block.hash == ^block_to_process.hash and block.refetch_needed == true + ) + ) + end) + + assert processed_block.is_empty == nil, "is_empty should remain untouched for unresolved blocks" - # The block should remain untouched (sanitizer skipped it without crashing). - assert Repo.get!(Block, block_to_process.hash).is_empty == nil + assert processed_block.refetch_needed == true, + "refetch_needed should be set so the block exits the sanitizer's query set" end test "only old enough blocks are sanitized", %{json_rpc_named_arguments: json_rpc_named_arguments} do