refactor: move spamoor benchmark into testify suite (#3107)

* refactor: move spamoor benchmark into testify suite in test/e2e/benchmark

- Create test/e2e/benchmark/ subpackage with SpamoorSuite (testify/suite)
- Move spamoor smoke test into suite as TestSpamoorSmoke
- Split helpers into focused files: traces.go, output.go, metrics.go
- Introduce resultWriter for defer-based benchmark JSON output
- Export shared symbols from evm_test_common.go for cross-package use
- Restructure CI to fan-out benchmark jobs and fan-in publishing
- Run benchmarks on PRs only when benchmark-related files change

* fix: correct BENCH_JSON_OUTPUT path for spamoor benchmark

go test sets the working directory to the package under test, so the
env var should be relative to test/e2e/benchmark/, not test/e2e/.

* fix: place package pattern before test binary flags in benchmark CI

go test treats all arguments after an unknown flag (--evm-binary) as
test binary args, so ./benchmark/ was never recognized as a package
pattern.

* fix: adjust evm-binary path for benchmark subpackage working directory

go test sets the cwd to the package directory (test/e2e/benchmark/),
so the binary path needs an extra parent traversal.

* fix: exclude benchmark subpackage from make test-e2e

The benchmark package doesn't define the --binary flag that test-e2e
passes. It has its own CI workflow so it doesn't need to run here.

* fix: address PR review feedback for benchmark suite

- make reth tag configurable via EV_RETH_TAG env var (default pr-140)
- fix OTLP config: remove duplicate env vars, use http/protobuf protocol
- use require.Eventually for host readiness polling
- rename requireHTTP to requireHostUp
- use non-fatal logging in resultWriter.flush deferred context
- fix stale doc comment (setupCommonEVMEnv -> SetupCommonEVMEnv)
- rename loop variable to avoid shadowing testing.TB convention
- add block/internal/executing/** to CI path trigger
- remove unused require import from output.go

* chore: specify http

* chore: filter out benchmark tests from test-e2e

Author: chatton

.github/workflows/benchmark.yml

@@ -8,6 +8,13 @@ permissions: {}
   pull_request:
     branches:
       - main
+    paths:
+      - 'test/e2e/benchmark/**'
+      - 'test/e2e/evm_contract_bench_test.go'
+      - 'test/e2e/evm_test_common.go'
+      - 'test/e2e/sut_helper.go'
+      - 'block/internal/executing/**'
+      - '.github/workflows/benchmark.yml'
   workflow_dispatch:
 
 jobs:
@@ -62,12 +69,13 @@ jobs:
       - name: Run Spamoor smoke test
         run: |
           cd test/e2e && BENCH_JSON_OUTPUT=spamoor_bench.json go test -tags evm \
-            -run='^TestSpamoorSmoke$' -v -timeout=15m --evm-binary=../../build/evm
+            -run='^TestSpamoorSuite$/^TestSpamoorSmoke$' -v -timeout=15m \
+            ./benchmark/ --evm-binary=../../../build/evm
       - name: Upload benchmark results
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: spamoor-benchmark-results
-          path: test/e2e/spamoor_bench.json
+          path: test/e2e/benchmark/spamoor_bench.json
 
   # single job to push all results to gh-pages sequentially, avoiding race conditions
   publish-benchmarks:
@@ -88,7 +96,7 @@ jobs:
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: spamoor-benchmark-results
-          path: test/e2e/
+          path: test/e2e/benchmark/
 
       # only update the benchmark baseline on push/dispatch, not on PRs
       - name: Store EVM Contract Roundtrip result
@@ -135,7 +143,7 @@ jobs:
         with:
           name: Spamoor Trace Benchmarks
           tool: 'customSmallerIsBetter'
-          output-file-path: test/e2e/spamoor_bench.json
+          output-file-path: test/e2e/benchmark/spamoor_bench.json
           auto-push: ${{ github.event_name != 'pull_request' }}
           save-data-file: ${{ github.event_name != 'pull_request' }}
           github-token: ${{ secrets.GITHUB_TOKEN }}

.just/test.just

@@ -25,7 +25,7 @@ test-integration:
 [group('test')]
 test-e2e: build build-da build-evm docker-build-if-local
     @echo "--> Running e2e tests"
-    @cd test/e2e && go test -mod=readonly -failfast -timeout=15m -tags='e2e evm' ./... --binary=../../build/testapp --evm-binary=../../build/evm
+    @cd test/e2e && go test -mod=readonly -failfast -timeout=15m -tags='e2e evm' $(go list -tags='e2e evm' ./... | grep -v /benchmark) --binary=../../build/testapp --evm-binary=../../build/evm
 
 # Run integration tests with coverage
 [group('test')]

test/e2e/benchmark/metrics.go

@@ -0,0 +1,41 @@
+//go:build evm
+
+package benchmark
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	dto "github.com/prometheus/client_model/go"
+)
+
+// requireHostUp polls a URL until it returns a 2xx status code or the timeout expires.
+func requireHostUp(t testing.TB, url string, timeout time.Duration) {
+	t.Helper()
+	client := &http.Client{Timeout: 200 * time.Millisecond}
+	require.Eventually(t, func() bool {
+		resp, err := client.Get(url)
+		if err != nil {
+			return false
+		}
+		_ = resp.Body.Close()
+		return resp.StatusCode >= 200 && resp.StatusCode < 300
+	}, timeout, 100*time.Millisecond, "daemon not ready at %s", url)
+}
+
+// sumCounter sums all counter values in a prometheus MetricFamily.
+func sumCounter(f *dto.MetricFamily) float64 {
+	if f == nil || f.GetType() != dto.MetricType_COUNTER {
+		return 0
+	}
+	var sum float64
+	for _, m := range f.GetMetric() {
+		if m.GetCounter() != nil && m.GetCounter().Value != nil {
+			sum += m.GetCounter().GetValue()
+		}
+	}
+	return sum
+}

test/e2e/benchmark/output.go

@@ -0,0 +1,82 @@
+//go:build evm
+
+package benchmark
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+	"testing"
+
+	e2e "github.com/evstack/ev-node/test/e2e"
+)
+
+// entry matches the customSmallerIsBetter format for github-action-benchmark.
+type entry struct {
+	Name  string  `json:"name"`
+	Unit  string  `json:"unit"`
+	Value float64 `json:"value"`
+}
+
+// resultWriter accumulates benchmark entries and writes them to a JSON file
+// when flush is called. Create one early in a test and defer flush so results
+// are written regardless of where the test exits.
+type resultWriter struct {
+	t       testing.TB
+	label   string
+	entries []entry
+}
+
+func newResultWriter(t testing.TB, label string) *resultWriter {
+	return &resultWriter{t: t, label: label}
+}
+
+// addSpans aggregates trace spans into per-operation avg duration entries.
+func (w *resultWriter) addSpans(spans []e2e.TraceSpan) {
+	m := e2e.AggregateSpanStats(spans)
+	if len(m) == 0 {
+		return
+	}
+
+	names := make([]string, 0, len(m))
+	for name := range m {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		s := m[name]
+		avg := float64(s.Total.Microseconds()) / float64(s.Count)
+		w.entries = append(w.entries, entry{
+			Name:  fmt.Sprintf("%s - %s (avg)", w.label, name),
+			Unit:  "us",
+			Value: avg,
+		})
+	}
+}
+
+// addEntry appends a custom entry to the results.
+func (w *resultWriter) addEntry(e entry) {
+	w.entries = append(w.entries, e)
+}
+
+// flush writes accumulated entries to the path in BENCH_JSON_OUTPUT.
+// It is a no-op when the env var is unset or no entries were added.
+func (w *resultWriter) flush() {
+	outputPath := os.Getenv("BENCH_JSON_OUTPUT")
+	if outputPath == "" || len(w.entries) == 0 {
+		return
+	}
+
+	data, err := json.MarshalIndent(w.entries, "", "  ")
+	if err != nil {
+		w.t.Logf("WARNING: failed to marshal benchmark JSON: %v", err)
+		return
+	}
+	if err := os.WriteFile(outputPath, data, 0644); err != nil {
+		w.t.Logf("WARNING: failed to write benchmark JSON to %s: %v", outputPath, err)
+		return
+	}
+	w.t.Logf("wrote %d benchmark entries to %s", len(w.entries), outputPath)
+}

test/e2e/benchmark/spamoor_smoke_test.go

@@ -0,0 +1,120 @@
+//go:build evm
+
+package benchmark
+
+import (
+	"os"
+	"time"
+
+	"github.com/celestiaorg/tastora/framework/docker/evstack/spamoor"
+	e2e "github.com/evstack/ev-node/test/e2e"
+)
+
+// TestSpamoorSmoke spins up reth + sequencer and a Spamoor node, starts a few
+// basic spammers, waits briefly, then validates trace spans and prints a concise
+// metrics summary.
+func (s *SpamoorSuite) TestSpamoorSmoke() {
+	t := s.T()
+	w := newResultWriter(t, "SpamoorSmoke")
+	defer w.flush()
+
+	// TODO: temporary hardcoded tag, will be replaced with a proper release tag
+	rethTag := os.Getenv("EV_RETH_TAG")
+	if rethTag == "" {
+		rethTag = "pr-140"
+	}
+	e := s.setupEnv(config{
+		rethTag:     rethTag,
+		serviceName: "ev-node-smoke",
+	})
+	api := e.spamoorAPI
+
+	eoatx := map[string]any{
+		"throughput":      100,
+		"total_count":     3000,
+		"max_pending":     4000,
+		"max_wallets":     300,
+		"amount":          100,
+		"random_amount":   true,
+		"random_target":   true,
+		"base_fee":        20,
+		"tip_fee":         2,
+		"refill_amount":   "1000000000000000000",
+		"refill_balance":  "500000000000000000",
+		"refill_interval": 600,
+	}
+
+	gasburner := map[string]any{
+		"throughput":        25,
+		"total_count":       2000,
+		"max_pending":       8000,
+		"max_wallets":       500,
+		"gas_units_to_burn": 3000000,
+		"base_fee":          20,
+		"tip_fee":           5,
+		"rebroadcast":       5,
+		"refill_amount":     "5000000000000000000",
+		"refill_balance":    "2000000000000000000",
+		"refill_interval":   300,
+	}
+
+	var ids []int
+	id, err := api.CreateSpammer("smoke-eoatx", spamoor.ScenarioEOATX, eoatx, true)
+	s.Require().NoError(err, "failed to create eoatx spammer")
+	ids = append(ids, id)
+	id, err = api.CreateSpammer("smoke-gasburner", spamoor.ScenarioGasBurnerTX, gasburner, true)
+	s.Require().NoError(err, "failed to create gasburner spammer")
+	ids = append(ids, id)
+
+	for _, id := range ids {
+		idToDelete := id
+		t.Cleanup(func() { _ = api.DeleteSpammer(idToDelete) })
+	}
+
+	// allow spamoor enough time to generate transaction throughput
+	// so that the expected tracing spans appear in Jaeger.
+	time.Sleep(60 * time.Second)
+
+	// fetch parsed metrics and print a concise summary.
+	metrics, err := api.GetMetrics()
+	s.Require().NoError(err, "failed to get metrics")
+	sent := sumCounter(metrics["spamoor_transactions_sent_total"])
+	fail := sumCounter(metrics["spamoor_transactions_failed_total"])
+
+	// collect traces
+	evNodeSpans := s.collectServiceTraces(e, "ev-node-smoke")
+	evRethSpans := s.collectServiceTraces(e, "ev-reth")
+	e2e.PrintTraceReport(t, "ev-node-smoke", evNodeSpans)
+	e2e.PrintTraceReport(t, "ev-reth", evRethSpans)
+
+	w.addSpans(append(evNodeSpans, evRethSpans...))
+
+	// assert expected ev-node span names
+	assertSpanNames(t, evNodeSpans, []string{
+		"BlockExecutor.ProduceBlock",
+		"BlockExecutor.ApplyBlock",
+		"BlockExecutor.CreateBlock",
+		"BlockExecutor.RetrieveBatch",
+		"Executor.ExecuteTxs",
+		"Executor.SetFinal",
+		"Engine.ForkchoiceUpdated",
+		"Engine.NewPayload",
+		"Engine.GetPayload",
+		"Eth.GetBlockByNumber",
+		"Sequencer.GetNextBatch",
+		"DASubmitter.SubmitHeaders",
+		"DASubmitter.SubmitData",
+		"DA.Submit",
+	}, "ev-node-smoke")
+
+	// assert expected ev-reth span names
+	assertSpanNames(t, evRethSpans, []string{
+		"build_payload",
+		"execute_tx",
+		"try_build",
+		"validate_transaction",
+	}, "ev-reth")
+
+	s.Require().Greater(sent, float64(0), "at least one transaction should have been sent")
+	s.Require().Zero(fail, "no transactions should have failed")
+}