fix(harness): unique chain id per run; stop reusing stale genesis

test/integration/benchmark_test.go

@@ -16,7 +16,7 @@ import (
 //
 // Inputs (env, mirroring k8s_nightly.yml):
 //
-//	SEI_CHAIN_ID     per-run chain id (e.g. bench-<run-id>)   [required]
+//	SEI_CHAIN_ID     base chain id (a per-run token is appended) [required]
 //	SEID_IMAGE       seid image under test                    [required]
 //	SEILOAD_IMAGE    sei-load benchmark image                 [required]
 //	SEI_RUN_ID       unique run id (sei.io/harness-run)       [default: SEI_CHAIN_ID]
@@ -33,7 +33,7 @@ import (
 func TestBenchmark(t *testing.T) {
 	requireCluster(t)
 
-	chainID := mustEnv(t, "SEI_CHAIN_ID")
+	chainID := runChainID(mustEnv(t, "SEI_CHAIN_ID"))
 	s := spec{
 		chainID:        chainID,
 		runID:          envOr("SEI_RUN_ID", chainID),

test/integration/chaossuite_test.go

@@ -71,7 +71,7 @@ var chaosScenarios = []chaosScenario{
 // CHAOS_DURATION [optional]. Run with -test.timeout 0 (see TestBenchmark).
 func TestChaosSuite(t *testing.T) {
 	requireCluster(t)
-	base := mustEnv(t, "SEI_CHAIN_ID")
+	base := runChainID(mustEnv(t, "SEI_CHAIN_ID"))
 	seid := mustEnv(t, "SEID_IMAGE")
 	ns := envOr("SEI_NAMESPACE", "")
 	duration := envOr("CHAOS_DURATION", "3m")

test/integration/harness_test.go

@@ -35,23 +35,29 @@ import (
 // runLabelKey marks a run's resources for the abnormal-exit reaper (t.Cleanup is
 // skipped on SIGKILL / a -test.timeout breach). provision stamps it on the
 // network + every node; a suite's directly-applied seiload Job + fault CRs stamp
-// it too. The matching nightly label-GC sweep is a pending platform deliverable;
-// until it ships, normal-exit teardown (t.Cleanup) + the SeiNetwork
-// DeletionPolicy cascade are the cleanup path.
+// it too. The nightly-gc CronJob reaps these by label (resources older than 5h —
+// above the longest suite deadline, so it never races a live run), cascading a
+// SeiNetwork delete to its validators + PVCs; normal-exit t.Cleanup is the fast
+// path.
 const runLabelKey = "sei.io/harness-run"
 
-// memiavlStorageConfig pins storage for the load/release/chaos suites (the
-// major-upgrade suite omits it — it tests the migration path). State commitment
-// stays on memiavl; the SeiDB state store is disabled because the latest image
-// defaults it on for full nodes, and enabling it on a fresh-genesis RPC follower
-// deadlocks seid at store-open before it binds listeners. Matches the validators
-// (ss-enable=false); FlatKV-migration coverage is unaffected — that's the SC
-// layer, not the historical state store. (storage.state_store.write_mode is gone:
-// the SS layer has no write-mode field on current sei-chain — EVM routing is the
-// evm-split bool — so the old key was a silently-ignored no-op.)
+// memiavlStorageConfig pins state commitment to memiavl for the load/release/chaos
+// suites (the major-upgrade suite omits it — it tests the migration path). The
+// controller default write-mode is rejected by the nightly image, so it must be
+// set explicitly; the state store is left at its image default.
 var memiavlStorageConfig = map[string]string{
 	"storage.state_commit.write_mode": "memiavl_only",
-	"storage.state_store.enable":      "false",
+}
+
+// runChainID appends a per-run token to the base chain id so every run gets a
+// fresh genesis and node keys. A static id reused across runs (e.g. "bench")
+// collides with the prior run's persisted genesis: the nodes boot with new keys
+// but the genesis — keyed by chain id — still names the prior run's validator
+// set, so the live nodes are not the validators genesis expects and consensus
+// halts at height 1. Nanosecond resolution so a same-second manual re-trigger
+// can't alias a prior run's not-yet-reaped chain.
+func runChainID(base string) string {
+	return base + "-" + strconv.FormatInt(time.Now().UnixNano(), 36)
 }
 
 // mergeConfig returns base overlaid with extra; extra wins on key collision.

test/integration/release_test.go

@@ -62,7 +62,7 @@ var releaseRPCConfig = map[string]string{
 // -test.timeout 0 (see TestBenchmark).
 func TestRelease(t *testing.T) {
 	requireCluster(t)
-	chainID := mustEnv(t, "SEI_CHAIN_ID")
+	chainID := runChainID(mustEnv(t, "SEI_CHAIN_ID"))
 	seid := mustEnv(t, "SEID_IMAGE")
 	releaseImage := mustEnv(t, "RELEASE_TEST_IMAGE")
 	ns := envOr("SEI_NAMESPACE", "")