fix(seidb): refuse a digest replay that would repair the live changelog

sei-db/tools/cmd/seidb/operations/evm_logical_digest.go

@@ -14,12 +14,14 @@ import (
 	"sort"
 
 	"github.com/sei-protocol/sei-chain/sei-db/common/keys"
+	"github.com/sei-protocol/sei-chain/sei-db/common/utils"
 	"github.com/sei-protocol/sei-chain/sei-db/proto"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/migration"
+	"github.com/sei-protocol/sei-chain/sei-db/wal"
 	"github.com/spf13/cobra"
 )
 
@@ -84,9 +86,12 @@ const (
 //   - replay (SLOW): opens a read-only DB, replays the changelog up to
 //     --height, then walks the in-memory/mmap tree. Roughly an order of
 //     magnitude slower than snapshot (changelog replay + per-leaf tree walk
-//     instead of a sequential file read). Use it only when no snapshot exists
-//     at the target height — e.g. nodes whose snapshot rewrite lags the tip, so
-//     an arbitrary comparison height has no snapshot-<height> on disk.
+//     instead of a sequential file read). It refuses to run on a changelog whose
+//     tail a live writer is still filling, and asks the operator to rerun,
+//     because opening such a changelog would truncate that tail. Use it only
+//     when no snapshot exists at the target height — e.g. nodes whose snapshot
+//     rewrite lags the tip, so an arbitrary comparison height has no
+//     snapshot-<height> on disk.
 //
 // The flatkv side is always a pebble WAL-replay-to-height and is fast
 // regardless. So when comparing across nodes, pick a height that is an existing
@@ -1105,6 +1110,17 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization
 }
 
 func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) {
+	// memiavl.OpenDB repairs a torn changelog tail by truncating it, even under
+	// ReadOnly. On a live node that tail is usually a write in progress, so
+	// refuse the run instead of letting the open damage the source.
+	if err := wal.VerifyIntact(utils.GetChangelogPath(dbDir)); err != nil {
+		if errors.Is(err, wal.ErrCorrupt) {
+			return nil, fmt.Errorf("memiavl changelog tail is incomplete or changing; live WAL was not "+
+				"modified; rerun the command, and if the error persists after stopping seid, repair the "+
+				"WAL offline: %w", err)
+		}
+		return nil, fmt.Errorf("verify memiavl changelog: %w", err)
+	}
 	db, err := memiavl.OpenDB(height, memiavl.Options{
 		Dir:      dbDir,
 		ReadOnly: true,
@@ -1113,9 +1129,47 @@ func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error)
 	if err != nil {
 		return nil, fmt.Errorf("open memiavl read-only replay: %w", err)
 	}
+	if err := verifyReplayCoverage(db, height); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
 	return db, nil
 }
 
+// verifyReplayCoverage reports whether the opened DB replayed every version
+// between its snapshot and height.
+//
+// The final version alone does not prove coverage. Catchup starts at the
+// changelog's first offset whenever the snapshot ends before it, so a changelog
+// pruned past the snapshot replays a contiguous suffix, reaches the requested
+// height, and silently omits the versions in between.
+func verifyReplayCoverage(db *memiavl.DB, height int64) error {
+	if height > 0 && db.Version() != height {
+		return fmt.Errorf("memiavl replay reached version %d, not the requested height %d; "+
+			"the changelog does not cover that height", db.Version(), height)
+	}
+	snapshotVersion := db.SnapshotVersion()
+	if db.Version() <= snapshotVersion {
+		return nil
+	}
+	firstOffset, err := db.GetWAL().FirstOffset()
+	if err != nil {
+		return fmt.Errorf("read memiavl changelog first offset: %w", err)
+	}
+	if firstOffset == 0 {
+		return nil
+	}
+	// #nosec G115 -- WAL offsets are far below MaxInt64 in practice.
+	firstVersion := int64(firstOffset) + db.GetWALIndexDelta()
+	if firstVersion > snapshotVersion+1 {
+		return fmt.Errorf("the memiavl changelog starts at version %d but the snapshot ends at version "+
+			"%d, so versions %d-%d would be missing from the replay; digest a height at or below %d, "+
+			"or use --memiavl-open-mode snapshot",
+			firstVersion, snapshotVersion, snapshotVersion+1, firstVersion-1, snapshotVersion)
+	}
+	return nil
+}
+
 func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error {
 	db, err := openMemiAVLReplayReadOnly(dbDir, height)
 	if err != nil {

sei-db/tools/cmd/seidb/operations/memiavl_open_test.go

@@ -0,0 +1,120 @@
+package operations
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/sei-protocol/sei-chain/sei-db/common/keys"
+	"github.com/sei-protocol/sei-chain/sei-db/common/utils"
+	"github.com/sei-protocol/sei-chain/sei-db/proto"
+	"github.com/sei-protocol/sei-chain/sei-db/wal"
+)
+
+func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing.T) {
+	homeDir := t.TempDir()
+	store := newTestMemiavlStore(t, homeDir)
+	require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{
+		Name:      keys.EVMStoreKey,
+		Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}},
+	}}))
+	_, err := store.Commit()
+	require.NoError(t, err)
+	require.NoError(t, store.Close())
+
+	dbDir := utils.GetCosmosSCStorePath(homeDir)
+	segment := lastOperationsMemiAVLWALSegment(t, dbDir)
+	committed, err := os.ReadFile(filepath.Clean(segment))
+	require.NoError(t, err)
+	require.NotEmpty(t, committed)
+	before := committed[:len(committed)-1]
+	require.NoError(t, os.WriteFile(filepath.Clean(segment), before, 0o600))
+
+	_, err = openMemiAVLReplayReadOnly(dbDir, 0)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "live WAL was not modified")
+	require.Contains(t, err.Error(), "rerun the command")
+
+	after, readErr := os.ReadFile(filepath.Clean(segment))
+	require.NoError(t, readErr)
+	require.Equal(t, before, after)
+}
+
+func TestOpenMemiAVLReplayReadOnlyAcceptsAFullyCoveredHeight(t *testing.T) {
+	homeDir := t.TempDir()
+	store := newTestMemiavlStore(t, homeDir)
+	for nonce := uint64(1); nonce <= 3; nonce++ {
+		require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{
+			Name:      keys.EVMStoreKey,
+			Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}},
+		}}))
+		_, err := store.Commit()
+		require.NoError(t, err)
+	}
+	require.NoError(t, store.Close())
+
+	db, err := openMemiAVLReplayReadOnly(utils.GetCosmosSCStorePath(homeDir), 3)
+	require.NoError(t, err)
+	defer func() { _ = db.Close() }()
+	require.Equal(t, int64(3), db.Version())
+}
+
+// TestOpenMemiAVLReplayReadOnlyRejectsAPrunedChangelogGap covers a changelog
+// pruned past the snapshot. Replay reaches the requested height from a
+// contiguous suffix, so the final version looks correct while the versions
+// between the snapshot and the changelog's first entry were never applied.
+func TestOpenMemiAVLReplayReadOnlyRejectsAPrunedChangelogGap(t *testing.T) {
+	homeDir := t.TempDir()
+	store := newTestMemiavlStore(t, homeDir)
+	commit := func(nonce uint64) {
+		require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{
+			Name:      keys.EVMStoreKey,
+			Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}},
+		}}))
+		_, err := store.Commit()
+		require.NoError(t, err)
+	}
+	commit(1)
+	commit(2)
+	// Snapshot version 2 so the evm tree survives without the changelog entry
+	// that creates it, leaving the pruned gap as the only defect.
+	require.NoError(t, store.GetDB().RewriteSnapshot(context.Background()))
+	commit(3)
+	commit(4)
+	commit(5)
+	require.NoError(t, store.Close())
+
+	dbDir := utils.GetCosmosSCStorePath(homeDir)
+	changelog, err := wal.NewChangelogWAL(utils.GetChangelogPath(dbDir), wal.Config{})
+	require.NoError(t, err)
+	require.NoError(t, changelog.TruncateBefore(5))
+	require.NoError(t, changelog.Close())
+
+	db, err := openMemiAVLReplayReadOnly(dbDir, 5)
+	if db != nil {
+		_ = db.Close()
+	}
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "would be missing from the replay")
+	require.Contains(t, err.Error(), "--memiavl-open-mode snapshot")
+}
+
+func lastOperationsMemiAVLWALSegment(t *testing.T, dbDir string) string {
+	t.Helper()
+	changelogDir := utils.GetChangelogPath(dbDir)
+	entries, err := os.ReadDir(changelogDir)
+	require.NoError(t, err)
+	var names []string
+	for _, entry := range entries {
+		if !entry.IsDir() && len(entry.Name()) == 20 {
+			names = append(names, entry.Name())
+		}
+	}
+	require.NotEmpty(t, names)
+	sort.Strings(names)
+	return filepath.Join(changelogDir, names[len(names)-1])
+}

sei-db/wal/utils.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"math"
 	"os"
 	"path/filepath"
+	"strings"
 	"unsafe"
 
 	"github.com/tidwall/gjson"
@@ -32,6 +34,62 @@ func GetLastIndex(dir string) (index uint64, err error) {
 	return rlog.LastIndex()
 }
 
+// ErrCorrupt reports a log that cannot be read without repair.
+var ErrCorrupt = wal.ErrCorrupt
+
+// segmentNameLen is the length of a log segment file name.
+const segmentNameLen = 20
+
+// VerifyIntact reports whether the binary log in dir can be opened without
+// repair. It returns ErrCorrupt when the tail segment ends mid-record or an
+// interrupted truncation is still in progress, and never modifies dir.
+//
+// A reader on a live node calls this before it opens the log, because open
+// repairs what it finds: truncateCorruptedTail cuts a torn tail, and tidwall
+// completes an interrupted truncation by renaming or removing segments. On a
+// live node a torn tail is usually a write in progress rather than lasting
+// damage, so the caller reruns instead of repairing.
+func VerifyIntact(dir string) error {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("read wal dir %s: %w", dir, err)
+	}
+
+	// os.ReadDir sorts by name, and segment names are zero-padded, so the last
+	// match is the tail segment open would truncate.
+	var tail string
+	for _, entry := range entries {
+		name := entry.Name()
+		if entry.IsDir() || len(name) < segmentNameLen {
+			continue
+		}
+		if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") {
+			return fmt.Errorf("%w: truncation marker %s is present in %s", ErrCorrupt, name, dir)
+		}
+		tail = name
+	}
+	if tail == "" {
+		return nil
+	}
+
+	path := filepath.Join(dir, tail)
+	data, err := os.ReadFile(filepath.Clean(path))
+	if err != nil {
+		return fmt.Errorf("read wal segment %s: %w", path, err)
+	}
+	for pos := 0; pos < len(data); {
+		n, err := loadNextBinaryEntry(data[pos:])
+		if err != nil {
+			return fmt.Errorf("%w: segment %s ends mid-record at offset %d", ErrCorrupt, path, pos)
+		}
+		pos += n
+	}
+	return nil
+}
+
 // truncateCorruptedTail truncates the corrupted tail
 func truncateCorruptedTail(path string, format wal.LogFormat) error {
 	data, err := os.ReadFile(filepath.Clean(path))