From c6308d5645b283e7f4faa31c3345b9f4b4a404ab Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 3 Aug 2026 00:51:05 -0400 Subject: [PATCH 1/6] fix(seidb): stop evm-logical-digest replay mode from mutating a live memiavl --memiavl-open-mode=replay relied on memiavl's ReadOnly option, which only skips the LOCK file. The changelog is still opened read-write, and sei-db's wal.open() answers wal.ErrCorrupt with os.Truncate on the node's segment. On a live node a torn tail is the writer mid-append, not corruption. Truncating it drops committed versions, and since the writer's fd keeps its old offset, its next append leaves a zero-filled hole that binary.Uvarint reads as valid empty records rather than an error. The node keeps producing blocks and finds out at its next restart. Clone instead, as openFlatKVReadOnly already does for the other backend: hardlink the snapshot, byte-copy the changelog, verify it still covers snapshotVersion+1, retry if the writer prunes mid-clone, then open the clone normally. Every source access is now a read. Also reject a clone that falls short of --height, since a tail repair costs the trailing version and a silently early digest looks like real divergence; and export memiavl's SeekSnapshotDir rather than reimplementing its snapshot layout in the tool. Replay mode now needs a writable dbDir and room for the changelog copy, as the FlatKV path already did. memiavl.OpenDB(ReadOnly) is still reachable from wasm snapshot export and storev2/rootmulti proof queries; fixing it at the WAL layer is tracked separately. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 22 +++ .../seidb/operations/evm_logical_digest.go | 44 ++--- .../tools/cmd/seidb/operations/flatkv_open.go | 20 ++- .../cmd/seidb/operations/flatkv_open_test.go | 4 +- .../cmd/seidb/operations/memiavl_open.go | 140 +++++++++++++++ .../cmd/seidb/operations/memiavl_open_test.go | 159 ++++++++++++++++++ 6 files changed, 351 insertions(+), 38 deletions(-) create mode 100644 sei-db/tools/cmd/seidb/operations/memiavl_open.go create mode 100644 sei-db/tools/cmd/seidb/operations/memiavl_open_test.go diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index ddd18f675c..2b352c3e9f 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -1205,6 +1205,28 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return snapshotVersion, nil } +// SeekSnapshotDir returns the directory name and version of the snapshot that +// OpenDB would start from for targetVersion: the "current" link when +// targetVersion is 0, otherwise the newest snapshot at or below it. +// +// Exported for readers that need to resolve a snapshot without opening the DB, +// so they inherit this package's layout rules instead of restating them. +func SeekSnapshotDir(root string, targetVersion int64) (string, int64, error) { + if targetVersion == 0 { + version, err := currentVersion(root) + if err != nil { + return "", 0, fmt.Errorf("read current snapshot: %w", err) + } + return snapshotName(version), version, nil + } + + version, err := seekSnapshot(root, targetVersion) + if err != nil { + return "", 0, err + } + return snapshotName(version), version, nil +} + // GetEarliestVersion returns the earliest snapshot name in the db func GetEarliestVersion(root string) (int64, error) { var found int64 diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 79ecd140c6..aceed4cc98 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -81,12 +81,14 @@ const ( // that exact height (or --height 0 for the current symlink). This is the // preferred mode whenever the target height lines up with an existing // snapshot boundary. -// - replay (SLOW): opens a read-only DB, replays the changelog up to -// --height, then walks the in-memory/mmap tree. Roughly an order of +// - replay (SLOW): clones the newest snapshot at or below --height plus the +// changelog into a temp directory, replays the changelog up to --height in +// that clone, then walks the in-memory/mmap tree. Roughly an order of // magnitude slower than snapshot (changelog replay + per-leaf tree walk -// instead of a sequential file read). Use it only when no snapshot exists -// at the target height — e.g. nodes whose snapshot rewrite lags the tip, so -// an arbitrary comparison height has no snapshot- on disk. +// instead of a sequential file read) and it byte-copies the changelog, so +// it needs free space alongside the source. Use it only when no snapshot +// exists at the target height — e.g. nodes whose snapshot rewrite lags the +// tip, so an arbitrary comparison height has no snapshot- on disk. // // The flatkv side is always a pebble WAL-replay-to-height and is fast // regardless. So when comparing across nodes, pick a height that is an existing @@ -165,7 +167,7 @@ func EvmLogicalDigestCmd() *cobra.Command { cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir") cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )") cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink)") - cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: replays changelog to --height then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") + cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: clones the snapshot + changelog to a temp dir, replays to --height there, then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") cmd.Flags().String("memiavl-normalization", memiavlNormSemantic, "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|misc) instead of printing the global digest") cmd.Flags().Int("key-offset", 0, "Inspect mode: byte offset into physical key before applying --key-prefix / sharding") @@ -465,7 +467,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT requestedHeight: height, version: opened.Version(), } - var memReplayDB *memiavl.DB + var memReplayDB *openedMemIAVL var memEvmSnapshotDir string var memVersion int64 switch memiavlOpenMode { @@ -481,13 +483,13 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl snapshot=%s", opened.Version(), memEvmSnapshotDir) ctx.normalization = fmt.Sprintf("flatkv rows plus memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) case memiavlOpenModeReplay: - memReplayDB, err = openMemiAVLReplayReadOnly(memIAVLDir, height) + memReplayDB, err = openMemiAVLReplay(memIAVLDir, height) if err != nil { return err } defer func() { _ = memReplayDB.Close() }() memVersion = memReplayDB.Version() - ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl read-only replay dir=%s", opened.Version(), memIAVLDir) + ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl clone replay dir=%s", opened.Version(), memIAVLDir) ctx.normalization = fmt.Sprintf("flatkv rows plus replayed memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) default: return fmt.Errorf("unknown --memiavl-open-mode %q (want snapshot|replay)", memiavlOpenMode) @@ -504,7 +506,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT if boundary.Status() != migration.MigrationComplete { if memReplayDB != nil { if err := consumeCompositeMemiavl(func(fn func(rawKey, rawVal []byte) error) error { - return scanMemiavlReplayEVMLeaves(memReplayDB, fn) + return scanMemiavlReplayEVMLeaves(memReplayDB.DB, fn) }, "memiavl-replay", boundary, &d, accounts); err != nil { return err } @@ -1104,20 +1106,8 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization } } -func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { - db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: dbDir, - ReadOnly: true, - ZeroCopy: true, - }) - if err != nil { - return nil, fmt.Errorf("open memiavl read-only replay: %w", err) - } - return db, nil -} - func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error { - db, err := openMemiAVLReplayReadOnly(dbDir, height) + db, err := openMemiAVLReplay(dbDir, height) if err != nil { return err } @@ -1125,9 +1115,9 @@ func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normaliz switch normalization { case "", memiavlNormSemantic, memiavlNormIndependent: - return digestMemIAVLReplaySemantic(dbDir, height, db, findTarget) + return digestMemIAVLReplaySemantic(dbDir, height, db.DB, findTarget) case memiavlNormTranslator: - return digestMemIAVLReplayTranslator(dbDir, height, db, findTarget) + return digestMemIAVLReplayTranslator(dbDir, height, db.DB, findTarget) default: return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization) } @@ -1291,7 +1281,7 @@ func digestMemIAVLReplaySemantic(dbDir string, height int64, db *memiavl.DB, fin backend: "memiavl", mode: "semantic-replay", dbDir: dbDir, - source: "read-only memiavl DB opened from snapshot + changelog replay", + source: "isolated memiavl clone opened from snapshot + changelog replay", normalization: "independent semantic decoder for replayed memiavl EVM keys; does not call flatkv.ImportTranslator", requestedHeight: height, version: db.Version(), @@ -1308,7 +1298,7 @@ func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, f backend: "memiavl", mode: "translator-replay", dbDir: dbDir, - source: "read-only memiavl DB opened from snapshot + changelog replay", + source: "isolated memiavl clone opened from snapshot + changelog replay", normalization: "replayed memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload", requestedHeight: height, version: db.Version(), diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index 04c9ebe138..e722de0086 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -29,11 +29,10 @@ const ( maxCloneRetries = 3 ) -// errSourceChurning marks transient races where the source FlatKV directory -// mutates (snapshot pruned, WAL truncated) between our reads. It is the -// sentinel that prepareFlatKVToolingCloneWith uses to decide whether to -// retry instead of bailing out. -var errSourceChurning = errors.New("flatkv source kept churning during clone") +// errSourceChurning marks transient races where the source directory mutates +// (snapshot pruned, WAL truncated) between our reads. It is the sentinel that +// retryToolingClone uses to decide whether to retry instead of bailing out. +var errSourceChurning = errors.New("source kept churning during clone") // openedFlatKV wraps a temp-cloned FlatKV store used by tooling. // @@ -108,10 +107,13 @@ func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { } func prepareFlatKVToolingClone(dbDir string, height int64) (string, error) { - return prepareFlatKVToolingCloneWith(dbDir, height, tryPrepareFlatKVToolingClone) + return retryToolingClone(dbDir, height, tryPrepareFlatKVToolingClone) } -func prepareFlatKVToolingCloneWith(dbDir string, height int64, tryClone func(string, int64) (string, error)) (string, error) { +// retryToolingClone runs tryClone, retrying while the live writer keeps +// mutating the source out from under us. Shared by the FlatKV and memiavl +// tooling clones, which race the same writer in the same ways. +func retryToolingClone(dbDir string, height int64, tryClone func(string, int64) (string, error)) (string, error) { var lastErr error for attempt := 0; attempt < maxCloneRetries; attempt++ { tempDir, err := tryClone(dbDir, height) @@ -302,7 +304,7 @@ func isFlatKVSnapshotName(name string) bool { // error: snapshots can be many GB, and the previous behavior of falling back // to a byte-copy on tmpfs (the historical $TMPDIR default) routinely OOM'd // nodes and exhausted /tmp. Callers must ensure the tool clone dir lives on -// the same filesystem as the source FlatKV directory. +// the same filesystem as the source directory. // // Hardlinking is safe because: // - snapshot-N files are immutable after Pebble Checkpoint + Rename. @@ -368,7 +370,7 @@ func linkOnly(src, dst string) error { if err := os.Link(src, dst); err != nil { if isCrossDeviceLinkError(err) { return fmt.Errorf("hardlink %s -> %s failed across filesystems; "+ - "FlatKV tooling requires the temp clone to share a filesystem with the source: %w", + "seidb tooling requires the temp clone to share a filesystem with the source: %w", src, dst, err) } return err diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go index 469a085e1b..fcc7dd5f5d 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go @@ -123,7 +123,7 @@ func TestPrepareFlatKVToolingCloneMissingCurrentAndSnapshot(t *testing.T) { func TestPrepareFlatKVToolingCloneRetriesENOENT(t *testing.T) { var attempts int - cloneDir, err := prepareFlatKVToolingCloneWith(t.TempDir(), 0, func(string, int64) (string, error) { + cloneDir, err := retryToolingClone(t.TempDir(), 0, func(string, int64) (string, error) { attempts++ if attempts < maxCloneRetries { return "", fmt.Errorf("source vanished: %w", os.ErrNotExist) @@ -135,7 +135,7 @@ func TestPrepareFlatKVToolingCloneRetriesENOENT(t *testing.T) { require.Equal(t, maxCloneRetries, attempts) attempts = 0 - _, err = prepareFlatKVToolingCloneWith(t.TempDir(), 0, func(string, int64) (string, error) { + _, err = retryToolingClone(t.TempDir(), 0, func(string, int64) (string, error) { attempts++ return "", errors.New("permission denied") }) diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open.go b/sei-db/tools/cmd/seidb/operations/memiavl_open.go new file mode 100644 index 0000000000..ee68720e8f --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open.go @@ -0,0 +1,140 @@ +package operations + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +// openedMemIAVL is a memiavl DB opened against a temp clone of the source. +type openedMemIAVL struct { + *memiavl.DB + tempDir string +} + +func (o *openedMemIAVL) Close() error { + var err error + if o.DB != nil { + err = o.DB.Close() + } + if o.tempDir != "" { + if rmErr := os.RemoveAll(o.tempDir); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; cleanup temp dir: %w", err, rmErr) + } + return fmt.Errorf("cleanup temp dir: %w", rmErr) + } + } + return err +} + +// openMemiAVLReplay opens memiavl at the given height (0 means latest) by +// replaying the changelog on top of the newest snapshot at or below it. +// +// It clones the source rather than using memiavl's ReadOnly option, which is +// weaker than it reads: ReadOnly skips the LOCK file but the changelog is still +// opened read-write, and that open "repairs" a torn tail record by truncating +// the segment. On a live node a torn tail is the writer mid-append, not +// corruption, so a read-only replay could truncate committed versions out from +// under a running seid. Repairing a torn tail in a private copy is harmless. +// This mirrors openFlatKVReadOnly. +func openMemiAVLReplay(dbDir string, height int64) (*openedMemIAVL, error) { + tempDir, err := prepareMemIAVLToolingClone(dbDir, height) + if err != nil { + return nil, err + } + + db, err := memiavl.OpenDB(height, memiavl.Options{ + Dir: tempDir, + ZeroCopy: true, + }) + if err != nil { + _ = os.RemoveAll(tempDir) + return nil, fmt.Errorf("open memiavl clone at version %d: %w", height, err) + } + opened := &openedMemIAVL{DB: db, tempDir: tempDir} + + // memiavl replays whatever the changelog holds and reports success even + // when that falls short of the requested height. Every tail repair inside + // the clone costs the trailing version, and a digest computed one version + // early is indistinguishable from real divergence when comparing nodes. + if reached := db.Version(); height > 0 && reached != height { + err := fmt.Errorf("memiavl clone version mismatch: requested %d, reached %d "+ + "(changelog does not cover the target height)", height, reached) + if closeErr := opened.Close(); closeErr != nil { + return nil, errors.Join(err, fmt.Errorf("close clone: %w", closeErr)) + } + return nil, err + } + + return opened, nil +} + +func prepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { + return retryToolingClone(dbDir, height, tryPrepareMemIAVLToolingClone) +} + +// tryPrepareMemIAVLToolingClone mirrors tryPrepareFlatKVToolingClone: the two +// layouts are the same shape (current -> snapshot-N/, changelog/, LOCK) and both +// publish snapshots by rename and drop them wholesale, so the same +// hardlink-the-snapshot / byte-copy-the-changelog split applies. +func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { + snapshotName, snapshotVersion, err := memiavl.SeekSnapshotDir(dbDir, height) + if err != nil { + return "", err + } + + // The clone must sit inside dbDir to share a filesystem with the source + // snapshot: dbDir is often its own mount point, so a sibling directory is + // not enough and hardlinks would fail across the boundary. + if err := os.MkdirAll(dbDir, 0o750); err != nil { + return "", fmt.Errorf("ensure clone root %s: %w", dbDir, err) + } + // Not a "-tmp" suffix: memiavl's removeTmpDirs deletes every "*-tmp" + // directory under its root when a node opens the DB read-write. + tempDir, err := os.MkdirTemp(dbDir, ".seidb-memiavl-tool-*") + if err != nil { + return "", fmt.Errorf("create temp dir under %s: %w", dbDir, err) + } + cleanup := func(err error) (string, error) { + _ = os.RemoveAll(tempDir) + return "", err + } + + srcSnapshotDir := filepath.Join(dbDir, snapshotName) + dstSnapshotDir := filepath.Join(tempDir, snapshotName) + if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { + return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) + } + + if err := os.Symlink(snapshotName, filepath.Join(tempDir, "current")); err != nil { + return cleanup(fmt.Errorf("create current symlink: %w", err)) + } + + srcChangelogDir := filepath.Join(dbDir, "changelog") + info, err := os.Stat(srcChangelogDir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return cleanup(fmt.Errorf("stat changelog: %w", err)) + } + if err == nil && !info.IsDir() { + return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) + } + if err == nil { + dstChangelogDir := filepath.Join(tempDir, "changelog") + if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { + return cleanup(fmt.Errorf("clone changelog: %w", err)) + } + // A live writer can roll a new snapshot between our snapshot clone and + // our changelog copy, then prune the changelog up to that newer + // version — leaving a copy that no longer covers snapshotVersion+1 and + // a catchup that would silently skip versions. Retryable. + if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { + return cleanup(err) + } + } + + return tempDir, nil +} diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go new file mode 100644 index 0000000000..6110b8e4f5 --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -0,0 +1,159 @@ +package operations + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// newMemiavlSourceDir builds a memiavl directory with `versions` committed +// blocks and returns the directory a tool would be pointed at. +func newMemiavlSourceDir(t *testing.T, versions int) string { + t.Helper() + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + for i := 1; i <= versions; i++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(i)), uint64(i))}}, + }})) + v, err := store.Commit() + require.NoError(t, err) + require.Equal(t, int64(i), v) + } + require.NoError(t, store.Close()) + return utils.GetCosmosSCStorePath(homeDir) +} + +// snapshotDirState records every regular file under root by relative path and +// content so a later comparison catches truncation, appends, and deletions. +func snapshotDirState(t *testing.T, root string) map[string][]byte { + t.Helper() + state := make(map[string][]byte) + require.NoError(t, filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + bz, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return err + } + state[rel] = bz + return nil + })) + return state +} + +// lastChangelogSegment returns the newest tidwall segment file in the memiavl +// changelog. Segment names are fixed-width, so lexical order is index order. +func lastChangelogSegment(t *testing.T, dbDir string) string { + t.Helper() + changelogDir := filepath.Join(dbDir, "changelog") + entries, err := os.ReadDir(changelogDir) + require.NoError(t, err) + var names []string + for _, e := range entries { + if !e.IsDir() && len(e.Name()) >= 20 { + names = append(names, e.Name()) + } + } + require.NotEmpty(t, names, "memiavl changelog should have at least one segment") + sort.Strings(names) + return filepath.Join(changelogDir, names[len(names)-1]) +} + +// TestOpenMemiAVLReplayLeavesSourceUntouched is the regression test for the +// audited hazard: a replay that advertises itself as read-only used to hand the +// live changelog straight to memiavl, whose WAL open truncates a torn tail. A +// torn tail on a running node is just the writer mid-append, so the "repair" +// destroyed committed versions. The tool must now repair only its own copy. +func TestOpenMemiAVLReplayLeavesSourceUntouched(t *testing.T) { + dbDir := newMemiavlSourceDir(t, 3) + + // A single length-prefix byte declaring a 16-byte record that never + // arrived is exactly what a reader observes while the writer is + // partway through appending a block. + segment := lastChangelogSegment(t, dbDir) + intact, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + torn := append(append([]byte{}, intact...), 0x10) + require.NoError(t, os.WriteFile(segment, torn, 0o600)) + + before := snapshotDirState(t, dbDir) + + db, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err, "replay must tolerate a torn tail by repairing its own clone") + require.Equal(t, int64(3), db.Version()) + require.NoError(t, db.Close()) + + require.Equal(t, before, snapshotDirState(t, dbDir), + "replay must not add, remove, truncate, or rewrite any file in the source directory") + + after, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + require.Equal(t, torn, after, "the torn tail must still be there for the live writer to finish") +} + +// TestOpenMemiAVLReplayWorksWhileWriterHoldsLock pins the other half of the +// contract: avoiding the mutation must not cost us the ability to read a live +// node. The clone is independent, so the source LOCK is irrelevant to us. +func TestOpenMemiAVLReplayWorksWhileWriterHoldsLock(t *testing.T) { + homeDir := t.TempDir() + writer := newTestMemiavlStore(t, homeDir) + require.NoError(t, writer.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}}, + }})) + _, err := writer.Commit() + require.NoError(t, err) + defer func() { require.NoError(t, writer.Close()) }() + + dbDir := utils.GetCosmosSCStorePath(homeDir) + require.FileExists(t, filepath.Join(dbDir, "LOCK")) + + db, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err, "tooling clone must not contend for the live writer's lock") + require.Equal(t, int64(1), db.Version()) + require.NoError(t, db.Close()) +} + +// TestOpenMemiAVLReplayRejectsShortChangelog guards the failure mode this +// design makes routine: repairing a torn tail inside the clone silently costs +// the trailing version, and memiavl reports success anyway. For a tool whose +// whole job is comparing digests across nodes, quietly digesting one version +// early is indistinguishable from a real state divergence. +func TestOpenMemiAVLReplayRejectsShortChangelog(t *testing.T) { + dbDir := newMemiavlSourceDir(t, 3) + + segment := lastChangelogSegment(t, dbDir) + intact, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + // Lop off the tail so the final committed record is torn and the + // repaired clone can only reach version 2. + require.NoError(t, os.WriteFile(segment, intact[:len(intact)-8], 0o600)) + + _, err = openMemiAVLReplay(dbDir, 3) + require.Error(t, err) + require.Contains(t, err.Error(), "requested 3, reached 2") + + // The rejected clone must not be left behind in the node's data dir. + entries, err := os.ReadDir(dbDir) + require.NoError(t, err) + for _, e := range entries { + require.NotContains(t, e.Name(), ".seidb-memiavl-tool-") + } +} From 87b886c1983fcfb918ff7ec61fb75fc5bb13a667 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 3 Aug 2026 11:36:02 -0400 Subject: [PATCH 2/6] fix(seidb): address review findings on the tooling clone path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clone-based replay open regressed nodes bootstrapped with a non-default initial height: memiavl keeps its bootstrap snapshot named snapshot-0 even after SetInitialVersion(N), while the first changelog entry is version N, so verifyClonedWALCovers treated the valid successor as a gap and replay mode failed deterministically with "source kept churning" until the first snapshot rewrite. A freshly recovered chain (genesis initial_height > 1) is exactly that shape. Derive the snapshot's successor from the cloned snapshot's metadata via utils.NextVersion instead of assuming snapshotVersion+1; FlatKV keeps the +1 rule because SetInitialVersion(N) seeds snapshot- there. Also from review: - Reap abandoned clones. A SIGKILLed tool leaves its clone inside the data dir, where hardlinks pin snapshot inodes so pruning frees no disk and the deliberate non-"-tmp" name escapes memiavl's cleanup. Every clone now carries a flock-held owner marker (OFD/flock locks die with the process); clone creation sweeps siblings whose lock is acquirable, and marker-less directories once they are older than the mkdir-to-lock window can explain. - Document that --height 0 against a live node is best-effort: the clone is a consistent committed prefix as of the copy instant, and a detected torn-tail repair now prints a warning telling the operator to trust the report's version line. Cross-node comparisons should use an explicit common height. - Drop the MkdirAll that could never do useful work (snapshot selection already reads dbDir and fails first when it is missing). - Rename SeekSnapshotDir to SeekSnapshotName — it returns a name relative to root, not a path — and pin the exported contract with a unit test, along with ReadMetadata which the successor derivation needs. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 11 +- sei-db/state_db/sc/memiavl/db_test.go | 35 +++++ sei-db/state_db/sc/memiavl/multitree.go | 10 +- .../seidb/operations/evm_logical_digest.go | 2 +- .../tools/cmd/seidb/operations/flatkv_open.go | 121 +++++++++------ .../cmd/seidb/operations/flatkv_open_test.go | 34 ++--- .../cmd/seidb/operations/memiavl_open.go | 86 +++++++---- .../cmd/seidb/operations/memiavl_open_test.go | 36 +++++ .../tools/cmd/seidb/operations/tool_clone.go | 139 ++++++++++++++++++ .../cmd/seidb/operations/tool_clone_test.go | 75 ++++++++++ 10 files changed, 444 insertions(+), 105 deletions(-) create mode 100644 sei-db/tools/cmd/seidb/operations/tool_clone.go create mode 100644 sei-db/tools/cmd/seidb/operations/tool_clone_test.go diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 2b352c3e9f..a924036130 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -1205,13 +1205,14 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return snapshotVersion, nil } -// SeekSnapshotDir returns the directory name and version of the snapshot that -// OpenDB would start from for targetVersion: the "current" link when -// targetVersion is 0, otherwise the newest snapshot at or below it. +// SeekSnapshotName returns the directory name (relative to root — callers +// join it themselves) and version of the snapshot that OpenDB would start +// from for targetVersion: the "current" link when targetVersion is 0, +// otherwise the newest snapshot at or below it. // // Exported for readers that need to resolve a snapshot without opening the DB, // so they inherit this package's layout rules instead of restating them. -func SeekSnapshotDir(root string, targetVersion int64) (string, int64, error) { +func SeekSnapshotName(root string, targetVersion int64) (string, int64, error) { if targetVersion == 0 { version, err := currentVersion(root) if err != nil { @@ -1349,7 +1350,7 @@ func isSnapshotName(name string) bool { // it's needed for upgrade module to check store upgrades, // it returns 0 if db doesn't exist or is empty. func GetLatestVersion(dir string) (int64, error) { - metadata, err := readMetadata(currentPath(dir)) + metadata, err := ReadMetadata(currentPath(dir)) if err != nil { if os.IsNotExist(err) { return 0, nil diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index 3cddd00f77..d047b29b11 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -1116,3 +1116,38 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) { require.NoError(t, err) require.Equal(t, "snapshot-1", target) } + +// TestSeekSnapshotName pins the exported snapshot-resolution contract that +// external readers (seidb tooling) rely on: the returned name is relative to +// root, targetVersion 0 resolves through the current link, a positive target +// selects the newest snapshot at or below it, and a target older than the +// earliest snapshot reports pruning instead of guessing. +func TestSeekSnapshotName(t *testing.T) { + root := t.TempDir() + for _, v := range []int64{5, 10} { + require.NoError(t, os.Mkdir(filepath.Join(root, snapshotName(v)), 0o750)) + } + require.NoError(t, os.Symlink(snapshotName(10), currentPath(root))) + + name, version, err := SeekSnapshotName(root, 0) + require.NoError(t, err) + require.Equal(t, snapshotName(10), name) + require.Equal(t, int64(10), version) + + name, version, err = SeekSnapshotName(root, 7) + require.NoError(t, err) + require.Equal(t, snapshotName(5), name) + require.Equal(t, int64(5), version) + + name, version, err = SeekSnapshotName(root, 10) + require.NoError(t, err) + require.Equal(t, snapshotName(10), name) + require.Equal(t, int64(10), version) + + _, _, err = SeekSnapshotName(root, 3) + require.Error(t, err) + require.Contains(t, err.Error(), "target version is pruned") + + _, _, err = SeekSnapshotName(filepath.Join(root, "does-not-exist"), 0) + require.Error(t, err) +} diff --git a/sei-db/state_db/sc/memiavl/multitree.go b/sei-db/state_db/sc/memiavl/multitree.go index 6e9101e339..f0f85e417a 100644 --- a/sei-db/state_db/sc/memiavl/multitree.go +++ b/sei-db/state_db/sc/memiavl/multitree.go @@ -76,7 +76,7 @@ func NewEmptyMultiTree(initialVersion uint32) *MultiTree { func LoadMultiTree(ctx context.Context, dir string, opts Options) (*MultiTree, error) { startTime := time.Now() - metadata, err := readMetadata(dir) + metadata, err := ReadMetadata(dir) if err != nil { return nil, err } @@ -600,7 +600,13 @@ func (t *MultiTree) ReplaceWith(other *MultiTree) error { return errors.Join(errs...) } -func readMetadata(dir string) (*proto.MultiTreeMetadata, error) { +// ReadMetadata loads a snapshot directory's MultiTreeMetadata (commit info + +// initial version). Exported for readers that need snapshot metadata without +// opening the DB — e.g. seidb tooling, whose changelog-coverage check must +// know the initial version because a DB bootstrapped with initial version N +// keeps its first snapshot named snapshot-0 while the first changelog entry +// is version N, not 1. +func ReadMetadata(dir string) (*proto.MultiTreeMetadata, error) { // load commit info bz, err := os.ReadFile(filepath.Join(filepath.Clean(dir), MetadataFileName)) if err != nil { diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index aceed4cc98..143acc1e50 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -166,7 +166,7 @@ func EvmLogicalDigestCmd() *cobra.Command { cmd.Flags().StringP("db-dir", "d", "", "For flatkv: the flatkv data dir. For memiavl: the memiavl root dir (contains current/ and snapshot-* )") cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir") cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )") - cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink)") + cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink). On a live node 0 = latest is best-effort — the printed version line records what was actually digested; always pass an explicit common height when comparing nodes") cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: clones the snapshot + changelog to a temp dir, replays to --height there, then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") cmd.Flags().String("memiavl-normalization", memiavlNormSemantic, "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|misc) instead of printing the global digest") diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index e722de0086..8f0d3ec086 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -40,7 +40,7 @@ var errSourceChurning = errors.New("source kept churning during clone") // WAL so they do not compete with a live node for the FlatKV writer lock. type openedFlatKV struct { *flatkv.CommitStore - tempDir string + clone *toolClone } func (o *openedFlatKV) Close() error { @@ -48,13 +48,11 @@ func (o *openedFlatKV) Close() error { if o.CommitStore != nil { err = o.CommitStore.Close() } - if o.tempDir != "" { - if rmErr := os.RemoveAll(o.tempDir); rmErr != nil { - if err != nil { - return fmt.Errorf("%w; cleanup temp dir: %w", err, rmErr) - } - return fmt.Errorf("cleanup temp dir: %w", rmErr) + if rmErr := o.clone.Remove(); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; %w", err, rmErr) } + return rmErr } return err } @@ -78,54 +76,73 @@ func (o *openedFlatKV) Close() error { // os.ReadDir and os.Link calls, we surface ENOENT, re-select the // snapshot, and retry up to maxCloneRetries times. // -// height=0 means latest version. +// height=0 means the latest version, best-effort on a live node: the clone is +// a consistent committed prefix as of the copy instant, and a torn-tail +// repair inside the clone (surfaced as a warning) can land it one version +// behind the source tip. Tools print the version actually opened; treat that +// line as authoritative when comparing across nodes. func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { - tempDir, err := prepareFlatKVToolingClone(dbDir, height) + clone, err := prepareFlatKVToolingClone(dbDir, height) if err != nil { return nil, err } + warnIfCloneRepaired(clone, "flatkv", height) cfg := config.DefaultConfig() - cfg.DataDir = tempDir + cfg.DataDir = clone.dir store, err := flatkv.NewCommitStore(context.Background(), cfg) if err != nil { - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("failed to create FlatKV store: %w", err) } if _, err := store.LoadVersion(height, false); err != nil { _ = store.Close() - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("failed to open FlatKV at version %d: %w", height, err) } return &openedFlatKV{ CommitStore: store, - tempDir: tempDir, + clone: clone, }, nil } -func prepareFlatKVToolingClone(dbDir string, height int64) (string, error) { +// warnIfCloneRepaired tells the operator that the cloned changelog had a torn +// tail (the byte-copy raced the live writer mid-append) and was repaired +// inside the clone. For an explicit --height the reached-version checks catch +// any resulting shortfall; for height 0 ("latest") there is no target to +// check against, so the printed version line is the only record of what was +// actually digested. +func warnIfCloneRepaired(clone *toolClone, backend string, height int64) { + if !clone.walRepaired || height != 0 { + return + } + fmt.Fprintf(os.Stderr, "warning: cloned %s changelog had a torn tail (live writer mid-append) and was repaired in the clone; "+ + "the opened version may trail the source tip by one — trust the printed version line\n", backend) +} + +func prepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error) { return retryToolingClone(dbDir, height, tryPrepareFlatKVToolingClone) } // retryToolingClone runs tryClone, retrying while the live writer keeps // mutating the source out from under us. Shared by the FlatKV and memiavl // tooling clones, which race the same writer in the same ways. -func retryToolingClone(dbDir string, height int64, tryClone func(string, int64) (string, error)) (string, error) { +func retryToolingClone(dbDir string, height int64, tryClone func(string, int64) (*toolClone, error)) (*toolClone, error) { var lastErr error for attempt := 0; attempt < maxCloneRetries; attempt++ { - tempDir, err := tryClone(dbDir, height) + clone, err := tryClone(dbDir, height) if err == nil { - return tempDir, nil + return clone, nil } if !isCloneRetryableError(err) { - return "", err + return nil, err } lastErr = err } - return "", fmt.Errorf("clone aborted after %d retries, source kept churning: %w", maxCloneRetries, lastErr) + return nil, fmt.Errorf("clone aborted after %d retries, source kept churning: %w", maxCloneRetries, lastErr) } // isCloneRetryableError reports whether err indicates a transient race with @@ -136,39 +153,37 @@ func isCloneRetryableError(err error) bool { return errors.Is(err, os.ErrNotExist) || errors.Is(err, errSourceChurning) } -func tryPrepareFlatKVToolingClone(dbDir string, height int64) (string, error) { +func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error) { snapshotName, err := selectFlatKVSnapshot(dbDir, height) if err != nil { - return "", err + return nil, err } snapshotVersion, err := strconv.ParseInt(snapshotName[len(flatkvSnapshotPrefix):], 10, 64) if err != nil { - return "", fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) + return nil, fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) } - // Place the temp clone inside dbDir so it is on the exact same mounted - // filesystem as the source snapshots. A sibling directory is not enough: - // dbDir itself is often a mount point on dedicated data volumes. - cloneRoot := dbDir - if err := os.MkdirAll(cloneRoot, 0o750); err != nil { - return "", fmt.Errorf("ensure clone root %s: %w", cloneRoot, err) - } - tempDir, err := os.MkdirTemp(cloneRoot, ".seidb-flatkv-tool-*") + // The clone must sit inside dbDir so it is on the exact same mounted + // filesystem as the source snapshots (dbDir is often its own mount + // point, so a sibling directory is not enough and hardlinks would fail + // across the boundary). selectFlatKVSnapshot already read dbDir, so it + // is known to exist. + clone, err := newToolClone(dbDir, ".seidb-flatkv-tool-") if err != nil { - return "", fmt.Errorf("create temp dir under %s: %w", cloneRoot, err) + return nil, err } - cleanup := func(err error) (string, error) { - _ = os.RemoveAll(tempDir) - return "", err + cleanup := func(err error) (*toolClone, error) { + _ = clone.Remove() + return nil, err } srcSnapshotDir := filepath.Join(dbDir, snapshotName) - dstSnapshotDir := filepath.Join(tempDir, snapshotName) + dstSnapshotDir := filepath.Join(clone.dir, snapshotName) if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) } - if err := os.Symlink(snapshotName, filepath.Join(tempDir, "current")); err != nil { + if err := os.Symlink(snapshotName, filepath.Join(clone.dir, "current")); err != nil { return cleanup(fmt.Errorf("create current symlink: %w", err)) } @@ -181,29 +196,39 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (string, error) { return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) } if err == nil { - dstChangelogDir := filepath.Join(tempDir, "changelog") + dstChangelogDir := filepath.Join(clone.dir, "changelog") if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { return cleanup(fmt.Errorf("clone changelog: %w", err)) } // Detect the snapshot/WAL race: a live writer can roll a new // snapshot between our snapshot clone and our changelog copy and // then truncateWAL up to that newer snapshot's version. If that - // happened, the cloned WAL no longer covers snapshotVersion+1, - // and a downstream catchup would silently jump over missing - // versions. Surface it as a retryable error so the outer loop - // re-selects the snapshot and tries again. - if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { + // happened, the cloned WAL no longer covers the snapshot's + // successor version, and a downstream catchup would silently jump + // over missing versions. Surface it as a retryable error so the + // outer loop re-selects the snapshot and tries again. + // + // FlatKV snapshots are always named with a real committed version — + // SetInitialVersion(N) seeds committedVersion N-1 and writes + // snapshot- — so the successor is unconditionally + // snapshotVersion+1 here (unlike memiavl, whose bootstrap + // snapshot-0 hides a configurable initial version). + sizeBefore := changelogByteSize(dstChangelogDir) + if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion, snapshotVersion+1); err != nil { return cleanup(err) } + clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore } - return tempDir, nil + return clone, nil } // verifyClonedWALCovers opens the cloned WAL just long enough to ensure it // either is empty, ends at or before snapshotVersion (no replay needed), or -// starts at or before snapshotVersion+1 (catchup can resume cleanly). -func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion int64) error { +// starts at or before firstNeeded — the first version catchup must replay on +// top of the snapshot (snapshotVersion+1, except for a memiavl bootstrap +// snapshot-0 whose successor is the configured initial version). +func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion, firstNeeded int64) error { walLog, err := wal.NewChangelogWAL(dstChangelogDir, wal.Config{}) if err != nil { return fmt.Errorf("open cloned changelog for validation: %w", err) @@ -234,11 +259,11 @@ func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion int64) error if lastVer <= snapshotVersion { return nil } - if firstVer <= snapshotVersion+1 { + if firstVer <= firstNeeded { return nil } - return fmt.Errorf("%w: cloned WAL starts at version %d but snapshot is %d (truncated past snapshot mid-clone)", - errSourceChurning, firstVer, snapshotVersion) + return fmt.Errorf("%w: cloned WAL starts at version %d but catchup needs %d over snapshot %d (truncated past snapshot mid-clone)", + errSourceChurning, firstVer, firstNeeded, snapshotVersion) } func readWALEntryVersion(walLog wal.ChangelogWAL, off uint64) (int64, error) { diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go index fcc7dd5f5d..839007c7ed 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go @@ -55,17 +55,17 @@ func TestPrepareFlatKVToolingCloneHardlinksSnapshotAndCopiesChangelog(t *testing srcChangelogFile := filepath.Join(dbDir, "changelog", "000001.log") require.NoError(t, os.WriteFile(srcChangelogFile, []byte("wal-data"), 0o600)) - cloneDir, err := prepareFlatKVToolingClone(dbDir, 0) + clone, err := prepareFlatKVToolingClone(dbDir, 0) require.NoError(t, err) - defer os.RemoveAll(cloneDir) //nolint:errcheck // test cleanup + defer clone.Remove() //nolint:errcheck // test cleanup - target, err := os.Readlink(filepath.Join(cloneDir, "current")) + target, err := os.Readlink(filepath.Join(clone.dir, "current")) require.NoError(t, err) require.Equal(t, snapshot, target) - require.FileExists(t, filepath.Join(cloneDir, snapshot, "account", "000001.sst")) - require.NoFileExists(t, filepath.Join(cloneDir, snapshot, "LOCK")) - dstSnapshotFile := filepath.Join(cloneDir, snapshot, "account", "000001.sst") - dstChangelogFile := filepath.Join(cloneDir, "changelog", "000001.log") + require.FileExists(t, filepath.Join(clone.dir, snapshot, "account", "000001.sst")) + require.NoFileExists(t, filepath.Join(clone.dir, snapshot, "LOCK")) + dstSnapshotFile := filepath.Join(clone.dir, snapshot, "account", "000001.sst") + dstChangelogFile := filepath.Join(clone.dir, "changelog", "000001.log") require.FileExists(t, dstChangelogFile) srcSnapshotInfo, err := os.Stat(srcSnapshotFile) @@ -123,21 +123,21 @@ func TestPrepareFlatKVToolingCloneMissingCurrentAndSnapshot(t *testing.T) { func TestPrepareFlatKVToolingCloneRetriesENOENT(t *testing.T) { var attempts int - cloneDir, err := retryToolingClone(t.TempDir(), 0, func(string, int64) (string, error) { + clone, err := retryToolingClone(t.TempDir(), 0, func(string, int64) (*toolClone, error) { attempts++ if attempts < maxCloneRetries { - return "", fmt.Errorf("source vanished: %w", os.ErrNotExist) + return nil, fmt.Errorf("source vanished: %w", os.ErrNotExist) } - return t.TempDir(), nil + return &toolClone{dir: t.TempDir()}, nil }) require.NoError(t, err) - require.NotEmpty(t, cloneDir) + require.NotEmpty(t, clone.dir) require.Equal(t, maxCloneRetries, attempts) attempts = 0 - _, err = retryToolingClone(t.TempDir(), 0, func(string, int64) (string, error) { + _, err = retryToolingClone(t.TempDir(), 0, func(string, int64) (*toolClone, error) { attempts++ - return "", errors.New("permission denied") + return nil, errors.New("permission denied") }) require.Error(t, err) require.Equal(t, 1, attempts) @@ -160,15 +160,15 @@ func TestPrepareFlatKVToolingClonePlacesTempDirInsideDBDir(t *testing.T) { require.NoError(t, store.WriteSnapshot("")) require.NoError(t, store.Close()) - cloneDir, err := prepareFlatKVToolingClone(dbDir, 0) + clone, err := prepareFlatKVToolingClone(dbDir, 0) require.NoError(t, err) - defer os.RemoveAll(cloneDir) //nolint:errcheck // test cleanup + defer clone.Remove() //nolint:errcheck // test cleanup - rel, err := filepath.Rel(dbDir, cloneDir) + rel, err := filepath.Rel(dbDir, clone.dir) require.NoError(t, err) require.NotEqual(t, ".", rel) require.False(t, strings.HasPrefix(rel, ".."), "tooling clone must be created inside dbDir to stay on dbDir's mounted filesystem") - require.Contains(t, filepath.Base(cloneDir), ".seidb-flatkv-tool-") + require.Contains(t, filepath.Base(clone.dir), ".seidb-flatkv-tool-") } // TestPrepareFlatKVToolingCloneDetectsWALTruncationRace simulates the audited diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open.go b/sei-db/tools/cmd/seidb/operations/memiavl_open.go index ee68720e8f..c7b8b96987 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open.go @@ -3,16 +3,18 @@ package operations import ( "errors" "fmt" + "math" "os" "path/filepath" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" ) // openedMemIAVL is a memiavl DB opened against a temp clone of the source. type openedMemIAVL struct { *memiavl.DB - tempDir string + clone *toolClone } func (o *openedMemIAVL) Close() error { @@ -20,13 +22,11 @@ func (o *openedMemIAVL) Close() error { if o.DB != nil { err = o.DB.Close() } - if o.tempDir != "" { - if rmErr := os.RemoveAll(o.tempDir); rmErr != nil { - if err != nil { - return fmt.Errorf("%w; cleanup temp dir: %w", err, rmErr) - } - return fmt.Errorf("cleanup temp dir: %w", rmErr) + if rmErr := o.clone.Remove(); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; %w", err, rmErr) } + return rmErr } return err } @@ -41,21 +41,29 @@ func (o *openedMemIAVL) Close() error { // corruption, so a read-only replay could truncate committed versions out from // under a running seid. Repairing a torn tail in a private copy is harmless. // This mirrors openFlatKVReadOnly. +// +// height=0 ("latest") is best-effort on a live node: the clone is a +// consistent committed prefix as of the copy instant, and a torn-tail repair +// inside the clone (surfaced as a warning) can land it one version behind the +// source tip. There is no target version to check against, so the report's +// printed version line is the authoritative record of what was digested; +// cross-node comparisons should always use an explicit common --height. func openMemiAVLReplay(dbDir string, height int64) (*openedMemIAVL, error) { - tempDir, err := prepareMemIAVLToolingClone(dbDir, height) + clone, err := prepareMemIAVLToolingClone(dbDir, height) if err != nil { return nil, err } + warnIfCloneRepaired(clone, "memiavl", height) db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: tempDir, + Dir: clone.dir, ZeroCopy: true, }) if err != nil { - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("open memiavl clone at version %d: %w", height, err) } - opened := &openedMemIAVL{DB: db, tempDir: tempDir} + opened := &openedMemIAVL{DB: db, clone: clone} // memiavl replays whatever the changelog holds and reports success even // when that falls short of the requested height. Every tail repair inside @@ -73,7 +81,7 @@ func openMemiAVLReplay(dbDir string, height int64) (*openedMemIAVL, error) { return opened, nil } -func prepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { +func prepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, error) { return retryToolingClone(dbDir, height, tryPrepareMemIAVLToolingClone) } @@ -81,39 +89,51 @@ func prepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { // layouts are the same shape (current -> snapshot-N/, changelog/, LOCK) and both // publish snapshots by rename and drop them wholesale, so the same // hardlink-the-snapshot / byte-copy-the-changelog split applies. -func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { - snapshotName, snapshotVersion, err := memiavl.SeekSnapshotDir(dbDir, height) +func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, error) { + snapshotName, snapshotVersion, err := memiavl.SeekSnapshotName(dbDir, height) if err != nil { - return "", err + return nil, err } // The clone must sit inside dbDir to share a filesystem with the source // snapshot: dbDir is often its own mount point, so a sibling directory is // not enough and hardlinks would fail across the boundary. - if err := os.MkdirAll(dbDir, 0o750); err != nil { - return "", fmt.Errorf("ensure clone root %s: %w", dbDir, err) - } - // Not a "-tmp" suffix: memiavl's removeTmpDirs deletes every "*-tmp" - // directory under its root when a node opens the DB read-write. - tempDir, err := os.MkdirTemp(dbDir, ".seidb-memiavl-tool-*") + // SeekSnapshotName already read dbDir, so it is known to exist. + clone, err := newToolClone(dbDir, ".seidb-memiavl-tool-") if err != nil { - return "", fmt.Errorf("create temp dir under %s: %w", dbDir, err) + return nil, err } - cleanup := func(err error) (string, error) { - _ = os.RemoveAll(tempDir) - return "", err + cleanup := func(err error) (*toolClone, error) { + _ = clone.Remove() + return nil, err } srcSnapshotDir := filepath.Join(dbDir, snapshotName) - dstSnapshotDir := filepath.Join(tempDir, snapshotName) + dstSnapshotDir := filepath.Join(clone.dir, snapshotName) if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) } - if err := os.Symlink(snapshotName, filepath.Join(tempDir, "current")); err != nil { + if err := os.Symlink(snapshotName, filepath.Join(clone.dir, "current")); err != nil { return cleanup(fmt.Errorf("create current symlink: %w", err)) } + // The version parsed from the snapshot directory name is not enough to + // know which changelog version catchup resumes from: memiavl bootstraps + // every DB as snapshot-0 even when it was initialized with + // SetInitialVersion(N), in which case the first changelog entry is + // version N, not 1. Read the initial version from the cloned snapshot's + // metadata (immune to source pruning — the files are hard links) and + // derive the successor the same way memiavl itself does. + metadata, err := memiavl.ReadMetadata(dstSnapshotDir) + if err != nil { + return cleanup(fmt.Errorf("read cloned snapshot metadata: %w", err)) + } + if metadata.InitialVersion < 0 || metadata.InitialVersion > math.MaxUint32 { + return cleanup(fmt.Errorf("cloned snapshot has invalid initial version: %d", metadata.InitialVersion)) + } + firstNeeded := utils.NextVersion(snapshotVersion, uint32(metadata.InitialVersion)) + srcChangelogDir := filepath.Join(dbDir, "changelog") info, err := os.Stat(srcChangelogDir) if err != nil && !errors.Is(err, os.ErrNotExist) { @@ -123,18 +143,20 @@ func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) } if err == nil { - dstChangelogDir := filepath.Join(tempDir, "changelog") + dstChangelogDir := filepath.Join(clone.dir, "changelog") if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { return cleanup(fmt.Errorf("clone changelog: %w", err)) } // A live writer can roll a new snapshot between our snapshot clone and // our changelog copy, then prune the changelog up to that newer - // version — leaving a copy that no longer covers snapshotVersion+1 and - // a catchup that would silently skip versions. Retryable. - if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { + // version — leaving a copy that no longer covers the snapshot's + // successor and a catchup that would silently skip versions. Retryable. + sizeBefore := changelogByteSize(dstChangelogDir) + if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion, firstNeeded); err != nil { return cleanup(err) } + clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore } - return tempDir, nil + return clone, nil } diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index 6110b8e4f5..b2bae9ba90 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -131,6 +131,42 @@ func TestOpenMemiAVLReplayWorksWhileWriterHoldsLock(t *testing.T) { require.NoError(t, db.Close()) } +// TestOpenMemiAVLReplayAfterSetInitialVersion pins the non-default +// initial-height layout: memiavl bootstraps every DB as snapshot-0 even when +// SetInitialVersion(100) makes the first changelog entry version 100, so the +// clone's WAL-coverage check must derive the snapshot's successor from the +// snapshot metadata instead of assuming version 1 follows snapshot-0. This is +// exactly the shape of a freshly recovered chain whose genesis initial_height +// is greater than 1 and whose first snapshot rewrite has not happened yet; +// before the fix, replay mode failed deterministically on such nodes with +// "source kept churning". +func TestOpenMemiAVLReplayAfterSetInitialVersion(t *testing.T) { + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + require.NoError(t, store.SetInitialVersion(100)) + for i := 0; i < 3; i++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(i+1)), uint64(i+1))}}, + }})) + v, err := store.Commit() + require.NoError(t, err) + require.Equal(t, int64(100+i), v) + } + require.NoError(t, store.Close()) + dbDir := utils.GetCosmosSCStorePath(homeDir) + + historical, err := openMemiAVLReplay(dbDir, 101) + require.NoError(t, err, "coverage check must accept a snapshot-0 whose successor is the initial version") + require.Equal(t, int64(101), historical.Version()) + require.NoError(t, historical.Close()) + + latest, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err) + require.Equal(t, int64(102), latest.Version()) + require.NoError(t, latest.Close()) +} + // TestOpenMemiAVLReplayRejectsShortChangelog guards the failure mode this // design makes routine: repairing a torn tail inside the clone silently costs // the trailing version, and memiavl reports success anyway. For a tool whose diff --git a/sei-db/tools/cmd/seidb/operations/tool_clone.go b/sei-db/tools/cmd/seidb/operations/tool_clone.go new file mode 100644 index 0000000000..82010d948f --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/tool_clone.go @@ -0,0 +1,139 @@ +package operations + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +const ( + // toolCloneOwnerLockName is the flock-held marker file every tooling + // clone carries. While the creating process lives, the lock is held and + // the clone is off-limits; once the process dies (including SIGKILL, + // where no deferred cleanup runs), the kernel releases the flock and the + // next tool invocation can reap the directory. + toolCloneOwnerLockName = ".seidb-tool-owner.lock" + + // staleUnmarkedCloneAge guards the marker-less case: a crash in the + // window between MkdirTemp and the owner-lock creation, where liveness + // cannot be probed. Age is a poor liveness signal for marked clones + // (a mainnet-scale digest can legitimately run for hours), so it is + // used only when no marker exists at all. + staleUnmarkedCloneAge = 24 * time.Hour +) + +// toolClone is a private, disposable copy of a live store's snapshot + +// changelog that seidb tooling operates on instead of the live directory. +// It lives inside the source dbDir (to share its filesystem for hardlinks), +// which is exactly why leaking one is costly: its hardlinks pin snapshot +// inodes, so the live node's snapshot pruning frees no disk space until the +// clone is removed. +type toolClone struct { + dir string + ownerLock memiavl.FileLock + + // walRepaired records that validating the cloned changelog shrank it: + // the byte-copy caught the live writer mid-append and the WAL open + // repaired the torn tail inside the clone, costing the trailing record. + walRepaired bool +} + +// newToolClone reaps abandoned sibling clones, creates a fresh clone +// directory under dbDir with the given prefix, and marks it owned via flock +// before any expensive cloning starts. +// +// The prefix deliberately has no "-tmp" suffix: memiavl's removeTmpDirs +// deletes every "*-tmp" directory under its root when a node opens the DB +// read-write, and these clones must never be reaped by a process that cannot +// see whether the owning tool is still alive. +func newToolClone(dbDir, prefix string) (*toolClone, error) { + sweepStaleToolClones(dbDir, prefix) + + dir, err := os.MkdirTemp(dbDir, prefix+"*") + if err != nil { + return nil, fmt.Errorf("create temp dir under %s: %w", dbDir, err) + } + ownerLock, err := memiavl.LockFile(filepath.Join(dir, toolCloneOwnerLockName)) + if err != nil { + _ = os.RemoveAll(dir) + return nil, fmt.Errorf("acquire clone owner lock in %s: %w", dir, err) + } + return &toolClone{dir: dir, ownerLock: ownerLock}, nil +} + +// Remove releases the ownership lock and deletes the clone directory. +func (c *toolClone) Remove() error { + if c == nil { + return nil + } + if c.ownerLock != nil { + _ = c.ownerLock.Unlock() + _ = c.ownerLock.Destroy() + c.ownerLock = nil + } + if c.dir == "" { + return nil + } + if err := os.RemoveAll(c.dir); err != nil { + return fmt.Errorf("cleanup temp dir: %w", err) + } + c.dir = "" + return nil +} + +// sweepStaleToolClones removes abandoned clone directories under dbDir whose +// owner is provably gone: either the owner flock is acquirable (the creating +// process died), or no marker exists and the directory is old enough that the +// mkdir-to-lock window cannot explain it. Clones whose lock is still held — +// a concurrently running tool — are left alone. Best-effort by design: a +// failed sweep must never block the read the tool was invoked for. +func sweepStaleToolClones(dbDir, prefix string) { + entries, err := os.ReadDir(dbDir) + if err != nil { + return + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { + continue + } + dir := filepath.Join(dbDir, entry.Name()) + lockPath := filepath.Join(dir, toolCloneOwnerLockName) + if _, err := os.Stat(lockPath); errors.Is(err, os.ErrNotExist) { + if info, err := entry.Info(); err == nil && time.Since(info.ModTime()) > staleUnmarkedCloneAge { + _ = os.RemoveAll(dir) + } + continue + } + lock, err := memiavl.LockFile(lockPath) + if err != nil { + // Lock held (owner alive) or unreadable — leave the clone alone. + continue + } + _ = lock.Unlock() + _ = lock.Destroy() + _ = os.RemoveAll(dir) + } +} + +// changelogByteSize sums the sizes of the regular files in a cloned changelog +// directory. Comparing it before and after the WAL-coverage validation open +// detects a torn-tail repair inside the clone (the only mutation that open +// can perform), which callers surface as a warning for latest-height reads. +func changelogByteSize(dir string) int64 { + var total int64 + _ = filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return nil //nolint:nilerr // best-effort size probe + } + if info.Mode().IsRegular() { + total += info.Size() + } + return nil + }) + return total +} diff --git a/sei-db/tools/cmd/seidb/operations/tool_clone_test.go b/sei-db/tools/cmd/seidb/operations/tool_clone_test.go new file mode 100644 index 0000000000..5ddf6941fb --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/tool_clone_test.go @@ -0,0 +1,75 @@ +package operations + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +// TestSweepStaleToolClones pins the reaping policy for abandoned tooling +// clones: a clone whose owner lock is acquirable was orphaned by a dead +// process (SIGKILL runs no deferred cleanup) and must be removed, a clone +// whose lock is held belongs to a live tool and must survive, and a +// marker-less clone is removed only once it is old enough that the +// mkdir-to-lock window cannot explain it. +func TestSweepStaleToolClones(t *testing.T) { + dbDir := t.TempDir() + prefix := ".seidb-flatkv-tool-" + + staleUnmarked := filepath.Join(dbDir, prefix+"stale-unmarked") + require.NoError(t, os.Mkdir(staleUnmarked, 0o750)) + old := time.Now().Add(-2 * staleUnmarkedCloneAge) + require.NoError(t, os.Chtimes(staleUnmarked, old, old)) + + freshUnmarked := filepath.Join(dbDir, prefix+"fresh-unmarked") + require.NoError(t, os.Mkdir(freshUnmarked, 0o750)) + + held := filepath.Join(dbDir, prefix+"held") + require.NoError(t, os.Mkdir(held, 0o750)) + heldLock, err := memiavl.LockFile(filepath.Join(held, toolCloneOwnerLockName)) + require.NoError(t, err) + + orphaned := filepath.Join(dbDir, prefix+"orphaned") + require.NoError(t, os.Mkdir(orphaned, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(orphaned, toolCloneOwnerLockName), nil, 0o600)) + + otherPrefix := filepath.Join(dbDir, ".seidb-memiavl-tool-orphaned") + require.NoError(t, os.Mkdir(otherPrefix, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(otherPrefix, toolCloneOwnerLockName), nil, 0o600)) + + sweepStaleToolClones(dbDir, prefix) + + require.NoDirExists(t, staleUnmarked, "old marker-less clone must be reaped") + require.DirExists(t, freshUnmarked, "fresh marker-less clone may still be mid-creation") + require.DirExists(t, held, "a held owner lock proves the owning tool is alive") + require.NoDirExists(t, orphaned, "an acquirable owner lock proves the owner died") + require.DirExists(t, otherPrefix, "the sweep must not touch other clone families") + + require.NoError(t, heldLock.Unlock()) + sweepStaleToolClones(dbDir, prefix) + require.NoDirExists(t, held, "once the owner releases the lock the clone is reapable") +} + +// TestNewToolCloneOwnershipLifecycle checks that a live clone defends itself +// against a concurrent sweep and that Remove releases everything. +func TestNewToolCloneOwnershipLifecycle(t *testing.T) { + dbDir := t.TempDir() + prefix := ".seidb-memiavl-tool-" + + clone, err := newToolClone(dbDir, prefix) + require.NoError(t, err) + require.DirExists(t, clone.dir) + require.FileExists(t, filepath.Join(clone.dir, toolCloneOwnerLockName)) + + sweepStaleToolClones(dbDir, prefix) + require.DirExists(t, clone.dir, "an owned clone must survive a concurrent sweep") + + require.NoError(t, clone.Remove()) + require.NoDirExists(t, clone.dir) + require.NoError(t, clone.Remove(), "Remove must be idempotent") +} From 69281d76c61d3046f7ced6301767db13e0be0961 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 7 Aug 2026 10:46:32 -0400 Subject: [PATCH 3/6] fix(seidb): keep main's WAL-coverage check untouched in the merge The merge resolution had folded main's verifyClonedWALCovers and the branch's memiavl equivalent into a shared helper. Main is the shared truth for the FlatKV path, so restore its function verbatim and let the memiavl verifier stand on its own: the two backends read different WAL formats and cannot share an implementation anyway. Co-authored-by: Cursor --- .../tools/cmd/seidb/operations/flatkv_open.go | 40 ++++++------------- .../cmd/seidb/operations/memiavl_open.go | 23 ++++++++--- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index 7bc0eb8795..7b30691e51 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -232,7 +232,7 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error // snapshotVersion+1 here (unlike memiavl, whose bootstrap // snapshot-0 hides a configurable initial version). sizeBefore := changelogByteSize(dstChangelogDir) - if err := verifyClonedFlatKVWALCovers(dstChangelogDir, snapshotVersion); err != nil { + if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { return cleanup(err) } clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore @@ -241,19 +241,12 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error return clone, nil } -// verifyClonedFlatKVWALCovers inspects the cloned WAL just long enough to -// ensure it either is empty, ends at or before snapshotVersion (no replay -// needed), or starts at or before snapshotVersion+1 (catchup can resume -// cleanly). The state WAL is keyed by block number, so its stored range is -// directly the version range; GetRange reads it offline without a live WAL -// instance. -// -// FlatKV snapshots are always named with a real committed version — -// SetInitialVersion(N) seeds committedVersion N-1 and writes snapshot- — -// so the first version catchup needs is unconditionally snapshotVersion+1, -// unlike memiavl, whose bootstrap snapshot-0 hides a configurable initial -// version. -func verifyClonedFlatKVWALCovers(dstChangelogDir string, snapshotVersion int64) error { +// verifyClonedWALCovers inspects the cloned WAL just long enough to ensure it +// either is empty, ends at or before snapshotVersion (no replay needed), or +// starts at or before snapshotVersion+1 (catchup can resume cleanly). The state +// WAL is keyed by block number, so its stored range is directly the version +// range; GetRange reads it offline without a live WAL instance. +func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion int64) error { ok, firstVer, lastVer, err := statewal.GetRange(statewal.DefaultConfig(dstChangelogDir, flatkvStateWALName)) if err != nil { return fmt.Errorf("read cloned changelog range: %w", err) @@ -261,21 +254,14 @@ func verifyClonedFlatKVWALCovers(dstChangelogDir string, snapshotVersion int64) if !ok { return nil } - //nolint:gosec // version fits int64 - return checkClonedWALCoverage(int64(firstVer), int64(lastVer), snapshotVersion, snapshotVersion+1) -} - -// checkClonedWALCoverage is the shared verdict for both backends: an empty or -// fully-superseded WAL needs no replay, and otherwise the WAL must begin at or -// before firstNeeded — the first version catchup replays on top of the -// snapshot. A later start means the source truncated past our snapshot -// mid-clone, so the caller should retry with a freshly selected snapshot. -func checkClonedWALCoverage(firstVer, lastVer, snapshotVersion, firstNeeded int64) error { - if lastVer <= snapshotVersion || firstVer <= firstNeeded { + if int64(lastVer) <= snapshotVersion { //nolint:gosec // version fits int64 + return nil + } + if int64(firstVer) <= snapshotVersion+1 { //nolint:gosec // version fits int64 return nil } - return fmt.Errorf("%w: cloned WAL starts at version %d but catchup needs %d over snapshot %d (truncated past snapshot mid-clone)", - errSourceChurning, firstVer, firstNeeded, snapshotVersion) + return fmt.Errorf("%w: cloned WAL starts at version %d but snapshot is %d (truncated past snapshot mid-clone)", + errSourceChurning, firstVer, snapshotVersion) } func selectFlatKVSnapshot(dbDir string, height int64) (string, error) { diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open.go b/sei-db/tools/cmd/seidb/operations/memiavl_open.go index 0700036ba8..b63279cbcb 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open.go @@ -164,10 +164,15 @@ func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, erro } // verifyClonedMemIAVLWALCovers is the memiavl counterpart to -// verifyClonedFlatKVWALCovers. memiavl still stores its changelog in the -// offset-indexed changelog WAL, where the offset says nothing about the -// version, so the range has to be read by replaying the first and last entries -// rather than from sealed file names as the block-keyed state WAL allows. +// verifyClonedWALCovers: it ensures the cloned changelog either is empty, ends +// at or before snapshotVersion (no replay needed), or starts at or before +// firstNeeded (catchup can resume cleanly). +// +// It cannot share the FlatKV implementation. FlatKV moved to the block-keyed +// state WAL, whose stored range comes straight from sealed file names, while +// memiavl still keeps its changelog in the offset-indexed changelog WAL, where +// the offset says nothing about the version. The range therefore has to be read +// by replaying the first and last entries. func verifyClonedMemIAVLWALCovers(dstChangelogDir string, snapshotVersion, firstNeeded int64) error { walLog, err := wal.NewChangelogWAL(dstChangelogDir, wal.Config{}) if err != nil { @@ -195,7 +200,15 @@ func verifyClonedMemIAVLWALCovers(dstChangelogDir string, snapshotVersion, first if err != nil { return fmt.Errorf("read last cloned changelog entry: %w", err) } - return checkClonedWALCoverage(firstVer, lastVer, snapshotVersion, firstNeeded) + + if lastVer <= snapshotVersion { + return nil + } + if firstVer <= firstNeeded { + return nil + } + return fmt.Errorf("%w: cloned WAL starts at version %d but catchup needs %d over snapshot %d (truncated past snapshot mid-clone)", + errSourceChurning, firstVer, firstNeeded, snapshotVersion) } func readWALEntryVersion(walLog wal.ChangelogWAL, off uint64) (int64, error) { From 81d110a6e53d99f7ee5833ba37e26f6689a025b2 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 15:40:01 -0400 Subject: [PATCH 4/6] fix(seidb): fail closed on live memiavl WAL reads Replace replay clones with a non-mutating point-in-time WAL reader so transient tails return a retryable error without touching node data. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 96 ++++--- sei-db/state_db/sc/memiavl/db_test.go | 117 ++++++-- sei-db/state_db/sc/memiavl/multitree.go | 10 +- .../seidb/operations/evm_logical_digest.go | 68 +++-- .../tools/cmd/seidb/operations/flatkv_open.go | 131 ++++----- .../cmd/seidb/operations/flatkv_open_test.go | 34 +-- .../cmd/seidb/operations/memiavl_open.go | 221 ---------------- .../cmd/seidb/operations/memiavl_open_test.go | 194 +++----------- .../tools/cmd/seidb/operations/tool_clone.go | 139 ---------- .../cmd/seidb/operations/tool_clone_test.go | 75 ------ sei-db/wal/changelog.go | 16 ++ sei-db/wal/readonly.go | 250 ++++++++++++++++++ sei-db/wal/readonly_test.go | 148 +++++++++++ 13 files changed, 724 insertions(+), 775 deletions(-) delete mode 100644 sei-db/tools/cmd/seidb/operations/memiavl_open.go delete mode 100644 sei-db/tools/cmd/seidb/operations/tool_clone.go delete mode 100644 sei-db/tools/cmd/seidb/operations/tool_clone_test.go create mode 100644 sei-db/wal/readonly.go create mode 100644 sei-db/wal/readonly_test.go diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index a924036130..e7ff87d9dd 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -27,7 +27,21 @@ import ( const LockFileName = "LOCK" -var errReadOnly = errors.New("db is read-only") +var ( + errReadOnly = errors.New("db is read-only") + + // ErrReadOnlyWALCorrupt means a read-only open observed an incomplete, + // corrupt, or concurrently recovering changelog. The source WAL is left + // untouched; callers can retry after the writer finishes its current WAL + // operation. + ErrReadOnlyWALCorrupt = errors.New("read-only changelog is incomplete or corrupt") + + // ErrReadOnlyWALUnavailable means the immutable WAL view cannot replay + // every version from the selected snapshot through the requested target. + // The live writer may have pruned or advanced the changelog while the + // reader opened it; callers can retry against a new point-in-time view. + ErrReadOnlyWALUnavailable = errors.New("read-only changelog cannot reach the requested version") +) // DB implements DB-like functionalities on top of MultiTree: // - async snapshot rewriting @@ -201,12 +215,23 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { // Snapshot mmap files are loaded with MADV_RANDOM in OpenSnapshot(). - // MemIAVL owns changelog lifecycle: always open the WAL here. - // Even in read-only mode we may need WAL replay to reconstruct non-snapshot versions. - streamHandler, err := wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ - WriteBufferSize: opts.AsyncCommitBuffer, - }) + // MemIAVL owns changelog lifecycle: always open the WAL here. Read-only + // callers still need replay to reconstruct non-snapshot versions, but they + // must not use the writable opener: it repairs a torn tail by truncating it + // and completes interrupted WAL truncations by renaming or removing files. + var streamHandler wal.ChangelogWAL + if opts.ReadOnly { + streamHandler, err = wal.OpenReadOnlyChangelogWAL(utils.GetChangelogPath(opts.Dir)) + } else { + streamHandler, err = wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ + WriteBufferSize: opts.AsyncCommitBuffer, + }) + } if err != nil { + _ = mtree.Close() + if opts.ReadOnly && errors.Is(err, wal.ErrCorrupt) { + return nil, fmt.Errorf("%w; source WAL was not modified: %w", ErrReadOnlyWALCorrupt, err) + } return nil, fmt.Errorf("failed to open changelog WAL: %w", err) } @@ -215,21 +240,55 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { var walHasEntries bool walIndexDelta, walHasEntries, err = computeWALIndexDelta(streamHandler) if err != nil { + _ = streamHandler.Close() + _ = mtree.Close() return nil, fmt.Errorf("failed to compute WAL index delta: %w", err) } // If WAL is empty, set delta so first WAL entry aligns with NextVersion(). if !walHasEntries { walIndexDelta = mtree.WorkingCommitInfo().Version - 1 } + if opts.ReadOnly && walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { + firstIndex, firstErr := streamHandler.FirstOffset() + if firstErr != nil { + _ = streamHandler.Close() + _ = mtree.Close() + return nil, fmt.Errorf("read read-only changelog first offset: %w", firstErr) + } + if firstIndex > math.MaxInt64 { + _ = streamHandler.Close() + _ = mtree.Close() + return nil, fmt.Errorf("%w: first WAL offset %d overflows int64", ErrReadOnlyWALUnavailable, firstIndex) + } + firstVersion := int64(firstIndex) + walIndexDelta + firstNeeded := utils.NextVersion(mtree.Version(), mtree.initialVersion.Load()) + if firstVersion > firstNeeded { + snapshotVersion := mtree.Version() + _ = streamHandler.Close() + _ = mtree.Close() + return nil, fmt.Errorf("%w: selected snapshot version %d needs changelog version %d, "+ + "but the immutable WAL view starts at version %d", + ErrReadOnlyWALUnavailable, snapshotVersion, firstNeeded, firstVersion) + } + } // Replay WAL to catch up to target version (if WAL has entries) if walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { logger.Info("Start catching up and replaying the MemIAVL changelog file") if err := mtree.Catchup(context.Background(), streamHandler, walIndexDelta, targetVersion); err != nil { + _ = streamHandler.Close() + _ = mtree.Close() return nil, err } logger.Info("finished replay and caught up to target version", "version", targetVersion) } + if opts.ReadOnly && targetVersion > 0 && mtree.Version() != targetVersion { + reached := mtree.Version() + _ = streamHandler.Close() + _ = mtree.Close() + return nil, fmt.Errorf("%w: requested %d, reached %d", + ErrReadOnlyWALUnavailable, targetVersion, reached) + } if opts.LoadForOverwriting && targetVersion > 0 { currentSnapshot, err := os.Readlink(currentPath(opts.Dir)) @@ -1205,29 +1264,6 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return snapshotVersion, nil } -// SeekSnapshotName returns the directory name (relative to root — callers -// join it themselves) and version of the snapshot that OpenDB would start -// from for targetVersion: the "current" link when targetVersion is 0, -// otherwise the newest snapshot at or below it. -// -// Exported for readers that need to resolve a snapshot without opening the DB, -// so they inherit this package's layout rules instead of restating them. -func SeekSnapshotName(root string, targetVersion int64) (string, int64, error) { - if targetVersion == 0 { - version, err := currentVersion(root) - if err != nil { - return "", 0, fmt.Errorf("read current snapshot: %w", err) - } - return snapshotName(version), version, nil - } - - version, err := seekSnapshot(root, targetVersion) - if err != nil { - return "", 0, err - } - return snapshotName(version), version, nil -} - // GetEarliestVersion returns the earliest snapshot name in the db func GetEarliestVersion(root string) (int64, error) { var found int64 @@ -1350,7 +1386,7 @@ func isSnapshotName(name string) bool { // it's needed for upgrade module to check store upgrades, // it returns 0 if db doesn't exist or is empty. func GetLatestVersion(dir string) (int64, error) { - metadata, err := ReadMetadata(currentPath(dir)) + metadata, err := readMetadata(currentPath(dir)) if err != nil { if os.IsNotExist(err) { return 0, nil diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index d047b29b11..91146d47af 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "runtime/debug" + "sort" "strconv" "sync" "testing" @@ -1117,37 +1118,105 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) { require.Equal(t, "snapshot-1", target) } -// TestSeekSnapshotName pins the exported snapshot-resolution contract that -// external readers (seidb tooling) rely on: the returned name is relative to -// root, targetVersion 0 resolves through the current link, a positive target -// selects the newest snapshot at or below it, and a target older than the -// earliest snapshot reports pruning instead of guessing. -func TestSeekSnapshotName(t *testing.T) { - root := t.TempDir() - for _, v := range []int64{5, 10} { - require.NoError(t, os.Mkdir(filepath.Join(root, snapshotName(v)), 0o750)) +func TestReadOnlyOpenRejectsTornWALWithoutRepair(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) + require.NoError(t, err) + for i := 0; i < 3; i++ { + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[i], + }})) + _, err := db.Commit() + require.NoError(t, err) } - require.NoError(t, os.Symlink(snapshotName(10), currentPath(root))) + require.NoError(t, db.Close()) - name, version, err := SeekSnapshotName(root, 0) + segment := lastMemiAVLWALSegment(t, dir) + file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) + require.NoError(t, err) + _, err = file.Write([]byte{0x10}) require.NoError(t, err) - require.Equal(t, snapshotName(10), name) - require.Equal(t, int64(10), version) + require.NoError(t, file.Close()) + before, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + + _, err = OpenDB(0, Options{Dir: dir, ReadOnly: true}) + require.ErrorIs(t, err, ErrReadOnlyWALCorrupt) + after, readErr := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, readErr) + require.Equal(t, before, after, "read-only open must leave a torn live tail untouched") + + repaired, err := OpenDB(0, Options{Dir: dir}) + require.NoError(t, err, "the writable owner must retain the existing tail-repair behavior") + require.Equal(t, int64(3), repaired.Version()) + require.NoError(t, repaired.Close()) +} - name, version, err = SeekSnapshotName(root, 7) +func TestReadOnlyOpenRejectsWALGap(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) require.NoError(t, err) - require.Equal(t, snapshotName(5), name) - require.Equal(t, int64(5), version) + for i := 0; i < 3; i++ { + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[i], + }})) + _, err := db.Commit() + require.NoError(t, err) + } + require.NoError(t, db.GetWAL().TruncateBefore(2)) + require.NoError(t, db.Close()) - name, version, err = SeekSnapshotName(root, 10) + _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true}) + require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) + require.Contains(t, err.Error(), "needs changelog version 1") + require.Contains(t, err.Error(), "starts at version 2") +} + +func TestReadOnlyOpenRejectsShortWAL(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) require.NoError(t, err) - require.Equal(t, snapshotName(10), name) - require.Equal(t, int64(10), version) + for i := 0; i < 3; i++ { + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[i], + }})) + _, err := db.Commit() + require.NoError(t, err) + } + require.NoError(t, db.GetWAL().TruncateAfter(2)) + require.NoError(t, db.Close()) - _, _, err = SeekSnapshotName(root, 3) - require.Error(t, err) - require.Contains(t, err.Error(), "target version is pruned") + _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true}) + require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) + require.Contains(t, err.Error(), "requested 3, reached 2") +} - _, _, err = SeekSnapshotName(filepath.Join(root, "does-not-exist"), 0) - require.Error(t, err) +func lastMemiAVLWALSegment(t *testing.T, dir string) string { + t.Helper() + entries, err := os.ReadDir(utils.GetChangelogPath(dir)) + require.NoError(t, err) + var names []string + for _, entry := range entries { + if !entry.IsDir() && len(entry.Name()) == 20 { + names = append(names, entry.Name()) + } + } + require.NotEmpty(t, names) + sort.Strings(names) + return filepath.Join(utils.GetChangelogPath(dir), names[len(names)-1]) } diff --git a/sei-db/state_db/sc/memiavl/multitree.go b/sei-db/state_db/sc/memiavl/multitree.go index f0f85e417a..6e9101e339 100644 --- a/sei-db/state_db/sc/memiavl/multitree.go +++ b/sei-db/state_db/sc/memiavl/multitree.go @@ -76,7 +76,7 @@ func NewEmptyMultiTree(initialVersion uint32) *MultiTree { func LoadMultiTree(ctx context.Context, dir string, opts Options) (*MultiTree, error) { startTime := time.Now() - metadata, err := ReadMetadata(dir) + metadata, err := readMetadata(dir) if err != nil { return nil, err } @@ -600,13 +600,7 @@ func (t *MultiTree) ReplaceWith(other *MultiTree) error { return errors.Join(errs...) } -// ReadMetadata loads a snapshot directory's MultiTreeMetadata (commit info + -// initial version). Exported for readers that need snapshot metadata without -// opening the DB — e.g. seidb tooling, whose changelog-coverage check must -// know the initial version because a DB bootstrapped with initial version N -// keeps its first snapshot named snapshot-0 while the first changelog entry -// is version N, not 1. -func ReadMetadata(dir string) (*proto.MultiTreeMetadata, error) { +func readMetadata(dir string) (*proto.MultiTreeMetadata, error) { // load commit info bz, err := os.ReadFile(filepath.Join(filepath.Clean(dir), MetadataFileName)) if err != nil { diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 143acc1e50..49a3693907 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -81,14 +81,14 @@ const ( // that exact height (or --height 0 for the current symlink). This is the // preferred mode whenever the target height lines up with an existing // snapshot boundary. -// - replay (SLOW): clones the newest snapshot at or below --height plus the -// changelog into a temp directory, replays the changelog up to --height in -// that clone, then walks the in-memory/mmap tree. Roughly an order of -// magnitude slower than snapshot (changelog replay + per-leaf tree walk -// instead of a sequential file read) and it byte-copies the changelog, so -// it needs free space alongside the source. Use it only when no snapshot -// exists at the target height — e.g. nodes whose snapshot rewrite lags the -// tip, so an arbitrary comparison height has no snapshot- on disk. +// - replay (SLOW): opens a non-mutating read-only WAL view, replays the +// changelog up to --height, then walks the in-memory/mmap tree. Roughly an +// order of magnitude slower than snapshot (changelog replay + per-leaf tree +// walk instead of a sequential file read). If a live writer leaves a torn +// tail in view, replay fails and asks the operator to rerun instead of +// repairing the source WAL. Use it only when no snapshot exists at the +// target height — e.g. nodes whose snapshot rewrite lags the tip, so an +// arbitrary comparison height has no snapshot- on disk. // // The flatkv side is always a pebble WAL-replay-to-height and is fast // regardless. So when comparing across nodes, pick a height that is an existing @@ -166,8 +166,8 @@ func EvmLogicalDigestCmd() *cobra.Command { cmd.Flags().StringP("db-dir", "d", "", "For flatkv: the flatkv data dir. For memiavl: the memiavl root dir (contains current/ and snapshot-* )") cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir") cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )") - cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink). On a live node 0 = latest is best-effort — the printed version line records what was actually digested; always pass an explicit common height when comparing nodes") - cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: clones the snapshot + changelog to a temp dir, replays to --height there, then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") + cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink)") + cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: replays changelog to --height then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") cmd.Flags().String("memiavl-normalization", memiavlNormSemantic, "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|misc) instead of printing the global digest") cmd.Flags().Int("key-offset", 0, "Inspect mode: byte offset into physical key before applying --key-prefix / sharding") @@ -467,7 +467,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT requestedHeight: height, version: opened.Version(), } - var memReplayDB *openedMemIAVL + var memReplayDB *memiavl.DB var memEvmSnapshotDir string var memVersion int64 switch memiavlOpenMode { @@ -483,13 +483,13 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl snapshot=%s", opened.Version(), memEvmSnapshotDir) ctx.normalization = fmt.Sprintf("flatkv rows plus memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) case memiavlOpenModeReplay: - memReplayDB, err = openMemiAVLReplay(memIAVLDir, height) + memReplayDB, err = openMemiAVLReplayReadOnly(memIAVLDir, height) if err != nil { return err } defer func() { _ = memReplayDB.Close() }() memVersion = memReplayDB.Version() - ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl clone replay dir=%s", opened.Version(), memIAVLDir) + ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl read-only replay dir=%s", opened.Version(), memIAVLDir) ctx.normalization = fmt.Sprintf("flatkv rows plus replayed memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) default: return fmt.Errorf("unknown --memiavl-open-mode %q (want snapshot|replay)", memiavlOpenMode) @@ -506,7 +506,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT if boundary.Status() != migration.MigrationComplete { if memReplayDB != nil { if err := consumeCompositeMemiavl(func(fn func(rawKey, rawVal []byte) error) error { - return scanMemiavlReplayEVMLeaves(memReplayDB.DB, fn) + return scanMemiavlReplayEVMLeaves(memReplayDB, fn) }, "memiavl-replay", boundary, &d, accounts); err != nil { return err } @@ -1106,8 +1106,38 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization } } +func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { + db, err := memiavl.OpenDB(height, memiavl.Options{ + Dir: dbDir, + ReadOnly: true, + ZeroCopy: true, + }) + if err != nil { + if errors.Is(err, memiavl.ErrReadOnlyWALCorrupt) { + return nil, fmt.Errorf("memiavl changelog tail is incomplete, corrupt, or changing; "+ + "live WAL was not modified; rerun the command, and if the error persists after stopping seid, "+ + "repair the WAL offline: %w", err) + } + if errors.Is(err, memiavl.ErrReadOnlyWALUnavailable) { + return nil, fmt.Errorf("the immutable memiavl changelog view could not reach height %d; "+ + "live WAL was not modified; rerun the command: %w", height, err) + } + return nil, fmt.Errorf("open memiavl read-only replay: %w", err) + } + if height > 0 && db.Version() != height { + versionErr := fmt.Errorf("memiavl replay version mismatch: requested %d, reached %d; "+ + "the live changelog did not provide a complete path to the target; rerun the command", + height, db.Version()) + if closeErr := db.Close(); closeErr != nil { + return nil, errors.Join(versionErr, fmt.Errorf("close memiavl read-only replay: %w", closeErr)) + } + return nil, versionErr + } + return db, nil +} + func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error { - db, err := openMemiAVLReplay(dbDir, height) + db, err := openMemiAVLReplayReadOnly(dbDir, height) if err != nil { return err } @@ -1115,9 +1145,9 @@ func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normaliz switch normalization { case "", memiavlNormSemantic, memiavlNormIndependent: - return digestMemIAVLReplaySemantic(dbDir, height, db.DB, findTarget) + return digestMemIAVLReplaySemantic(dbDir, height, db, findTarget) case memiavlNormTranslator: - return digestMemIAVLReplayTranslator(dbDir, height, db.DB, findTarget) + return digestMemIAVLReplayTranslator(dbDir, height, db, findTarget) default: return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization) } @@ -1281,7 +1311,7 @@ func digestMemIAVLReplaySemantic(dbDir string, height int64, db *memiavl.DB, fin backend: "memiavl", mode: "semantic-replay", dbDir: dbDir, - source: "isolated memiavl clone opened from snapshot + changelog replay", + source: "read-only memiavl DB opened from snapshot + changelog replay", normalization: "independent semantic decoder for replayed memiavl EVM keys; does not call flatkv.ImportTranslator", requestedHeight: height, version: db.Version(), @@ -1298,7 +1328,7 @@ func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, f backend: "memiavl", mode: "translator-replay", dbDir: dbDir, - source: "isolated memiavl clone opened from snapshot + changelog replay", + source: "read-only memiavl DB opened from snapshot + changelog replay", normalization: "replayed memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload", requestedHeight: height, version: db.Version(), diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index 7b30691e51..60c53c8f18 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -32,10 +32,11 @@ const ( maxCloneRetries = 3 ) -// errSourceChurning marks transient races where the source directory mutates -// (snapshot pruned, WAL truncated) between our reads. It is the sentinel that -// retryToolingClone uses to decide whether to retry instead of bailing out. -var errSourceChurning = errors.New("source kept churning during clone") +// errSourceChurning marks transient races where the source FlatKV directory +// mutates (snapshot pruned, WAL truncated) between our reads. It is the +// sentinel that prepareFlatKVToolingCloneWith uses to decide whether to +// retry instead of bailing out. +var errSourceChurning = errors.New("flatkv source kept churning during clone") // openedFlatKV wraps a temp-cloned FlatKV store used by tooling. // @@ -43,7 +44,7 @@ var errSourceChurning = errors.New("source kept churning during clone") // WAL so they do not compete with a live node for the FlatKV writer lock. type openedFlatKV struct { flatkv.Store - clone *toolClone + tempDir string } func (o *openedFlatKV) Close() error { @@ -51,11 +52,13 @@ func (o *openedFlatKV) Close() error { if o.Store != nil { err = o.Store.Close() } - if rmErr := o.clone.Remove(); rmErr != nil { - if err != nil { - return fmt.Errorf("%w; %w", err, rmErr) + if o.tempDir != "" { + if rmErr := os.RemoveAll(o.tempDir); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; cleanup temp dir: %w", err, rmErr) + } + return fmt.Errorf("cleanup temp dir: %w", rmErr) } - return rmErr } return err } @@ -79,30 +82,25 @@ func (o *openedFlatKV) Close() error { // os.ReadDir and os.Link calls, we surface ENOENT, re-select the // snapshot, and retry up to maxCloneRetries times. // -// height=0 means the latest version, best-effort on a live node: the clone is -// a consistent committed prefix as of the copy instant, and a torn-tail -// repair inside the clone (surfaced as a warning) can land it one version -// behind the source tip. Tools print the version actually opened; treat that -// line as authoritative when comparing across nodes. +// height=0 means latest version. func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { - clone, err := prepareFlatKVToolingClone(dbDir, height) + tempDir, err := prepareFlatKVToolingClone(dbDir, height) if err != nil { return nil, err } - warnIfCloneRepaired(clone, "flatkv", height) cfg := config.DefaultConfig() - cfg.DataDir = clone.dir + cfg.DataDir = tempDir stateWAL, err := flatkv.OpenStateWAL(cfg) if err != nil { - _ = clone.Remove() + _ = os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err) } primary, err := flatkv.NewCommitStore(context.Background(), cfg, stateWAL) if err != nil { _ = stateWAL.Close() - _ = clone.Remove() + _ = os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to create FlatKV store: %w", err) } @@ -112,55 +110,38 @@ func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { view, err := primary.LoadVersionReadOnly(height) if err != nil { _ = primary.Close() - _ = clone.Remove() + _ = os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to open FlatKV at version %d: %w", height, err) } if err := primary.Close(); err != nil { _ = view.Close() - _ = clone.Remove() + _ = os.RemoveAll(tempDir) return nil, fmt.Errorf("failed to close FlatKV clone writer: %w", err) } return &openedFlatKV{ - Store: view, - clone: clone, + Store: view, + tempDir: tempDir, }, nil } -// warnIfCloneRepaired tells the operator that the cloned changelog had a torn -// tail (the byte-copy raced the live writer mid-append) and was repaired -// inside the clone. For an explicit --height the reached-version checks catch -// any resulting shortfall; for height 0 ("latest") there is no target to -// check against, so the printed version line is the only record of what was -// actually digested. -func warnIfCloneRepaired(clone *toolClone, backend string, height int64) { - if !clone.walRepaired || height != 0 { - return - } - fmt.Fprintf(os.Stderr, "warning: cloned %s changelog had a torn tail (live writer mid-append) and was repaired in the clone; "+ - "the opened version may trail the source tip by one — trust the printed version line\n", backend) -} - -func prepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error) { - return retryToolingClone(dbDir, height, tryPrepareFlatKVToolingClone) +func prepareFlatKVToolingClone(dbDir string, height int64) (string, error) { + return prepareFlatKVToolingCloneWith(dbDir, height, tryPrepareFlatKVToolingClone) } -// retryToolingClone runs tryClone, retrying while the live writer keeps -// mutating the source out from under us. Shared by the FlatKV and memiavl -// tooling clones, which race the same writer in the same ways. -func retryToolingClone(dbDir string, height int64, tryClone func(string, int64) (*toolClone, error)) (*toolClone, error) { +func prepareFlatKVToolingCloneWith(dbDir string, height int64, tryClone func(string, int64) (string, error)) (string, error) { var lastErr error for attempt := 0; attempt < maxCloneRetries; attempt++ { - clone, err := tryClone(dbDir, height) + tempDir, err := tryClone(dbDir, height) if err == nil { - return clone, nil + return tempDir, nil } if !isCloneRetryableError(err) { - return nil, err + return "", err } lastErr = err } - return nil, fmt.Errorf("clone aborted after %d retries, source kept churning: %w", maxCloneRetries, lastErr) + return "", fmt.Errorf("clone aborted after %d retries, source kept churning: %w", maxCloneRetries, lastErr) } // isCloneRetryableError reports whether err indicates a transient race with @@ -171,37 +152,39 @@ func isCloneRetryableError(err error) bool { return errors.Is(err, os.ErrNotExist) || errors.Is(err, errSourceChurning) } -func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error) { +func tryPrepareFlatKVToolingClone(dbDir string, height int64) (string, error) { snapshotName, err := selectFlatKVSnapshot(dbDir, height) if err != nil { - return nil, err + return "", err } snapshotVersion, err := strconv.ParseInt(snapshotName[len(flatkvSnapshotPrefix):], 10, 64) if err != nil { - return nil, fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) + return "", fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) } - // The clone must sit inside dbDir so it is on the exact same mounted - // filesystem as the source snapshots (dbDir is often its own mount - // point, so a sibling directory is not enough and hardlinks would fail - // across the boundary). selectFlatKVSnapshot already read dbDir, so it - // is known to exist. - clone, err := newToolClone(dbDir, ".seidb-flatkv-tool-") + // Place the temp clone inside dbDir so it is on the exact same mounted + // filesystem as the source snapshots. A sibling directory is not enough: + // dbDir itself is often a mount point on dedicated data volumes. + cloneRoot := dbDir + if err := os.MkdirAll(cloneRoot, 0o750); err != nil { + return "", fmt.Errorf("ensure clone root %s: %w", cloneRoot, err) + } + tempDir, err := os.MkdirTemp(cloneRoot, ".seidb-flatkv-tool-*") if err != nil { - return nil, err + return "", fmt.Errorf("create temp dir under %s: %w", cloneRoot, err) } - cleanup := func(err error) (*toolClone, error) { - _ = clone.Remove() - return nil, err + cleanup := func(err error) (string, error) { + _ = os.RemoveAll(tempDir) + return "", err } srcSnapshotDir := filepath.Join(dbDir, snapshotName) - dstSnapshotDir := filepath.Join(clone.dir, snapshotName) + dstSnapshotDir := filepath.Join(tempDir, snapshotName) if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) } - if err := os.Symlink(snapshotName, filepath.Join(clone.dir, "current")); err != nil { + if err := os.Symlink(snapshotName, filepath.Join(tempDir, "current")); err != nil { return cleanup(fmt.Errorf("create current symlink: %w", err)) } @@ -214,31 +197,23 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) } if err == nil { - dstChangelogDir := filepath.Join(clone.dir, "changelog") + dstChangelogDir := filepath.Join(tempDir, "changelog") if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { return cleanup(fmt.Errorf("clone changelog: %w", err)) } // Detect the snapshot/WAL race: a live writer can roll a new // snapshot between our snapshot clone and our changelog copy and // then truncateWAL up to that newer snapshot's version. If that - // happened, the cloned WAL no longer covers the snapshot's - // successor version, and a downstream catchup would silently jump - // over missing versions. Surface it as a retryable error so the - // outer loop re-selects the snapshot and tries again. - // - // FlatKV snapshots are always named with a real committed version — - // SetInitialVersion(N) seeds committedVersion N-1 and writes - // snapshot- — so the successor is unconditionally - // snapshotVersion+1 here (unlike memiavl, whose bootstrap - // snapshot-0 hides a configurable initial version). - sizeBefore := changelogByteSize(dstChangelogDir) + // happened, the cloned WAL no longer covers snapshotVersion+1, + // and a downstream catchup would silently jump over missing + // versions. Surface it as a retryable error so the outer loop + // re-selects the snapshot and tries again. if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { return cleanup(err) } - clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore } - return clone, nil + return tempDir, nil } // verifyClonedWALCovers inspects the cloned WAL just long enough to ensure it @@ -318,7 +293,7 @@ func isFlatKVSnapshotName(name string) bool { // error: snapshots can be many GB, and the previous behavior of falling back // to a byte-copy on tmpfs (the historical $TMPDIR default) routinely OOM'd // nodes and exhausted /tmp. Callers must ensure the tool clone dir lives on -// the same filesystem as the source directory. +// the same filesystem as the source FlatKV directory. // // Hardlinking is safe because: // - snapshot-N files are immutable after Pebble Checkpoint + Rename. @@ -384,7 +359,7 @@ func linkOnly(src, dst string) error { if err := os.Link(src, dst); err != nil { if isCrossDeviceLinkError(err) { return fmt.Errorf("hardlink %s -> %s failed across filesystems; "+ - "seidb tooling requires the temp clone to share a filesystem with the source: %w", + "FlatKV tooling requires the temp clone to share a filesystem with the source: %w", src, dst, err) } return err diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go index 59b0274951..ecd04238ee 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go @@ -55,17 +55,17 @@ func TestPrepareFlatKVToolingCloneHardlinksSnapshotAndCopiesChangelog(t *testing srcChangelogFile := filepath.Join(dbDir, "changelog", "000001.log") require.NoError(t, os.WriteFile(srcChangelogFile, []byte("wal-data"), 0o600)) - clone, err := prepareFlatKVToolingClone(dbDir, 0) + cloneDir, err := prepareFlatKVToolingClone(dbDir, 0) require.NoError(t, err) - defer clone.Remove() //nolint:errcheck // test cleanup + defer os.RemoveAll(cloneDir) //nolint:errcheck // test cleanup - target, err := os.Readlink(filepath.Join(clone.dir, "current")) + target, err := os.Readlink(filepath.Join(cloneDir, "current")) require.NoError(t, err) require.Equal(t, snapshot, target) - require.FileExists(t, filepath.Join(clone.dir, snapshot, "account", "000001.sst")) - require.NoFileExists(t, filepath.Join(clone.dir, snapshot, "LOCK")) - dstSnapshotFile := filepath.Join(clone.dir, snapshot, "account", "000001.sst") - dstChangelogFile := filepath.Join(clone.dir, "changelog", "000001.log") + require.FileExists(t, filepath.Join(cloneDir, snapshot, "account", "000001.sst")) + require.NoFileExists(t, filepath.Join(cloneDir, snapshot, "LOCK")) + dstSnapshotFile := filepath.Join(cloneDir, snapshot, "account", "000001.sst") + dstChangelogFile := filepath.Join(cloneDir, "changelog", "000001.log") require.FileExists(t, dstChangelogFile) srcSnapshotInfo, err := os.Stat(srcSnapshotFile) @@ -123,21 +123,21 @@ func TestPrepareFlatKVToolingCloneMissingCurrentAndSnapshot(t *testing.T) { func TestPrepareFlatKVToolingCloneRetriesENOENT(t *testing.T) { var attempts int - clone, err := retryToolingClone(t.TempDir(), 0, func(string, int64) (*toolClone, error) { + cloneDir, err := prepareFlatKVToolingCloneWith(t.TempDir(), 0, func(string, int64) (string, error) { attempts++ if attempts < maxCloneRetries { - return nil, fmt.Errorf("source vanished: %w", os.ErrNotExist) + return "", fmt.Errorf("source vanished: %w", os.ErrNotExist) } - return &toolClone{dir: t.TempDir()}, nil + return t.TempDir(), nil }) require.NoError(t, err) - require.NotEmpty(t, clone.dir) + require.NotEmpty(t, cloneDir) require.Equal(t, maxCloneRetries, attempts) attempts = 0 - _, err = retryToolingClone(t.TempDir(), 0, func(string, int64) (*toolClone, error) { + _, err = prepareFlatKVToolingCloneWith(t.TempDir(), 0, func(string, int64) (string, error) { attempts++ - return nil, errors.New("permission denied") + return "", errors.New("permission denied") }) require.Error(t, err) require.Equal(t, 1, attempts) @@ -160,15 +160,15 @@ func TestPrepareFlatKVToolingClonePlacesTempDirInsideDBDir(t *testing.T) { require.NoError(t, store.WriteSnapshot("")) require.NoError(t, store.Close()) - clone, err := prepareFlatKVToolingClone(dbDir, 0) + cloneDir, err := prepareFlatKVToolingClone(dbDir, 0) require.NoError(t, err) - defer clone.Remove() //nolint:errcheck // test cleanup + defer os.RemoveAll(cloneDir) //nolint:errcheck // test cleanup - rel, err := filepath.Rel(dbDir, clone.dir) + rel, err := filepath.Rel(dbDir, cloneDir) require.NoError(t, err) require.NotEqual(t, ".", rel) require.False(t, strings.HasPrefix(rel, ".."), "tooling clone must be created inside dbDir to stay on dbDir's mounted filesystem") - require.Contains(t, filepath.Base(clone.dir), ".seidb-flatkv-tool-") + require.Contains(t, filepath.Base(cloneDir), ".seidb-flatkv-tool-") } // TestPrepareFlatKVToolingCloneDetectsWALTruncationRace simulates the audited diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open.go b/sei-db/tools/cmd/seidb/operations/memiavl_open.go deleted file mode 100644 index b63279cbcb..0000000000 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open.go +++ /dev/null @@ -1,221 +0,0 @@ -package operations - -import ( - "errors" - "fmt" - "math" - "os" - "path/filepath" - - "github.com/sei-protocol/sei-chain/sei-db/common/utils" - "github.com/sei-protocol/sei-chain/sei-db/proto" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" - "github.com/sei-protocol/sei-chain/sei-db/wal" -) - -// openedMemIAVL is a memiavl DB opened against a temp clone of the source. -type openedMemIAVL struct { - *memiavl.DB - clone *toolClone -} - -func (o *openedMemIAVL) Close() error { - var err error - if o.DB != nil { - err = o.DB.Close() - } - if rmErr := o.clone.Remove(); rmErr != nil { - if err != nil { - return fmt.Errorf("%w; %w", err, rmErr) - } - return rmErr - } - return err -} - -// openMemiAVLReplay opens memiavl at the given height (0 means latest) by -// replaying the changelog on top of the newest snapshot at or below it. -// -// It clones the source rather than using memiavl's ReadOnly option, which is -// weaker than it reads: ReadOnly skips the LOCK file but the changelog is still -// opened read-write, and that open "repairs" a torn tail record by truncating -// the segment. On a live node a torn tail is the writer mid-append, not -// corruption, so a read-only replay could truncate committed versions out from -// under a running seid. Repairing a torn tail in a private copy is harmless. -// This mirrors openFlatKVReadOnly. -// -// height=0 ("latest") is best-effort on a live node: the clone is a -// consistent committed prefix as of the copy instant, and a torn-tail repair -// inside the clone (surfaced as a warning) can land it one version behind the -// source tip. There is no target version to check against, so the report's -// printed version line is the authoritative record of what was digested; -// cross-node comparisons should always use an explicit common --height. -func openMemiAVLReplay(dbDir string, height int64) (*openedMemIAVL, error) { - clone, err := prepareMemIAVLToolingClone(dbDir, height) - if err != nil { - return nil, err - } - warnIfCloneRepaired(clone, "memiavl", height) - - db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: clone.dir, - ZeroCopy: true, - }) - if err != nil { - _ = clone.Remove() - return nil, fmt.Errorf("open memiavl clone at version %d: %w", height, err) - } - opened := &openedMemIAVL{DB: db, clone: clone} - - // memiavl replays whatever the changelog holds and reports success even - // when that falls short of the requested height. Every tail repair inside - // the clone costs the trailing version, and a digest computed one version - // early is indistinguishable from real divergence when comparing nodes. - if reached := db.Version(); height > 0 && reached != height { - err := fmt.Errorf("memiavl clone version mismatch: requested %d, reached %d "+ - "(changelog does not cover the target height)", height, reached) - if closeErr := opened.Close(); closeErr != nil { - return nil, errors.Join(err, fmt.Errorf("close clone: %w", closeErr)) - } - return nil, err - } - - return opened, nil -} - -func prepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, error) { - return retryToolingClone(dbDir, height, tryPrepareMemIAVLToolingClone) -} - -// tryPrepareMemIAVLToolingClone mirrors tryPrepareFlatKVToolingClone: the two -// layouts are the same shape (current -> snapshot-N/, changelog/, LOCK) and both -// publish snapshots by rename and drop them wholesale, so the same -// hardlink-the-snapshot / byte-copy-the-changelog split applies. -func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, error) { - snapshotName, snapshotVersion, err := memiavl.SeekSnapshotName(dbDir, height) - if err != nil { - return nil, err - } - - // The clone must sit inside dbDir to share a filesystem with the source - // snapshot: dbDir is often its own mount point, so a sibling directory is - // not enough and hardlinks would fail across the boundary. - // SeekSnapshotName already read dbDir, so it is known to exist. - clone, err := newToolClone(dbDir, ".seidb-memiavl-tool-") - if err != nil { - return nil, err - } - cleanup := func(err error) (*toolClone, error) { - _ = clone.Remove() - return nil, err - } - - srcSnapshotDir := filepath.Join(dbDir, snapshotName) - dstSnapshotDir := filepath.Join(clone.dir, snapshotName) - if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { - return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) - } - - if err := os.Symlink(snapshotName, filepath.Join(clone.dir, "current")); err != nil { - return cleanup(fmt.Errorf("create current symlink: %w", err)) - } - - // The version parsed from the snapshot directory name is not enough to - // know which changelog version catchup resumes from: memiavl bootstraps - // every DB as snapshot-0 even when it was initialized with - // SetInitialVersion(N), in which case the first changelog entry is - // version N, not 1. Read the initial version from the cloned snapshot's - // metadata (immune to source pruning — the files are hard links) and - // derive the successor the same way memiavl itself does. - metadata, err := memiavl.ReadMetadata(dstSnapshotDir) - if err != nil { - return cleanup(fmt.Errorf("read cloned snapshot metadata: %w", err)) - } - if metadata.InitialVersion < 0 || metadata.InitialVersion > math.MaxUint32 { - return cleanup(fmt.Errorf("cloned snapshot has invalid initial version: %d", metadata.InitialVersion)) - } - firstNeeded := utils.NextVersion(snapshotVersion, uint32(metadata.InitialVersion)) - - srcChangelogDir := filepath.Join(dbDir, "changelog") - info, err := os.Stat(srcChangelogDir) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return cleanup(fmt.Errorf("stat changelog: %w", err)) - } - if err == nil && !info.IsDir() { - return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) - } - if err == nil { - dstChangelogDir := filepath.Join(clone.dir, "changelog") - if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { - return cleanup(fmt.Errorf("clone changelog: %w", err)) - } - // A live writer can roll a new snapshot between our snapshot clone and - // our changelog copy, then prune the changelog up to that newer - // version — leaving a copy that no longer covers the snapshot's - // successor and a catchup that would silently skip versions. Retryable. - sizeBefore := changelogByteSize(dstChangelogDir) - if err := verifyClonedMemIAVLWALCovers(dstChangelogDir, snapshotVersion, firstNeeded); err != nil { - return cleanup(err) - } - clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore - } - - return clone, nil -} - -// verifyClonedMemIAVLWALCovers is the memiavl counterpart to -// verifyClonedWALCovers: it ensures the cloned changelog either is empty, ends -// at or before snapshotVersion (no replay needed), or starts at or before -// firstNeeded (catchup can resume cleanly). -// -// It cannot share the FlatKV implementation. FlatKV moved to the block-keyed -// state WAL, whose stored range comes straight from sealed file names, while -// memiavl still keeps its changelog in the offset-indexed changelog WAL, where -// the offset says nothing about the version. The range therefore has to be read -// by replaying the first and last entries. -func verifyClonedMemIAVLWALCovers(dstChangelogDir string, snapshotVersion, firstNeeded int64) error { - walLog, err := wal.NewChangelogWAL(dstChangelogDir, wal.Config{}) - if err != nil { - return fmt.Errorf("open cloned changelog for validation: %w", err) - } - defer func() { _ = walLog.Close() }() - - firstOff, err := walLog.FirstOffset() - if err != nil { - return fmt.Errorf("cloned changelog first offset: %w", err) - } - lastOff, err := walLog.LastOffset() - if err != nil { - return fmt.Errorf("cloned changelog last offset: %w", err) - } - if firstOff == 0 || lastOff == 0 || firstOff > lastOff { - return nil - } - - firstVer, err := readWALEntryVersion(walLog, firstOff) - if err != nil { - return fmt.Errorf("read first cloned changelog entry: %w", err) - } - lastVer, err := readWALEntryVersion(walLog, lastOff) - if err != nil { - return fmt.Errorf("read last cloned changelog entry: %w", err) - } - - if lastVer <= snapshotVersion { - return nil - } - if firstVer <= firstNeeded { - return nil - } - return fmt.Errorf("%w: cloned WAL starts at version %d but catchup needs %d over snapshot %d (truncated past snapshot mid-clone)", - errSourceChurning, firstVer, firstNeeded, snapshotVersion) -} - -func readWALEntryVersion(walLog wal.ChangelogWAL, off uint64) (int64, error) { - var ver int64 - err := walLog.Replay(off, off, func(_ uint64, entry proto.ChangelogEntry) error { - ver = entry.Version - return nil - }) - return ver, err -} diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index b2bae9ba90..ad06579c35 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -13,183 +13,49 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/proto" ) -// newMemiavlSourceDir builds a memiavl directory with `versions` committed -// blocks and returns the directory a tool would be pointed at. -func newMemiavlSourceDir(t *testing.T, versions int) string { - t.Helper() +func TestOpenMemiAVLReplayReadOnlyReportsRetryWithoutRepair(t *testing.T) { homeDir := t.TempDir() store := newTestMemiavlStore(t, homeDir) - for i := 1; i <= versions; i++ { - require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: keys.EVMStoreKey, - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(i)), uint64(i))}}, - }})) - v, err := store.Commit() - require.NoError(t, err) - require.Equal(t, int64(i), v) - } - require.NoError(t, store.Close()) - return utils.GetCosmosSCStorePath(homeDir) -} - -// snapshotDirState records every regular file under root by relative path and -// content so a later comparison catches truncation, appends, and deletions. -func snapshotDirState(t *testing.T, root string) map[string][]byte { - t.Helper() - state := make(map[string][]byte) - require.NoError(t, filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.Mode().IsRegular() { - return nil - } - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - bz, err := os.ReadFile(filepath.Clean(path)) - if err != nil { - return err - } - state[rel] = bz - return nil - })) - return state -} - -// lastChangelogSegment returns the newest tidwall segment file in the memiavl -// changelog. Segment names are fixed-width, so lexical order is index order. -func lastChangelogSegment(t *testing.T, dbDir string) string { - t.Helper() - changelogDir := filepath.Join(dbDir, "changelog") - entries, err := os.ReadDir(changelogDir) - require.NoError(t, err) - var names []string - for _, e := range entries { - if !e.IsDir() && len(e.Name()) >= 20 { - names = append(names, e.Name()) - } - } - require.NotEmpty(t, names, "memiavl changelog should have at least one segment") - sort.Strings(names) - return filepath.Join(changelogDir, names[len(names)-1]) -} - -// TestOpenMemiAVLReplayLeavesSourceUntouched is the regression test for the -// audited hazard: a replay that advertises itself as read-only used to hand the -// live changelog straight to memiavl, whose WAL open truncates a torn tail. A -// torn tail on a running node is just the writer mid-append, so the "repair" -// destroyed committed versions. The tool must now repair only its own copy. -func TestOpenMemiAVLReplayLeavesSourceUntouched(t *testing.T) { - dbDir := newMemiavlSourceDir(t, 3) - - // A single length-prefix byte declaring a 16-byte record that never - // arrived is exactly what a reader observes while the writer is - // partway through appending a block. - segment := lastChangelogSegment(t, dbDir) - intact, err := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, err) - torn := append(append([]byte{}, intact...), 0x10) - require.NoError(t, os.WriteFile(segment, torn, 0o600)) - - before := snapshotDirState(t, dbDir) - - db, err := openMemiAVLReplay(dbDir, 0) - require.NoError(t, err, "replay must tolerate a torn tail by repairing its own clone") - require.Equal(t, int64(3), db.Version()) - require.NoError(t, db.Close()) - - require.Equal(t, before, snapshotDirState(t, dbDir), - "replay must not add, remove, truncate, or rewrite any file in the source directory") - - after, err := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, err) - require.Equal(t, torn, after, "the torn tail must still be there for the live writer to finish") -} - -// TestOpenMemiAVLReplayWorksWhileWriterHoldsLock pins the other half of the -// contract: avoiding the mutation must not cost us the ability to read a live -// node. The clone is independent, so the source LOCK is irrelevant to us. -func TestOpenMemiAVLReplayWorksWhileWriterHoldsLock(t *testing.T) { - homeDir := t.TempDir() - writer := newTestMemiavlStore(t, homeDir) - require.NoError(t, writer.ApplyChangeSets([]*proto.NamedChangeSet{{ + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ Name: keys.EVMStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}}, }})) - _, err := writer.Commit() + _, err := store.Commit() require.NoError(t, err) - defer func() { require.NoError(t, writer.Close()) }() - - dbDir := utils.GetCosmosSCStorePath(homeDir) - require.FileExists(t, filepath.Join(dbDir, "LOCK")) - - db, err := openMemiAVLReplay(dbDir, 0) - require.NoError(t, err, "tooling clone must not contend for the live writer's lock") - require.Equal(t, int64(1), db.Version()) - require.NoError(t, db.Close()) -} - -// TestOpenMemiAVLReplayAfterSetInitialVersion pins the non-default -// initial-height layout: memiavl bootstraps every DB as snapshot-0 even when -// SetInitialVersion(100) makes the first changelog entry version 100, so the -// clone's WAL-coverage check must derive the snapshot's successor from the -// snapshot metadata instead of assuming version 1 follows snapshot-0. This is -// exactly the shape of a freshly recovered chain whose genesis initial_height -// is greater than 1 and whose first snapshot rewrite has not happened yet; -// before the fix, replay mode failed deterministically on such nodes with -// "source kept churning". -func TestOpenMemiAVLReplayAfterSetInitialVersion(t *testing.T) { - homeDir := t.TempDir() - store := newTestMemiavlStore(t, homeDir) - require.NoError(t, store.SetInitialVersion(100)) - for i := 0; i < 3; i++ { - require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: keys.EVMStoreKey, - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(i+1)), uint64(i+1))}}, - }})) - v, err := store.Commit() - require.NoError(t, err) - require.Equal(t, int64(100+i), v) - } require.NoError(t, store.Close()) - dbDir := utils.GetCosmosSCStorePath(homeDir) - - historical, err := openMemiAVLReplay(dbDir, 101) - require.NoError(t, err, "coverage check must accept a snapshot-0 whose successor is the initial version") - require.Equal(t, int64(101), historical.Version()) - require.NoError(t, historical.Close()) - latest, err := openMemiAVLReplay(dbDir, 0) + dbDir := utils.GetCosmosSCStorePath(homeDir) + segment := lastOperationsMemiAVLWALSegment(t, dbDir) + file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) require.NoError(t, err) - require.Equal(t, int64(102), latest.Version()) - require.NoError(t, latest.Close()) -} - -// TestOpenMemiAVLReplayRejectsShortChangelog guards the failure mode this -// design makes routine: repairing a torn tail inside the clone silently costs -// the trailing version, and memiavl reports success anyway. For a tool whose -// whole job is comparing digests across nodes, quietly digesting one version -// early is indistinguishable from a real state divergence. -func TestOpenMemiAVLReplayRejectsShortChangelog(t *testing.T) { - dbDir := newMemiavlSourceDir(t, 3) - - segment := lastChangelogSegment(t, dbDir) - intact, err := os.ReadFile(filepath.Clean(segment)) + _, err = file.Write([]byte{0x10}) + require.NoError(t, err) + require.NoError(t, file.Close()) + before, err := os.ReadFile(filepath.Clean(segment)) require.NoError(t, err) - // Lop off the tail so the final committed record is torn and the - // repaired clone can only reach version 2. - require.NoError(t, os.WriteFile(segment, intact[:len(intact)-8], 0o600)) - _, err = openMemiAVLReplay(dbDir, 3) + _, err = openMemiAVLReplayReadOnly(dbDir, 0) require.Error(t, err) - require.Contains(t, err.Error(), "requested 3, reached 2") + require.Contains(t, err.Error(), "live WAL was not modified") + require.Contains(t, err.Error(), "rerun the command") + + after, readErr := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, readErr) + require.Equal(t, before, after) +} - // The rejected clone must not be left behind in the node's data dir. - entries, err := os.ReadDir(dbDir) +func lastOperationsMemiAVLWALSegment(t *testing.T, dbDir string) string { + t.Helper() + changelogDir := utils.GetChangelogPath(dbDir) + entries, err := os.ReadDir(changelogDir) require.NoError(t, err) - for _, e := range entries { - require.NotContains(t, e.Name(), ".seidb-memiavl-tool-") + var names []string + for _, entry := range entries { + if !entry.IsDir() && len(entry.Name()) == 20 { + names = append(names, entry.Name()) + } } + require.NotEmpty(t, names) + sort.Strings(names) + return filepath.Join(changelogDir, names[len(names)-1]) } diff --git a/sei-db/tools/cmd/seidb/operations/tool_clone.go b/sei-db/tools/cmd/seidb/operations/tool_clone.go deleted file mode 100644 index 82010d948f..0000000000 --- a/sei-db/tools/cmd/seidb/operations/tool_clone.go +++ /dev/null @@ -1,139 +0,0 @@ -package operations - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" -) - -const ( - // toolCloneOwnerLockName is the flock-held marker file every tooling - // clone carries. While the creating process lives, the lock is held and - // the clone is off-limits; once the process dies (including SIGKILL, - // where no deferred cleanup runs), the kernel releases the flock and the - // next tool invocation can reap the directory. - toolCloneOwnerLockName = ".seidb-tool-owner.lock" - - // staleUnmarkedCloneAge guards the marker-less case: a crash in the - // window between MkdirTemp and the owner-lock creation, where liveness - // cannot be probed. Age is a poor liveness signal for marked clones - // (a mainnet-scale digest can legitimately run for hours), so it is - // used only when no marker exists at all. - staleUnmarkedCloneAge = 24 * time.Hour -) - -// toolClone is a private, disposable copy of a live store's snapshot + -// changelog that seidb tooling operates on instead of the live directory. -// It lives inside the source dbDir (to share its filesystem for hardlinks), -// which is exactly why leaking one is costly: its hardlinks pin snapshot -// inodes, so the live node's snapshot pruning frees no disk space until the -// clone is removed. -type toolClone struct { - dir string - ownerLock memiavl.FileLock - - // walRepaired records that validating the cloned changelog shrank it: - // the byte-copy caught the live writer mid-append and the WAL open - // repaired the torn tail inside the clone, costing the trailing record. - walRepaired bool -} - -// newToolClone reaps abandoned sibling clones, creates a fresh clone -// directory under dbDir with the given prefix, and marks it owned via flock -// before any expensive cloning starts. -// -// The prefix deliberately has no "-tmp" suffix: memiavl's removeTmpDirs -// deletes every "*-tmp" directory under its root when a node opens the DB -// read-write, and these clones must never be reaped by a process that cannot -// see whether the owning tool is still alive. -func newToolClone(dbDir, prefix string) (*toolClone, error) { - sweepStaleToolClones(dbDir, prefix) - - dir, err := os.MkdirTemp(dbDir, prefix+"*") - if err != nil { - return nil, fmt.Errorf("create temp dir under %s: %w", dbDir, err) - } - ownerLock, err := memiavl.LockFile(filepath.Join(dir, toolCloneOwnerLockName)) - if err != nil { - _ = os.RemoveAll(dir) - return nil, fmt.Errorf("acquire clone owner lock in %s: %w", dir, err) - } - return &toolClone{dir: dir, ownerLock: ownerLock}, nil -} - -// Remove releases the ownership lock and deletes the clone directory. -func (c *toolClone) Remove() error { - if c == nil { - return nil - } - if c.ownerLock != nil { - _ = c.ownerLock.Unlock() - _ = c.ownerLock.Destroy() - c.ownerLock = nil - } - if c.dir == "" { - return nil - } - if err := os.RemoveAll(c.dir); err != nil { - return fmt.Errorf("cleanup temp dir: %w", err) - } - c.dir = "" - return nil -} - -// sweepStaleToolClones removes abandoned clone directories under dbDir whose -// owner is provably gone: either the owner flock is acquirable (the creating -// process died), or no marker exists and the directory is old enough that the -// mkdir-to-lock window cannot explain it. Clones whose lock is still held — -// a concurrently running tool — are left alone. Best-effort by design: a -// failed sweep must never block the read the tool was invoked for. -func sweepStaleToolClones(dbDir, prefix string) { - entries, err := os.ReadDir(dbDir) - if err != nil { - return - } - for _, entry := range entries { - if !entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { - continue - } - dir := filepath.Join(dbDir, entry.Name()) - lockPath := filepath.Join(dir, toolCloneOwnerLockName) - if _, err := os.Stat(lockPath); errors.Is(err, os.ErrNotExist) { - if info, err := entry.Info(); err == nil && time.Since(info.ModTime()) > staleUnmarkedCloneAge { - _ = os.RemoveAll(dir) - } - continue - } - lock, err := memiavl.LockFile(lockPath) - if err != nil { - // Lock held (owner alive) or unreadable — leave the clone alone. - continue - } - _ = lock.Unlock() - _ = lock.Destroy() - _ = os.RemoveAll(dir) - } -} - -// changelogByteSize sums the sizes of the regular files in a cloned changelog -// directory. Comparing it before and after the WAL-coverage validation open -// detects a torn-tail repair inside the clone (the only mutation that open -// can perform), which callers surface as a warning for latest-height reads. -func changelogByteSize(dir string) int64 { - var total int64 - _ = filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error { - if err != nil { - return nil //nolint:nilerr // best-effort size probe - } - if info.Mode().IsRegular() { - total += info.Size() - } - return nil - }) - return total -} diff --git a/sei-db/tools/cmd/seidb/operations/tool_clone_test.go b/sei-db/tools/cmd/seidb/operations/tool_clone_test.go deleted file mode 100644 index 5ddf6941fb..0000000000 --- a/sei-db/tools/cmd/seidb/operations/tool_clone_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package operations - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" -) - -// TestSweepStaleToolClones pins the reaping policy for abandoned tooling -// clones: a clone whose owner lock is acquirable was orphaned by a dead -// process (SIGKILL runs no deferred cleanup) and must be removed, a clone -// whose lock is held belongs to a live tool and must survive, and a -// marker-less clone is removed only once it is old enough that the -// mkdir-to-lock window cannot explain it. -func TestSweepStaleToolClones(t *testing.T) { - dbDir := t.TempDir() - prefix := ".seidb-flatkv-tool-" - - staleUnmarked := filepath.Join(dbDir, prefix+"stale-unmarked") - require.NoError(t, os.Mkdir(staleUnmarked, 0o750)) - old := time.Now().Add(-2 * staleUnmarkedCloneAge) - require.NoError(t, os.Chtimes(staleUnmarked, old, old)) - - freshUnmarked := filepath.Join(dbDir, prefix+"fresh-unmarked") - require.NoError(t, os.Mkdir(freshUnmarked, 0o750)) - - held := filepath.Join(dbDir, prefix+"held") - require.NoError(t, os.Mkdir(held, 0o750)) - heldLock, err := memiavl.LockFile(filepath.Join(held, toolCloneOwnerLockName)) - require.NoError(t, err) - - orphaned := filepath.Join(dbDir, prefix+"orphaned") - require.NoError(t, os.Mkdir(orphaned, 0o750)) - require.NoError(t, os.WriteFile(filepath.Join(orphaned, toolCloneOwnerLockName), nil, 0o600)) - - otherPrefix := filepath.Join(dbDir, ".seidb-memiavl-tool-orphaned") - require.NoError(t, os.Mkdir(otherPrefix, 0o750)) - require.NoError(t, os.WriteFile(filepath.Join(otherPrefix, toolCloneOwnerLockName), nil, 0o600)) - - sweepStaleToolClones(dbDir, prefix) - - require.NoDirExists(t, staleUnmarked, "old marker-less clone must be reaped") - require.DirExists(t, freshUnmarked, "fresh marker-less clone may still be mid-creation") - require.DirExists(t, held, "a held owner lock proves the owning tool is alive") - require.NoDirExists(t, orphaned, "an acquirable owner lock proves the owner died") - require.DirExists(t, otherPrefix, "the sweep must not touch other clone families") - - require.NoError(t, heldLock.Unlock()) - sweepStaleToolClones(dbDir, prefix) - require.NoDirExists(t, held, "once the owner releases the lock the clone is reapable") -} - -// TestNewToolCloneOwnershipLifecycle checks that a live clone defends itself -// against a concurrent sweep and that Remove releases everything. -func TestNewToolCloneOwnershipLifecycle(t *testing.T) { - dbDir := t.TempDir() - prefix := ".seidb-memiavl-tool-" - - clone, err := newToolClone(dbDir, prefix) - require.NoError(t, err) - require.DirExists(t, clone.dir) - require.FileExists(t, filepath.Join(clone.dir, toolCloneOwnerLockName)) - - sweepStaleToolClones(dbDir, prefix) - require.DirExists(t, clone.dir, "an owned clone must survive a concurrent sweep") - - require.NoError(t, clone.Remove()) - require.NoDirExists(t, clone.dir) - require.NoError(t, clone.Remove(), "Remove must be idempotent") -} diff --git a/sei-db/wal/changelog.go b/sei-db/wal/changelog.go index 07ded92c22..046860e682 100644 --- a/sei-db/wal/changelog.go +++ b/sei-db/wal/changelog.go @@ -24,3 +24,19 @@ func NewChangelogWAL(dir string, config Config) (ChangelogWAL, error) { config, ) } + +// OpenReadOnlyChangelogWAL opens an immutable point-in-time view of the +// changelog segment files. It never creates, truncates, removes, or renames WAL +// files. A torn tail or an in-progress tidwall recovery marker returns +// tidwall/wal.ErrCorrupt so callers can fail and retry after the writer moves +// on. +func OpenReadOnlyChangelogWAL(dir string) (ChangelogWAL, error) { + return openReadOnlyWAL( + dir, + func(data []byte) (proto.ChangelogEntry, error) { + var entry proto.ChangelogEntry + err := entry.Unmarshal(data) + return entry, err + }, + ) +} diff --git a/sei-db/wal/readonly.go b/sei-db/wal/readonly.go new file mode 100644 index 0000000000..68222dacc0 --- /dev/null +++ b/sei-db/wal/readonly.go @@ -0,0 +1,250 @@ +package wal + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + + tidwallwal "github.com/tidwall/wal" +) + +var ( + // ErrReadOnly is returned when a caller tries to mutate a read-only WAL. + ErrReadOnly = errors.New("WAL is read-only") + + // ErrCorrupt identifies a malformed or unstable WAL view. It aliases the + // underlying tidwall sentinel so callers do not need to import the storage + // implementation only to decide whether a read-only open can be retried. + ErrCorrupt = tidwallwal.ErrCorrupt +) + +type readOnlySegment struct { + name string + index uint64 +} + +type readOnlyEntry struct { + file *os.File + dataOffset int64 + size int +} + +// readOnlyWAL is an immutable view of the plain segment files present when it +// opens. It does not use tidwall/wal.Open because that function creates files, +// opens the tail for writing, and completes interrupted truncations by removing +// and renaming segment files. +type readOnlyWAL[T any] struct { + unmarshal UnmarshalFn[T] + files []*os.File + entries []readOnlyEntry + firstOffset uint64 + closed atomic.Bool + closeOnce sync.Once + closeErr error +} + +func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[T], error) { + segments, err := listReadOnlySegments(dir) + if err != nil { + return nil, err + } + + log := &readOnlyWAL[T]{unmarshal: unmarshal} + cleanup := func(err error) (*readOnlyWAL[T], error) { + _ = log.Close() + return nil, err + } + + var nextIndex uint64 + for i, segment := range segments { + if i > 0 && segment.index != nextIndex { + return cleanup(fmt.Errorf("%w: segment %s starts at index %d, expected %d", + tidwallwal.ErrCorrupt, segment.name, segment.index, nextIndex)) + } + + path := filepath.Join(dir, segment.name) + file, err := os.Open(filepath.Clean(path)) + if err != nil { + return cleanup(fmt.Errorf("open WAL segment %s: %w", path, err)) + } + log.files = append(log.files, file) + + info, err := file.Stat() + if err != nil { + return cleanup(fmt.Errorf("stat WAL segment %s: %w", path, err)) + } + if !info.Mode().IsRegular() { + return cleanup(fmt.Errorf("%w: WAL segment %s is not a regular file", tidwallwal.ErrCorrupt, path)) + } + + data, err := io.ReadAll(io.NewSectionReader(file, 0, info.Size())) + if err != nil { + return cleanup(fmt.Errorf("read WAL segment %s: %w", path, err)) + } + if int64(len(data)) != info.Size() { + return cleanup(fmt.Errorf("%w: WAL segment %s changed while it was read", + tidwallwal.ErrCorrupt, path)) + } + + entries, err := indexReadOnlySegment(file, data) + if err != nil { + return cleanup(fmt.Errorf("index WAL segment %s: %w", path, err)) + } + if len(entries) == 0 && i != len(segments)-1 { + return cleanup(fmt.Errorf("%w: non-tail WAL segment %s is empty", tidwallwal.ErrCorrupt, path)) + } + if len(log.entries) == 0 && len(entries) > 0 { + log.firstOffset = segment.index + } + log.entries = append(log.entries, entries...) + nextIndex = segment.index + uint64(len(entries)) + } + return log, nil +} + +func listReadOnlySegments(dir string) ([]readOnlySegment, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read WAL directory %s: %w", dir, err) + } + + segments := make([]readOnlySegment, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { + return nil, fmt.Errorf("%w: WAL recovery marker %s is present; retry after the writer finishes", + tidwallwal.ErrCorrupt, name) + } + if len(name) != 20 { + continue + } + index, err := strconv.ParseUint(name, 10, 64) + if err != nil || index == 0 { + continue + } + segments = append(segments, readOnlySegment{name: name, index: index}) + } + sort.Slice(segments, func(i, j int) bool { + return segments[i].index < segments[j].index + }) + return segments, nil +} + +func indexReadOnlySegment(file *os.File, data []byte) ([]readOnlyEntry, error) { + entries := make([]readOnlyEntry, 0) + for pos := 0; pos < len(data); { + size, prefixLen := binary.Uvarint(data[pos:]) + if prefixLen <= 0 || size > math.MaxInt32 { + return nil, tidwallwal.ErrCorrupt + } + if len(data)-pos-prefixLen < int(size) { + return nil, tidwallwal.ErrCorrupt + } + entries = append(entries, readOnlyEntry{ + file: file, + dataOffset: int64(pos + prefixLen), + size: int(size), + }) + pos += prefixLen + int(size) + } + return entries, nil +} + +func (log *readOnlyWAL[T]) Write(T) error { + return ErrReadOnly +} + +func (log *readOnlyWAL[T]) TruncateBefore(uint64) error { + return ErrReadOnly +} + +func (log *readOnlyWAL[T]) TruncateAfter(uint64) error { + return ErrReadOnly +} + +func (log *readOnlyWAL[T]) TruncateAll() error { + return ErrReadOnly +} + +func (log *readOnlyWAL[T]) FirstOffset() (uint64, error) { + if log.closed.Load() { + return 0, os.ErrClosed + } + if len(log.entries) == 0 { + return 0, nil + } + return log.firstOffset, nil +} + +func (log *readOnlyWAL[T]) LastOffset() (uint64, error) { + if log.closed.Load() { + return 0, os.ErrClosed + } + if len(log.entries) == 0 { + return 0, nil + } + return log.firstOffset + uint64(len(log.entries)) - 1, nil +} + +func (log *readOnlyWAL[T]) ReadAt(index uint64) (T, error) { + var zero T + if log.closed.Load() { + return zero, os.ErrClosed + } + if index < log.firstOffset || index-log.firstOffset >= uint64(len(log.entries)) { + return zero, fmt.Errorf("read WAL offset %d: out of range", index) + } + + entry := log.entries[index-log.firstOffset] + data := make([]byte, entry.size) + if _, err := entry.file.ReadAt(data, entry.dataOffset); err != nil { + return zero, fmt.Errorf("read WAL offset %d: %w", index, err) + } + value, err := log.unmarshal(data) + if err != nil { + return zero, fmt.Errorf("unmarshal WAL offset %d: %w", index, err) + } + return value, nil +} + +func (log *readOnlyWAL[T]) Replay(start, end uint64, processFn func(index uint64, entry T) error) error { + if end < start { + return nil + } + for index := start; index <= end; index++ { + entry, err := log.ReadAt(index) + if err != nil { + return err + } + if err := processFn(index, entry); err != nil { + return fmt.Errorf("process WAL offset %d: %w", index, err) + } + } + return nil +} + +func (log *readOnlyWAL[T]) Close() error { + log.closeOnce.Do(func() { + log.closed.Store(true) + var errs []error + for _, file := range log.files { + if err := file.Close(); err != nil { + errs = append(errs, err) + } + } + log.closeErr = errors.Join(errs...) + }) + return log.closeErr +} diff --git a/sei-db/wal/readonly_test.go b/sei-db/wal/readonly_test.go new file mode 100644 index 0000000000..187126c0da --- /dev/null +++ b/sei-db/wal/readonly_test.go @@ -0,0 +1,148 @@ +package wal + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + tidwallwal "github.com/tidwall/wal" + + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +func TestOpenReadOnlyChangelogWALReplaysWithoutMutation(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + writeReadOnlyTestData(t, writable) + require.NoError(t, writable.Close()) + + before := snapshotWALFiles(t, dir) + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + + first, err := readOnly.FirstOffset() + require.NoError(t, err) + require.Equal(t, uint64(1), first) + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Equal(t, uint64(3), last) + + var names []string + require.NoError(t, readOnly.Replay(first, last, func(_ uint64, entry proto.ChangelogEntry) error { + names = append(names, entry.Changesets[0].Name) + return nil + })) + require.Equal(t, []string{"test", "test", "test"}, names) + require.Equal(t, before, snapshotWALFiles(t, dir)) + + require.ErrorIs(t, readOnly.Write(proto.ChangelogEntry{}), ErrReadOnly) + require.ErrorIs(t, readOnly.TruncateBefore(2), ErrReadOnly) + require.ErrorIs(t, readOnly.TruncateAfter(2), ErrReadOnly) + require.NoError(t, readOnly.Close()) + require.NoError(t, readOnly.Close()) +} + +func TestOpenReadOnlyChangelogWALRejectsTornTailWithoutRepair(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + writeReadOnlyTestData(t, writable) + require.NoError(t, writable.Close()) + + segment := lastPlainWALSegment(t, dir) + file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) + require.NoError(t, err) + _, err = file.Write([]byte{0x10}) // declares a 16-byte record whose payload has not arrived + require.NoError(t, err) + require.NoError(t, file.Close()) + before := snapshotWALFiles(t, dir) + + _, err = OpenReadOnlyChangelogWAL(dir) + require.ErrorIs(t, err, tidwallwal.ErrCorrupt) + require.Equal(t, before, snapshotWALFiles(t, dir), "read-only open must not repair the source tail") +} + +func TestOpenReadOnlyChangelogWALKeepsPointInTimeView(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + require.NoError(t, writable.Write(proto.ChangelogEntry{Version: 1})) + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, readOnly.Close()) }) + + require.NoError(t, writable.Write(proto.ChangelogEntry{Version: 2})) + require.NoError(t, writable.Close()) + + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Equal(t, uint64(1), last) + _, err = readOnly.ReadAt(2) + require.Error(t, err) +} + +func TestOpenReadOnlyChangelogWALRejectsRecoveryMarkers(t *testing.T) { + dir := t.TempDir() + marker := filepath.Join(dir, "00000000000000000001.START") + require.NoError(t, os.WriteFile(marker, nil, 0o600)) + + _, err := OpenReadOnlyChangelogWAL(dir) + require.ErrorIs(t, err, tidwallwal.ErrCorrupt) + require.FileExists(t, marker, "read-only open must not complete writable WAL recovery") +} + +func TestOpenReadOnlyChangelogWALDoesNotCreateMissingDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "missing") + + _, err := OpenReadOnlyChangelogWAL(dir) + require.Error(t, err) + require.NoDirExists(t, dir) +} + +func lastPlainWALSegment(t *testing.T, dir string) string { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + var names []string + for _, entry := range entries { + if !entry.IsDir() && len(entry.Name()) == 20 { + names = append(names, entry.Name()) + } + } + require.NotEmpty(t, names) + sort.Strings(names) + return filepath.Join(dir, names[len(names)-1]) +} + +func snapshotWALFiles(t *testing.T, dir string) map[string][]byte { + t.Helper() + files := make(map[string][]byte) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + for _, entry := range entries { + if entry.IsDir() { + continue + } + data, err := os.ReadFile(filepath.Clean(filepath.Join(dir, entry.Name()))) + require.NoError(t, err) + files[entry.Name()] = data + } + return files +} + +func writeReadOnlyTestData(t *testing.T, changelog ChangelogWAL) { + t.Helper() + for i, changeset := range ChangeSets { + require.NoError(t, changelog.Write(proto.ChangelogEntry{ + Version: int64(i + 1), + Changesets: []*proto.NamedChangeSet{{ + Name: "test", + Changeset: changeset, + }}, + })) + } +} From add44481a7073d49fcc898a73c966d32db3a18fb Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 15:42:56 -0400 Subject: [PATCH 5/6] test(wal): harden read-only concurrency coverage Exercise append and prune races, both recovery markers, and empty-directory behavior while reusing the canonical binary record validator. Co-authored-by: Cursor --- sei-db/wal/readonly.go | 16 +++--- sei-db/wal/readonly_test.go | 103 ++++++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 14 deletions(-) diff --git a/sei-db/wal/readonly.go b/sei-db/wal/readonly.go index 68222dacc0..f66430138f 100644 --- a/sei-db/wal/readonly.go +++ b/sei-db/wal/readonly.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "math" "os" "path/filepath" "sort" @@ -145,19 +144,18 @@ func listReadOnlySegments(dir string) ([]readOnlySegment, error) { func indexReadOnlySegment(file *os.File, data []byte) ([]readOnlyEntry, error) { entries := make([]readOnlyEntry, 0) for pos := 0; pos < len(data); { - size, prefixLen := binary.Uvarint(data[pos:]) - if prefixLen <= 0 || size > math.MaxInt32 { - return nil, tidwallwal.ErrCorrupt - } - if len(data)-pos-prefixLen < int(size) { - return nil, tidwallwal.ErrCorrupt + recordLen, err := loadNextBinaryEntry(data[pos:]) + if err != nil { + return nil, err } + size, prefixLen := binary.Uvarint(data[pos:]) + entrySize := int(size) //nolint:gosec // loadNextBinaryEntry rejects sizes above math.MaxInt32. entries = append(entries, readOnlyEntry{ file: file, dataOffset: int64(pos + prefixLen), - size: int(size), + size: entrySize, }) - pos += prefixLen + int(size) + pos += recordLen } return entries, nil } diff --git a/sei-db/wal/readonly_test.go b/sei-db/wal/readonly_test.go index 187126c0da..f512f0f091 100644 --- a/sei-db/wal/readonly_test.go +++ b/sei-db/wal/readonly_test.go @@ -86,13 +86,37 @@ func TestOpenReadOnlyChangelogWALKeepsPointInTimeView(t *testing.T) { } func TestOpenReadOnlyChangelogWALRejectsRecoveryMarkers(t *testing.T) { + for _, suffix := range []string{".START", ".END"} { + t.Run(suffix, func(t *testing.T) { + dir := t.TempDir() + marker := filepath.Join(dir, "00000000000000000001"+suffix) + require.NoError(t, os.WriteFile(marker, nil, 0o600)) + before := snapshotWALFiles(t, dir) + + _, err := OpenReadOnlyChangelogWAL(dir) + require.ErrorIs(t, err, tidwallwal.ErrCorrupt) + require.Equal(t, before, snapshotWALFiles(t, dir), + "read-only open must not complete writable WAL recovery") + }) + } +} + +func TestOpenReadOnlyChangelogWALEmptyDirectory(t *testing.T) { dir := t.TempDir() - marker := filepath.Join(dir, "00000000000000000001.START") - require.NoError(t, os.WriteFile(marker, nil, 0o600)) - _, err := OpenReadOnlyChangelogWAL(dir) - require.ErrorIs(t, err, tidwallwal.ErrCorrupt) - require.FileExists(t, marker, "read-only open must not complete writable WAL recovery") + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + first, err := readOnly.FirstOffset() + require.NoError(t, err) + require.Zero(t, first) + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Zero(t, last) + require.NoError(t, readOnly.Close()) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, "read-only open must not create an initial segment") } func TestOpenReadOnlyChangelogWALDoesNotCreateMissingDirectory(t *testing.T) { @@ -103,6 +127,75 @@ func TestOpenReadOnlyChangelogWALDoesNotCreateMissingDirectory(t *testing.T) { require.NoDirExists(t, dir) } +func TestOpenReadOnlyChangelogWALConcurrentWriter(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + + const versions = 500 + done := make(chan struct{}) + writerErr := make(chan error, 1) + go func() { + var runErr error + for version := 1; version <= versions && runErr == nil; version++ { + runErr = writable.Write(proto.ChangelogEntry{Version: int64(version)}) + if runErr != nil || version <= 20 || version%10 != 0 { + continue + } + first, err := writable.FirstOffset() + if err != nil { + runErr = err + break + } + keepFrom := uint64(version - 20) + if keepFrom > first { + runErr = writable.TruncateBefore(keepFrom) + } + } + if closeErr := writable.Close(); runErr == nil { + runErr = closeErr + } + writerErr <- runErr + close(done) + }() + +reading: + for { + select { + case <-done: + break reading + default: + } + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + if err != nil { + continue // source changed while opening; fail-closed and retry is valid + } + first, err := readOnly.FirstOffset() + require.NoError(t, err) + last, err := readOnly.LastOffset() + require.NoError(t, err) + if first > 0 { + require.GreaterOrEqual(t, last, first) + entry, err := readOnly.ReadAt(last) + require.NoError(t, err) + require.Equal(t, int64(last), entry.Version) + } + require.NoError(t, readOnly.Close()) + } + require.NoError(t, <-writerErr) + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + defer func() { require.NoError(t, readOnly.Close()) }() + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Equal(t, uint64(versions), last) + entry, err := readOnly.ReadAt(last) + require.NoError(t, err) + require.Equal(t, int64(versions), entry.Version) +} + func lastPlainWALSegment(t *testing.T, dir string) string { t.Helper() entries, err := os.ReadDir(dir) From 9ecc984c0985a428efe5c0c52b5dafab5d47498c Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 15:46:00 -0400 Subject: [PATCH 6/6] fix(memiavl): release resources after failed opens Centralize failed-open cleanup so WAL handles and exclusive locks cannot leak, and keep failed read-only WAL opens nil-safe. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 44 ++++++++++++++++----------- sei-db/state_db/sc/memiavl/db_test.go | 26 ++++++++++++++++ sei-db/wal/changelog.go | 11 ++++--- sei-db/wal/readonly.go | 10 +++--- sei-db/wal/readonly_test.go | 5 ++- 5 files changed, 66 insertions(+), 30 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index e7ff87d9dd..feb1fee485 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -172,9 +172,26 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { ) }() var ( - err error - fileLock FileLock + err error + fileLock FileLock + mtree *MultiTree + streamHandler wal.ChangelogWAL ) + defer func() { + if _err == nil { + return + } + if streamHandler != nil { + _ = streamHandler.Close() + } + if mtree != nil { + _ = mtree.Close() + } + if fileLock != nil { + _ = fileLock.Unlock() + _ = fileLock.Destroy() + } + }() if err := opts.Validate(); err != nil { return nil, fmt.Errorf("invalid commit store options: %w", err) } @@ -208,7 +225,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { } path := filepath.Join(opts.Dir, snapshot) - mtree, err := LoadMultiTree(context.Background(), path, opts) + mtree, err = LoadMultiTree(context.Background(), path, opts) if err != nil { return nil, err } @@ -219,7 +236,6 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { // callers still need replay to reconstruct non-snapshot versions, but they // must not use the writable opener: it repairs a torn tail by truncating it // and completes interrupted WAL truncations by renaming or removing files. - var streamHandler wal.ChangelogWAL if opts.ReadOnly { streamHandler, err = wal.OpenReadOnlyChangelogWAL(utils.GetChangelogPath(opts.Dir)) } else { @@ -228,7 +244,6 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { }) } if err != nil { - _ = mtree.Close() if opts.ReadOnly && errors.Is(err, wal.ErrCorrupt) { return nil, fmt.Errorf("%w; source WAL was not modified: %w", ErrReadOnlyWALCorrupt, err) } @@ -240,8 +255,6 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { var walHasEntries bool walIndexDelta, walHasEntries, err = computeWALIndexDelta(streamHandler) if err != nil { - _ = streamHandler.Close() - _ = mtree.Close() return nil, fmt.Errorf("failed to compute WAL index delta: %w", err) } // If WAL is empty, set delta so first WAL entry aligns with NextVersion(). @@ -251,21 +264,15 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { if opts.ReadOnly && walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { firstIndex, firstErr := streamHandler.FirstOffset() if firstErr != nil { - _ = streamHandler.Close() - _ = mtree.Close() - return nil, fmt.Errorf("read read-only changelog first offset: %w", firstErr) + return nil, fmt.Errorf("read changelog first offset: %w", firstErr) } if firstIndex > math.MaxInt64 { - _ = streamHandler.Close() - _ = mtree.Close() return nil, fmt.Errorf("%w: first WAL offset %d overflows int64", ErrReadOnlyWALUnavailable, firstIndex) } firstVersion := int64(firstIndex) + walIndexDelta firstNeeded := utils.NextVersion(mtree.Version(), mtree.initialVersion.Load()) if firstVersion > firstNeeded { snapshotVersion := mtree.Version() - _ = streamHandler.Close() - _ = mtree.Close() return nil, fmt.Errorf("%w: selected snapshot version %d needs changelog version %d, "+ "but the immutable WAL view starts at version %d", ErrReadOnlyWALUnavailable, snapshotVersion, firstNeeded, firstVersion) @@ -276,16 +283,12 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { if walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { logger.Info("Start catching up and replaying the MemIAVL changelog file") if err := mtree.Catchup(context.Background(), streamHandler, walIndexDelta, targetVersion); err != nil { - _ = streamHandler.Close() - _ = mtree.Close() return nil, err } logger.Info("finished replay and caught up to target version", "version", targetVersion) } if opts.ReadOnly && targetVersion > 0 && mtree.Version() != targetVersion { reached := mtree.Version() - _ = streamHandler.Close() - _ = mtree.Close() return nil, fmt.Errorf("%w: requested %d, reached %d", ErrReadOnlyWALUnavailable, targetVersion, reached) } @@ -355,6 +358,10 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { snapshotWriterPool: workerPool, opts: opts, } + // The DB owns these resources from this point forward. + mtree = nil + streamHandler = nil + fileLock = nil // Apply initial stores on a fresh DB (version 0) so they get persisted to WAL. // This creates the trees and populates pendingLogEntry, which will be written @@ -366,6 +373,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { upgrades = append(upgrades, &proto.TreeNameUpgrade{Name: name}) } if err := db.ApplyUpgrades(upgrades); err != nil { + _ = db.Close() return nil, fmt.Errorf("failed to apply initial stores: %w", err) } } diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index 91146d47af..d804189902 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -1206,6 +1206,32 @@ func TestReadOnlyOpenRejectsShortWAL(t *testing.T) { require.Contains(t, err.Error(), "requested 3, reached 2") } +func TestOpenDBFailureReleasesFileLock(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) + require.NoError(t, err) + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[0], + }})) + _, err = db.Commit() + require.NoError(t, err) + require.NoError(t, db.Close()) + + require.NoError(t, os.Remove(currentPath(dir))) + _, err = OpenDB(1, Options{Dir: dir, LoadForOverwriting: true}) + require.ErrorContains(t, err, "fail to read current version") + + lock, err := LockFile(filepath.Join(dir, LockFileName)) + require.NoError(t, err, "failed OpenDB must release its exclusive lock") + require.NoError(t, lock.Unlock()) + require.NoError(t, lock.Destroy()) +} + func lastMemiAVLWALSegment(t *testing.T, dir string) string { t.Helper() entries, err := os.ReadDir(utils.GetChangelogPath(dir)) diff --git a/sei-db/wal/changelog.go b/sei-db/wal/changelog.go index 046860e682..40d2361f48 100644 --- a/sei-db/wal/changelog.go +++ b/sei-db/wal/changelog.go @@ -27,11 +27,10 @@ func NewChangelogWAL(dir string, config Config) (ChangelogWAL, error) { // OpenReadOnlyChangelogWAL opens an immutable point-in-time view of the // changelog segment files. It never creates, truncates, removes, or renames WAL -// files. A torn tail or an in-progress tidwall recovery marker returns -// tidwall/wal.ErrCorrupt so callers can fail and retry after the writer moves -// on. +// files. A torn tail or an in-progress recovery marker returns ErrCorrupt so +// callers can fail and retry after the writer moves on. func OpenReadOnlyChangelogWAL(dir string) (ChangelogWAL, error) { - return openReadOnlyWAL( + readOnly, err := openReadOnlyWAL( dir, func(data []byte) (proto.ChangelogEntry, error) { var entry proto.ChangelogEntry @@ -39,4 +38,8 @@ func OpenReadOnlyChangelogWAL(dir string) (ChangelogWAL, error) { return entry, err }, ) + if err != nil { + return nil, err + } + return readOnly, nil } diff --git a/sei-db/wal/readonly.go b/sei-db/wal/readonly.go index f66430138f..884ca77394 100644 --- a/sei-db/wal/readonly.go +++ b/sei-db/wal/readonly.go @@ -67,7 +67,7 @@ func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[ for i, segment := range segments { if i > 0 && segment.index != nextIndex { return cleanup(fmt.Errorf("%w: segment %s starts at index %d, expected %d", - tidwallwal.ErrCorrupt, segment.name, segment.index, nextIndex)) + ErrCorrupt, segment.name, segment.index, nextIndex)) } path := filepath.Join(dir, segment.name) @@ -82,7 +82,7 @@ func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[ return cleanup(fmt.Errorf("stat WAL segment %s: %w", path, err)) } if !info.Mode().IsRegular() { - return cleanup(fmt.Errorf("%w: WAL segment %s is not a regular file", tidwallwal.ErrCorrupt, path)) + return cleanup(fmt.Errorf("%w: WAL segment %s is not a regular file", ErrCorrupt, path)) } data, err := io.ReadAll(io.NewSectionReader(file, 0, info.Size())) @@ -91,7 +91,7 @@ func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[ } if int64(len(data)) != info.Size() { return cleanup(fmt.Errorf("%w: WAL segment %s changed while it was read", - tidwallwal.ErrCorrupt, path)) + ErrCorrupt, path)) } entries, err := indexReadOnlySegment(file, data) @@ -99,7 +99,7 @@ func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[ return cleanup(fmt.Errorf("index WAL segment %s: %w", path, err)) } if len(entries) == 0 && i != len(segments)-1 { - return cleanup(fmt.Errorf("%w: non-tail WAL segment %s is empty", tidwallwal.ErrCorrupt, path)) + return cleanup(fmt.Errorf("%w: non-tail WAL segment %s is empty", ErrCorrupt, path)) } if len(log.entries) == 0 && len(entries) > 0 { log.firstOffset = segment.index @@ -124,7 +124,7 @@ func listReadOnlySegments(dir string) ([]readOnlySegment, error) { name := entry.Name() if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { return nil, fmt.Errorf("%w: WAL recovery marker %s is present; retry after the writer finishes", - tidwallwal.ErrCorrupt, name) + ErrCorrupt, name) } if len(name) != 20 { continue diff --git a/sei-db/wal/readonly_test.go b/sei-db/wal/readonly_test.go index f512f0f091..9fb0ebc3c0 100644 --- a/sei-db/wal/readonly_test.go +++ b/sei-db/wal/readonly_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/stretchr/testify/require" - tidwallwal "github.com/tidwall/wal" "github.com/sei-protocol/sei-chain/sei-db/proto" ) @@ -61,7 +60,7 @@ func TestOpenReadOnlyChangelogWALRejectsTornTailWithoutRepair(t *testing.T) { before := snapshotWALFiles(t, dir) _, err = OpenReadOnlyChangelogWAL(dir) - require.ErrorIs(t, err, tidwallwal.ErrCorrupt) + require.ErrorIs(t, err, ErrCorrupt) require.Equal(t, before, snapshotWALFiles(t, dir), "read-only open must not repair the source tail") } @@ -94,7 +93,7 @@ func TestOpenReadOnlyChangelogWALRejectsRecoveryMarkers(t *testing.T) { before := snapshotWALFiles(t, dir) _, err := OpenReadOnlyChangelogWAL(dir) - require.ErrorIs(t, err, tidwallwal.ErrCorrupt) + require.ErrorIs(t, err, ErrCorrupt) require.Equal(t, before, snapshotWALFiles(t, dir), "read-only open must not complete writable WAL recovery") })