diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index 54475c8fdd..cb41f0cbc6 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "math" "runtime" "strings" @@ -12,6 +13,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config" + "github.com/spf13/cast" "github.com/spf13/viper" ) @@ -65,6 +67,10 @@ type BaseConfig struct { // Note: Commitment of state will be attempted on the corresponding block. HaltHeight uint64 `mapstructure:"halt-height"` + // FreezeHeight contains a non-zero block height at which the node stops + // before executing the block while continuing to serve RPC. + FreezeHeight uint64 `mapstructure:"freeze-height"` + // HaltTime contains a non-zero minimum block time (in Unix seconds) at which // a node will gracefully halt and shutdown that can be used to assist // upgrades and testing. @@ -258,6 +264,7 @@ func DefaultConfig() *Config { PruningKeepRecent: "0", PruningKeepEvery: "0", PruningInterval: "0", + FreezeHeight: 0, MinRetainBlocks: 0, IndexEvents: nil, CompactionInterval: 0, @@ -314,6 +321,10 @@ func GetConfig(v *viper.Viper) (Config, error) { if !ok { return Config{}, fmt.Errorf("failed to parse global-labels config") } + freezeHeight, err := cast.ToUint64E(v.Get("freeze-height")) + if err != nil { + return Config{}, fmt.Errorf("invalid freeze-height: %w", err) + } globalLabels := make([][]string, 0, len(globalLabelsRaw)) for idx, glr := range globalLabelsRaw { @@ -347,6 +358,7 @@ func GetConfig(v *viper.Viper) (Config, error) { PruningKeepRecent: v.GetString("pruning-keep-recent"), PruningInterval: v.GetString("pruning-interval"), HaltHeight: v.GetUint64("halt-height"), + FreezeHeight: freezeHeight, HaltTime: v.GetUint64("halt-time"), IndexEvents: v.GetStringSlice("index-events"), MinRetainBlocks: v.GetUint64("min-retain-blocks"), @@ -427,7 +439,7 @@ func GetConfig(v *viper.Viper) (Config, error) { }, nil } -// ValidateBasic returns an error if min-gas-prices field is empty in BaseConfig. Otherwise, it returns nil. +// ValidateBasic validates the server configuration. func (c Config) ValidateBasic(tendermintConfig *tmcfg.Config) error { if c.MinGasPrices == "" { return sdkerrors.ErrAppConfig.Wrap("set min gas price in app.toml or flag or env variable") @@ -437,6 +449,17 @@ func (c Config) ValidateBasic(tendermintConfig *tmcfg.Config) error { "cannot enable state sync snapshots with '%s' pruning setting", storetypes.PruningOptionEverything, ) } + return c.ValidateFreeze() +} + +// ValidateFreeze validates the configuration that controls freeze mode. +func (c Config) ValidateFreeze() error { + if c.FreezeHeight > math.MaxInt64 { + return sdkerrors.ErrAppConfig.Wrapf("freeze-height must not exceed %d", int64(math.MaxInt64)) + } + if c.FreezeHeight > 0 && (c.HaltHeight > 0 || c.HaltTime > 0) { + return sdkerrors.ErrAppConfig.Wrap("freeze-height cannot be combined with halt-height or halt-time") + } return nil } diff --git a/sei-cosmos/server/config/config_test.go b/sei-cosmos/server/config/config_test.go index 9d0951a32b..e7b44a52f3 100644 --- a/sei-cosmos/server/config/config_test.go +++ b/sei-cosmos/server/config/config_test.go @@ -142,6 +142,15 @@ func TestValidateBasic(t *testing.T) { } } +func TestGetConfigRejectsNegativeFreezeHeight(t *testing.T) { + v := viper.New() + v.Set("telemetry.global-labels", []interface{}{}) + v.Set("freeze-height", -1) + + _, err := GetConfig(v) + require.Error(t, err) +} + func TestGetMinGasPrices(t *testing.T) { tests := []struct { name string diff --git a/sei-cosmos/server/config/freeze_test.go b/sei-cosmos/server/config/freeze_test.go new file mode 100644 index 0000000000..8e3dcbc0c1 --- /dev/null +++ b/sei-cosmos/server/config/freeze_test.go @@ -0,0 +1,39 @@ +package config + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidateFreezeBackport(t *testing.T) { + tests := []struct { + name string + freezeHeight uint64 + haltHeight uint64 + haltTime uint64 + wantErr string + }{ + {name: "disabled"}, + {name: "enabled", freezeHeight: 100}, + {name: "height overflow", freezeHeight: uint64(math.MaxInt64) + 1, wantErr: "freeze-height must not exceed"}, + {name: "halt height", freezeHeight: 100, haltHeight: 100, wantErr: "cannot be combined"}, + {name: "halt time", freezeHeight: 100, haltTime: 100, wantErr: "cannot be combined"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := DefaultConfig() + cfg.FreezeHeight = tt.freezeHeight + cfg.HaltHeight = tt.haltHeight + cfg.HaltTime = tt.haltTime + err := cfg.ValidateFreeze() + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, tt.wantErr) + }) + } +} diff --git a/sei-cosmos/server/config/toml.go b/sei-cosmos/server/config/toml.go index ac931084be..5283bea538 100644 --- a/sei-cosmos/server/config/toml.go +++ b/sei-cosmos/server/config/toml.go @@ -41,6 +41,10 @@ occ-enabled = {{ .BaseConfig.OccEnabled }} # Note: Commitment of state will be attempted on the corresponding block. halt-height = {{ .BaseConfig.HaltHeight }} +# FreezeHeight contains a non-zero block height at which the node stops before +# executing the block while continuing to serve RPC. +freeze-height = {{ .BaseConfig.FreezeHeight }} + # HaltTime contains a non-zero minimum block time (in Unix seconds) at which # a node will gracefully halt and shutdown that can be used to assist upgrades # and testing. diff --git a/sei-cosmos/server/start.go b/sei-cosmos/server/start.go index 514c922cc7..93dd34e6e3 100644 --- a/sei-cosmos/server/start.go +++ b/sei-cosmos/server/start.go @@ -44,6 +44,7 @@ const ( flagCPUProfile = "cpu-profile" FlagMinGasPrices = "minimum-gas-prices" FlagHaltHeight = "halt-height" + FlagFreezeHeight = "freeze-height" FlagHaltTime = "halt-time" FlagInterBlockCache = "inter-block-cache" FlagUnsafeSkipUpgrades = "unsafe-skip-upgrades" @@ -102,6 +103,8 @@ the ABCI Commit phase, the node will check if the current block height is greate the halt-height or if the current block time is greater than or equal to the halt-time. If so, the node will attempt to gracefully shutdown and the block will not be committed. In addition, the node will not be able to commit subsequent blocks. +The '--freeze-height' flag instead keeps the process and RPC servers running while preventing block +sync and consensus from executing the block at the configured height or advancing beyond it. For profiling and benchmarking purposes, CPU profiling can be enabled via the '--cpu-profile' flag which accepts a path for the resulting pprof file. The node may be started in a 'query only' mode where only the gRPC and JSON HTTP @@ -211,6 +214,7 @@ func addStartNodeFlags(cmd *cobra.Command, defaultNodeHome string) { cmd.Flags().String(FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)") cmd.Flags().IntSlice(FlagUnsafeSkipUpgrades, []int{}, "Skip a set of upgrade heights to continue the old binary") cmd.Flags().Uint64(FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node") + cmd.Flags().Uint64(FlagFreezeHeight, 0, "Block height to stop before executing while continuing to serve RPC") cmd.Flags().Uint64(FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node") cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching") cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file") @@ -295,6 +299,13 @@ func startInProcess( if err != nil { return err } + if err := config.ValidateFreeze(); err != nil { + return err + } + gRPCOnly := ctx.Viper.GetBool(flagGRPCOnly) + if gRPCOnly && config.FreezeHeight > 0 { + return errors.New("freeze-height cannot be used with grpc-only mode") + } if err := config.ValidateBasic(ctx.Config); err != nil { logger.Error("WARNING: The minimum-gas-prices config in app.toml is set to the empty string. " + @@ -309,8 +320,6 @@ func startInProcess( } }() - gRPCOnly := ctx.Viper.GetBool(flagGRPCOnly) - var restartMtx sync.Mutex restartCh := make(chan struct{}) restartEvent := func() { @@ -351,6 +360,7 @@ func startInProcess( tracerProviderOptions, nodeMetricsProvider, tmtypes.DefaultConsensusPolicy(), + node.WithFreezeHeight(config.FreezeHeight), ) if err != nil { return fmt.Errorf("error creating node: %w", err) diff --git a/sei-tendermint/internal/blocksync/reactor.go b/sei-tendermint/internal/blocksync/reactor.go index 24db695a30..ebd4bbdd23 100644 --- a/sei-tendermint/internal/blocksync/reactor.go +++ b/sei-tendermint/internal/blocksync/reactor.go @@ -109,6 +109,7 @@ type SyncerConfig struct { EventBus *eventbus.EventBus RestartEvent func() SelfRemediationConfig *config.SelfRemediationConfig + FreezeHeight uint64 } // Reactor owns the blocksync channel and always-on query serving path, while @@ -156,6 +157,7 @@ type syncController struct { blocksBehindThreshold uint64 blocksBehindCheckInterval time.Duration restartCooldownSeconds uint64 + freezeHeight uint64 // blocksyncReady fires when the active sync routines should begin processing // work, either during OnStart or later via SwitchToBlockSync. @@ -189,6 +191,7 @@ func NewReactor( blocksBehindThreshold: cfg.SelfRemediationConfig.BlocksBehindThreshold, blocksBehindCheckInterval: time.Duration(cfg.SelfRemediationConfig.BlocksBehindCheckIntervalSeconds) * time.Second, //nolint:gosec // validated in config.ValidateBasic against MaxInt64 restartCooldownSeconds: cfg.SelfRemediationConfig.RestartCooldownSeconds, + freezeHeight: cfg.FreezeHeight, blocksyncReady: utils.NewAtomicSend(utils.None[blocksyncResult]()), startInBlockSync: cfg.BlockSync, } @@ -378,6 +381,9 @@ func (s *syncController) run(ctx context.Context) error { if r, ok := s.consReactor.Get(); ok { logger.Info("switching to consensus reactor", "height", handoff.height, "blocks_synced", handoff.blocksSynced, "state_synced", handoff.stateSynced, "max_peer_height", handoff.maxPeerHeight) r.SwitchToConsensus(handoff.state, handoff.blocksSynced > 0 || handoff.stateSynced) + if s.shouldFreeze(handoff.state) { + return nil + } s.autoRestartIfBehind(ctx, pool) } return nil @@ -465,6 +471,10 @@ func (s *syncController) requestRoutine(ctx context.Context, pool *BlockPool) er // // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initialState sm.State, stateSynced bool) (consensusHandoff, error) { + if handoff, frozen := s.frozenHandoff(pool, initialState, 0, stateSynced); frozen { + return handoff, nil + } + var ( trySyncTicker = time.NewTicker(trySyncIntervalMS * time.Millisecond) switchToConsensusTicker = time.NewTicker(switchToConsensusIntervalSeconds * time.Second) @@ -581,6 +591,9 @@ func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initi s.metrics.RecordConsMetrics(first) blocksSynced++ + if handoff, frozen := s.frozenHandoff(pool, state, blocksSynced, stateSynced); frozen { + return handoff, nil + } if blocksSynced%100 == 0 { lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) @@ -596,6 +609,26 @@ func (s *syncController) poolRoutine(ctx context.Context, pool *BlockPool, initi } } +func (s *syncController) frozenHandoff(pool *BlockPool, state sm.State, blocksSynced uint64, stateSynced bool) (consensusHandoff, bool) { + if !s.shouldFreeze(state) { + return consensusHandoff{}, false + } + height, _, _ := pool.GetStatus() + logger.Info("Block sync stopped before configured freeze height", "last_block_height", state.LastBlockHeight, "freeze_height", s.freezeHeight) + return consensusHandoff{ + state: state, + blocksSynced: blocksSynced, + stateSynced: stateSynced, + height: height, + maxPeerHeight: pool.MaxPeerHeight(), + }, true +} + +func (s *syncController) shouldFreeze(state sm.State) bool { + height := startHeightForState(state) + return s.freezeHeight > 0 && height >= 0 && uint64(height) >= s.freezeHeight //nolint:gosec // negative heights are rejected first. +} + // autoRestartIfBehind will check if the node is behind the max peer height by // a certain threshold. If it is, the node will attempt to restart itself. // TODO(gprusak): this should be a sub task of the consensus reactor instead. @@ -612,6 +645,10 @@ func (s *syncController) autoRestartIfBehind(ctx context.Context, pool *BlockPoo select { case <-time.After(s.blocksBehindCheckInterval): selfHeight := s.store.Height() + if s.freezeHeight > 0 && selfHeight >= 0 && uint64(selfHeight) >= s.freezeHeight-1 { //nolint:gosec // negative heights are rejected first. + logger.Info("Auto remediation stopped at configured freeze height", "selfHeight", selfHeight, "freeze_height", s.freezeHeight) + return + } maxPeerHeight := pool.MaxPeerHeight() threshold := int64(s.blocksBehindThreshold) //nolint:gosec // validated in config.ValidateBasic against MaxInt64 behindHeight := maxPeerHeight - selfHeight diff --git a/sei-tendermint/internal/blocksync/reactor_test.go b/sei-tendermint/internal/blocksync/reactor_test.go index 075c06f46a..d6e3dcdb42 100644 --- a/sei-tendermint/internal/blocksync/reactor_test.go +++ b/sei-tendermint/internal/blocksync/reactor_test.go @@ -6,6 +6,7 @@ import ( "runtime" "strings" "testing" + "testing/synctest" "time" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" @@ -27,6 +28,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/internal/store" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/test/factory" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + utilsrequire "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" pb "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/blocksync" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) @@ -618,6 +620,30 @@ func TestPoolRoutine_RetriesAfterValidationFailure(t *testing.T) { } } +func TestAutoRestartStopsAtFreezeBoundary(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + const freezeHeight = uint64(101) + mockBlockStore := new(MockBlockStore) + mockBlockStore.On("Height").Return(int64(freezeHeight - 1)) + + blockPool := &BlockPool{ + height: int64(freezeHeight), + maxPeerHeight: int64(freezeHeight + 100), + } + restart := utils.NewAtomicSend(false) + syncer := &syncController{ + store: mockBlockStore, + blocksBehindThreshold: 1, + blocksBehindCheckInterval: time.Hour, + freezeHeight: freezeHeight, + restartEvent: func() { restart.Store(true) }, + } + + syncer.autoRestartIfBehind(t.Context(), blockPool) + utilsrequire.False(t, restart.Load()) + }) +} + func TestQueryResponder_ServesBlockRequestsWhenBlockSyncDisabled(t *testing.T) { ctx := t.Context() @@ -667,6 +693,27 @@ func TestQueryResponder_ServesBlockRequestsWhenBlockSyncDisabled(t *testing.T) { t.Fatal("did not receive block response") } +func TestPoolRoutineHandsOffAtFreezeHeight(t *testing.T) { + const freezeHeight = int64(10) + pool := NewBlockPool(freezeHeight, nil) + syncer := &syncController{freezeHeight: uint64(freezeHeight)} //nolint:gosec // the test height is positive. + state := sm.State{InitialHeight: 1, LastBlockHeight: freezeHeight - 1} + + handoff, err := syncer.poolRoutine(t.Context(), pool, state, false) + if err != nil { + t.Fatalf("poolRoutine: %v", err) + } + if handoff.state.LastBlockHeight != freezeHeight-1 { + t.Fatalf("handoff state height = %d, want %d", handoff.state.LastBlockHeight, freezeHeight-1) + } + if handoff.height != freezeHeight { + t.Fatalf("handoff pool height = %d, want %d", handoff.height, freezeHeight) + } + if handoff.blocksSynced != 0 { + t.Fatalf("handoff blocks synced = %d, want 0", handoff.blocksSynced) + } +} + func TestQueryResponder_ServesStatusRequestsWhenBlockSyncDisabled(t *testing.T) { ctx := t.Context() diff --git a/sei-tendermint/internal/consensus/state.go b/sei-tendermint/internal/consensus/state.go index 37ded169d8..991e8f7177 100644 --- a/sei-tendermint/internal/consensus/state.go +++ b/sei-tendermint/internal/consensus/state.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "sync" + "sync/atomic" "time" "github.com/gogo/protobuf/proto" @@ -157,6 +158,9 @@ type State struct { heightSpan otrace.Span heightBeingTraced int64 tracingCtx context.Context + + freezeHeight uint64 + frozen atomic.Bool } // NewState returns a new State. @@ -283,6 +287,30 @@ func (cs *State) SetPrivValidator(ctx context.Context, priv utils.Option[types.P } } +// SetFreezeHeight configures the first block height consensus must not execute. +// It must be called before the state starts. +func (cs *State) SetFreezeHeight(height uint64) { + cs.freezeHeight = height + cs.markFrozen(nextHeightForState(cs.state), cs.state.LastBlockHeight) +} + +func (cs *State) markFrozen(nextHeight, lastBlockHeight int64) { + if cs.freezeHeight == 0 || nextHeight < 0 || uint64(nextHeight) < cs.freezeHeight { //nolint:gosec // negative heights are rejected first. + return + } + if cs.frozen.CompareAndSwap(false, true) { + logger.Info("Consensus frozen before configured height", "freeze_height", cs.freezeHeight, "last_block_height", lastBlockHeight) + } +} + +func nextHeightForState(state sm.State) int64 { + height := state.LastBlockHeight + 1 + if height == 1 { + height = state.InitialHeight + } + return height +} + // SetTimeoutTicker sets the local timer. It may be useful to overwrite for // testing. func (cs *State) SetTimeoutTicker(timeoutTicker TimeoutTicker) { @@ -323,14 +351,16 @@ func (cs *State) Run(ctx context.Context) error { // We may have lost some votes if the process crashed reload from consensus // log to catchup. - if cs.doWALCatchup { + if cs.doWALCatchup && !cs.frozen.Load() { if err := cs.catchupReplay(ctx, cs.roundState.Height()); err != nil { return fmt.Errorf("cs.catchupReplay(): %w", err) } } // Double Signing Risk Reduction - if err := cs.checkDoubleSigningRisk(cs.roundState.Height()); err != nil { - return err + if !cs.frozen.Load() { + if err := cs.checkDoubleSigningRisk(cs.roundState.Height()); err != nil { + return err + } } // now start the receiveRoutine @@ -339,7 +369,9 @@ func (cs *State) Run(ctx context.Context) error { // schedule the first round! // use GetRoundState so we don't race the receiveRoutine for access - cs.scheduleRound0(cs.GetRoundState()) + if !cs.frozen.Load() { + cs.scheduleRound0(cs.GetRoundState()) + } return nil }) } @@ -520,6 +552,9 @@ func (cs *State) votesFromSeenCommit(state sm.State) (*types.VoteSet, error) { // Updates State and increments height to match that of state. // The round becomes 0 and cs.Step becomes cstypes.RoundStepNewHeight. func (cs *State) updateToState(state sm.State) { + height := nextHeightForState(state) + cs.markFrozen(height, state.LastBlockHeight) + if cs.roundState.CommitRound() > -1 && 0 < cs.roundState.Height() && cs.roundState.Height() != state.LastBlockHeight { panic(fmt.Sprintf( "updateToState() expected state height of %v but found %v", @@ -584,12 +619,6 @@ func (cs *State) updateToState(state sm.State) { )) } - // Next desired block height - height := state.LastBlockHeight + 1 - if height == 1 { - height = state.InitialHeight - } - // RoundState fields cs.updateHeight(height) cs.updateRoundStep(0, cstypes.RoundStepNewHeight) @@ -633,8 +662,10 @@ func (cs *State) updateToState(state sm.State) { func (cs *State) newStep() { rs := cs.roundState.RoundStateEvent() - if err := cs.wal.Append(NewWALMessage(rs)); err != nil { - panic(fmt.Errorf("failed writing to WAL: %w", err)) + if !cs.frozen.Load() { + if err := cs.wal.Append(NewWALMessage(rs)); err != nil { + panic(fmt.Errorf("failed writing to WAL: %w", err)) + } } cs.nSteps++ @@ -700,6 +731,9 @@ func (cs *State) receiveRoutine(ctx context.Context, maxSteps int) error { } for { + if cs.frozen.Load() { + return cs.receiveWhileFrozen(ctx, txsAvailable) + } if maxSteps > 0 { if cs.nSteps >= maxSteps { logger.Debug("reached max steps; exiting receive routine") @@ -746,6 +780,20 @@ func (cs *State) receiveRoutine(ctx context.Context, maxSteps int) error { // TODO should we handle context cancels here? } } + +func (cs *State) receiveWhileFrozen(ctx context.Context, txsAvailable <-chan struct{}) error { + for { + select { + case <-txsAvailable: + case <-cs.peerMsgQueue: + case <-cs.internalMsgQueue: + case <-cs.timeoutTicker.Chan(): + case <-ctx.Done(): + return ctx.Err() + } + } +} + func (cs *State) fsyncAndCompleteProposal(ctx context.Context, fsyncUponCompletion bool, height int64, span otrace.Span, onPropose bool) { cs.metrics.ProposalBlockCreatedOnPropose.With("success", strconv.FormatBool(onPropose)).Add(1) if fsyncUponCompletion { @@ -974,6 +1022,9 @@ func (cs *State) getTracingCtx(defaultCtx context.Context) context.Context { // Enter: +2/3 prevotes any or +2/3 precommits for block or any from (height, round) // NOTE: cs.StartTime was already set for height. func (cs *State) enterNewRound(ctx context.Context, height int64, round int32, entryLabel string) { + if cs.frozen.Load() { + return + } if height > cs.heightBeingTraced { if cs.heightSpan != nil { cs.heightSpan.End() diff --git a/sei-tendermint/internal/consensus/state_test.go b/sei-tendermint/internal/consensus/state_test.go index 1eedfd7ecf..b0e03536ce 100644 --- a/sei-tendermint/internal/consensus/state_test.go +++ b/sei-tendermint/internal/consensus/state_test.go @@ -316,6 +316,49 @@ func TestStateFullRound1(t *testing.T) { cs.validateLastPrecommit(ctx, t, vss[0], propBlock.Hash) } +func TestStateFreezesAfterTargetBlock(t *testing.T) { + config := configSetup(t) + ctx := t.Context() + + cs, _ := makeState(ctx, t, makeStateArgs{config: config, validators: 1}) + height, round := cs.roundState.Height(), cs.roundState.Round() + cs.SetFreezeHeight(uint64(height + 1)) //nolint:gosec // consensus heights are non-negative. + require.False(t, cs.frozen.Load()) + + voteCh := subscribe(ctx, t, cs.eventBus, types.EventQueryVote) + proposalCh := subscribe(ctx, t, cs.eventBus, types.EventQueryCompleteProposal) + newRoundCh := subscribe(ctx, t, cs.eventBus, types.EventQueryNewRound) + frozenHeightCh := make(chan *cstypes.RoundState, 1) + cs.eventNewRoundStep = func(rs *cstypes.RoundState) { + if rs.Height == height+1 { + frozenHeightCh <- rs + } + } + + cs.startTestRound(ctx, height, round) + ensureNewRound(t, newRoundCh, height, round) + proposal := ensureNewProposal(t, proposalCh, height, round) + ensurePrevoteMatch(t, voteCh, height, round, proposal.Hash) + ensurePrecommit(t, voteCh, height, round) + + frozenState := <-frozenHeightCh + require.True(t, cs.frozen.Load()) + require.Equal(t, height+1, frozenState.Height) + require.Equal(t, cstypes.RoundStepNewHeight, frozenState.Step) + walHeight, walMessages, err := cs.wal.ReadLastHeightMsgs() + require.NoError(t, err) + require.Equal(t, height+1, walHeight) + require.Empty(t, walMessages) + + cs.enterNewRound(ctx, height+1, 0, "test") + require.Equal(t, cstypes.RoundStepNewHeight, cs.GetRoundState().Step) + select { + case event := <-newRoundCh: + t.Fatalf("consensus entered a round above the freeze height: %v", event) + default: + } +} + // nil is proposed, so prevote and precommit nil func TestStateFullRoundNil(t *testing.T) { config := configSetup(t) diff --git a/sei-tendermint/node/freeze_test.go b/sei-tendermint/node/freeze_test.go new file mode 100644 index 0000000000..12125881cf --- /dev/null +++ b/sei-tendermint/node/freeze_test.go @@ -0,0 +1,41 @@ +package node + +import ( + "math" + "testing" +) + +func TestValidateFreezeHeight(t *testing.T) { + for _, tc := range []struct { + name string + freezeHeight uint64 + initialHeight int64 + stateHeight int64 + blockHeight int64 + appHeight int64 + wantErr bool + }{ + {name: "disabled"}, + {name: "below target", freezeHeight: 10, initialHeight: 1, stateHeight: 8, blockHeight: 9, appHeight: 8}, + {name: "immediately before target", freezeHeight: 10, initialHeight: 1, stateHeight: 9, blockHeight: 9, appHeight: 9}, + {name: "target below initial height", freezeHeight: 9, initialHeight: 10, wantErr: true}, + {name: "state at target", freezeHeight: 10, initialHeight: 1, stateHeight: 10, wantErr: true}, + {name: "block store at target", freezeHeight: 10, initialHeight: 1, blockHeight: 10, wantErr: true}, + {name: "application at target", freezeHeight: 10, initialHeight: 1, appHeight: 10, wantErr: true}, + {name: "target above max height", freezeHeight: uint64(math.MaxInt64) + 1, initialHeight: 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := validateFreezeHeight(tc.freezeHeight, tc.initialHeight, tc.stateHeight, tc.blockHeight, tc.appHeight) + if (err != nil) != tc.wantErr { + t.Fatalf("validateFreezeHeight() error = %v, wantErr %t", err, tc.wantErr) + } + }) + } +} + +func TestWithFreezeHeight(t *testing.T) { + const height = uint64(123) + if got := resolveOptions(WithFreezeHeight(height)).freezeHeight; got != height { + t.Fatalf("freeze height = %d, want %d", got, height) + } +} diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index c834cf8f11..2e74d9b133 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -2,7 +2,9 @@ package node import ( "context" + "errors" "fmt" + "math" "net" "net/http" _ "net/http/pprof" // nolint: gosec // securely exposed on separate, optional port @@ -13,6 +15,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "go.opentelemetry.io/otel/sdk/trace" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/config" "github.com/sei-protocol/sei-chain/sei-tendermint/crypto" @@ -44,6 +47,31 @@ import ( _ "github.com/lib/pq" // provide the psql db driver ) +func validateFreezeHeight(freezeHeight uint64, initialHeight, stateHeight, blockStoreHeight, appHeight int64) error { + if freezeHeight == 0 { + return nil + } + if freezeHeight > math.MaxInt64 { + return fmt.Errorf("freeze height %d exceeds the maximum block height", freezeHeight) + } + if initialHeight > int64(freezeHeight) { //nolint:gosec // freezeHeight is bounded above. + return fmt.Errorf("freeze height %d is below initial height %d", freezeHeight, initialHeight) + } + for _, current := range []struct { + source string + height int64 + }{ + {source: "application", height: appHeight}, + {source: "block store", height: blockStoreHeight}, + {source: "state store", height: stateHeight}, + } { + if current.height >= int64(freezeHeight) { //nolint:gosec // freezeHeight is bounded above. + return fmt.Errorf("%s height %d has already reached freeze height %d", current.source, current.height, freezeHeight) + } + } + return nil +} + // nodeImpl is the highest level interface to a full Tendermint node. // It includes all configuration information and running services. type nodeImpl struct { @@ -55,6 +83,7 @@ type nodeImpl struct { privValidator types.PrivValidator // local node's validator key shouldHandshake bool // set during makeNode consensusPolicy types.ConsensusPolicy + freezeHeight uint64 // network router *p2p.Router @@ -90,7 +119,9 @@ func makeNode( tracerProviderOptions []trace.TracerProviderOption, nodeMetrics *NodeMetrics, consensusPolicy types.ConsensusPolicy, + nodeOptions ...Option, ) (_ local.NodeService, err error) { + opts := resolveOptions(nodeOptions...) var cancel context.CancelFunc ctx, cancel = context.WithCancel(ctx) closers := []closer{convertCancelCloser(cancel)} @@ -120,6 +151,18 @@ func makeNode( if err != nil { return nil, fmt.Errorf("LoadStateFromDBOrGenesisDocProvider(): %w", err) } + if opts.freezeHeight > 0 { + info, err := proxyApp.Info(ctx, &abci.RequestInfo{}) + if err != nil { + return nil, err + } + if err := validateFreezeHeight(opts.freezeHeight, genDoc.InitialHeight, state.LastBlockHeight, blockStore.Height(), info.LastBlockHeight); err != nil { + return nil, err + } + if cfg.AutobahnConfigFile != "" { + return nil, errors.New("freeze height is not supported with Autobahn") + } + } eventBus := eventbus.NewDefault() @@ -168,6 +211,7 @@ func makeNode( genesisDoc: genDoc, privValidator: privValidator, consensusPolicy: consensusPolicy, + freezeHeight: opts.freezeHeight, nodeKey: nodeKey, @@ -260,6 +304,10 @@ func makeNode( // Determine whether we should attempt state sync. stateSync := cfg.StateSync.Enable && !onlyValidatorIsUs(state, pubKey) + if stateSync && opts.freezeHeight > 0 { + logger.Info("Freeze mode disables state sync; falling back to block sync", "freeze_height", opts.freezeHeight) + stateSync = false + } if stateSync && state.LastBlockHeight > 0 { logger.Info("Found local state with non-zero height, skipping state sync") stateSync = false @@ -289,6 +337,7 @@ func makeNode( tracerProviderOptions, nodeMetrics.consensus, ) + csState.SetFreezeHeight(opts.freezeHeight) node.rpcEnv.ConsensusState = utils.Some[rpccore.ConsensusState](csState) csReactor, err := consensus.NewReactor( @@ -320,6 +369,7 @@ func makeNode( EventBus: eventBus, RestartEvent: restartEvent, SelfRemediationConfig: cfg.SelfRemediation, + FreezeHeight: opts.freezeHeight, }), ) if err != nil { @@ -415,6 +465,9 @@ func makeNode( // OnStart starts the Node. It implements service.Service. func (n *nodeImpl) OnStart(ctx context.Context) error { + if n.freezeHeight > 0 { + logger.Info("Freeze mode enabled", "freeze_height", n.freezeHeight) + } // EventBus and IndexerService must be started before the handshake because // we might need to index the txs of the replayed block as this might not have happened // when the node stopped last time (i.e. the node stopped or crashed after it saved the block diff --git a/sei-tendermint/node/public.go b/sei-tendermint/node/public.go index 628e5e628b..9e69cfe416 100644 --- a/sei-tendermint/node/public.go +++ b/sei-tendermint/node/public.go @@ -17,11 +17,30 @@ import ( var logger = seilog.NewLogger("tendermint", "node") -// New constructs a tendermint node. The provided app runs in the same -// process as the tendermint node and will be wrapped in a local ABCI client -// inside this function. The final option is a pointer to a Genesis document: -// if the value is nil, the genesis document is read from the file specified -// in the config, and otherwise the node uses value of the final argument. +type options struct { + freezeHeight uint64 +} + +// Option configures optional node behavior. +type Option func(*options) + +// WithFreezeHeight stops block sync and consensus before executing height; 0 disables freezing. +func WithFreezeHeight(height uint64) Option { + return func(opts *options) { + opts.freezeHeight = height + } +} + +func resolveOptions(nodeOptions ...Option) options { + var opts options + for _, apply := range nodeOptions { + apply(&opts) + } + return opts +} + +// New constructs a Tendermint node around an in-process ABCI application. +// A non-nil genesis document overrides the file selected by the node config. func New( ctx context.Context, conf *config.Config, @@ -31,6 +50,7 @@ func New( tracerProviderOptions []trace.TracerProviderOption, nodeMetrics *NodeMetrics, consensusPolicy tmtypes.ConsensusPolicy, + nodeOptions ...Option, ) (local.NodeService, error) { proxyApp := proxy.New(app, nodeMetrics.proxy) nodeKey, err := tmtypes.LoadOrGenNodeKey(conf.NodeKeyFile()) @@ -65,8 +85,12 @@ func New( tracerProviderOptions, nodeMetrics, consensusPolicy, + nodeOptions..., ) case config.ModeSeed: + if resolveOptions(nodeOptions...).freezeHeight > 0 { + return nil, fmt.Errorf("freeze height is not supported in seed mode") + } return makeSeedNode( conf, config.DefaultDBProvider,