diff --git a/CHANGELOG.md b/CHANGELOG.md index c1eec8e21c..cb91aad9b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,11 +30,15 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## Unreleased +### Features +* [#3948](https://github.com/sei-protocol/sei-chain/pull/3948) feat(query): origin-aware pagination scan limits on the ABCI/gRPC query path. Untrusted callers are capped at 10k store entries per paginator scan, and `limit` / `offset` above that cap are rejected before iteration; operators can relax the cap for trusted origins via `[query].trusted-cidrs` and `[query].trusted-scan-limit` in `app.toml`. + ### Improvements * [#3818](https://github.com/sei-protocol/sei-chain/pull/3818) feat(evmrpc): extend HTTP admission control (`max_request_body_bytes`, `max_concurrent_request_bytes`, `ws_admission_timeout`) to the WebSocket plane (:8546). WS oversize frames close with WebSocket close code 1009; budget-wait timeouts return JSON-RPC error `-32005` before the connection closes. `evmrpc_requests_rejected_total` gains a `protocol` label (`http` / `ws`). ### Upgrade guide * [#3958](https://github.com/sei-protocol/sei-chain/pull/3958) **Feegrant removal.** Removes feegrant execution, module APIs, and the unreleased feegrant EVM precompile. The feegrant store remains mounted for historical state access. Transactions with a fee granter different from the payer are rejected. +* [#3948](https://github.com/sei-protocol/sei-chain/pull/3948) **Query pagination scan limits.** Upgraded nodes will not have a `[query]` section in `app.toml` until one is added. With the default empty `trusted-cidrs`, every ABCI/gRPC query origin — including the operator's own CLI over localhost — receives a 10k cap on paginated queries: `limit` and `offset` above 10k are rejected, and a paginator that scans more than 10k store entries (for example `count_total=true` over a large token list, or wide `Paginate`/`FilteredPaginate` keeper queries) returns `InvalidArgument` until the caller is trusted. **Operators running indexers or internal tooling that paginate large stores should add the caller's IP or CIDR to `query.trusted-cidrs` before or immediately after upgrade**, and may raise `query.trusted-scan-limit` (default `100000`; `0` = unlimited for trusted origins only). * **WebSocket frame size default drops from 10 MiB to 5 MiB.** Before this release, :8546 used a hardcoded 10 MiB frame cap. Both HTTP and WebSocket now share `[evm].max_request_body_bytes`, whose default is 5 MiB (`5242880`). WS clients that send frames in the 5-10 MiB range (large `eth_sendRawTransaction` batches, wide filter payloads, etc.) will be disconnected after upgrade unless the limit is raised. **Operators who relied on the old 10 MiB WS cap should set `max_request_body_bytes = 10485760` in `app.toml` before upgrading.** This also raises the HTTP body limit to 10 MiB. The exported `DefaultWebsocketMaxMessageSize` constant was removed; use the config knob instead. * [#3927](https://github.com/sei-protocol/sei-chain/pull/3927) **Legacy Sei JSON-RPC and CLI removal.** Removes `sei_associate`, `sei_getBlockByHash`, `sei_getBlockByHashExcludeTraceFail`, `sei_getBlockTransactionCountByHash`, `sei_getBlockTransactionCountByNumber`, `sei_getEvmTx`, `sei_getFilterChanges`, `sei_getFilterLogs`, `sei_getLogs`, `sei_getTransactionByBlockHashAndIndex`, `sei_getTransactionByBlockNumberAndIndex`, `sei_getTransactionByHash`, `sei_getTransactionCount`, `sei_getTransactionErrorByHash`, `sei_getTransactionReceiptExcludeTraceFail`, `sei_getVMError`, `sei_newBlockFilter`, `sei_newFilter`, `sei_sign`, and `sei_uninstallFilter`. Use standard `eth_*` methods for EVM-originated data and `seid tx evm native-associate -y` for address association. There is no block- or filter-level replacement for discovering Cosmos-originated synthetic logs; clients that know the synthetic transaction hash can enable `sei_getTransactionReceipt`. diff --git a/sei-cosmos/baseapp/abci.go b/sei-cosmos/baseapp/abci.go index d6d4f6662b..9458be92fd 100644 --- a/sei-cosmos/baseapp/abci.go +++ b/sei-cosmos/baseapp/abci.go @@ -462,7 +462,7 @@ func (app *BaseApp) Query(ctx context.Context, req *abci.RequestQuery) (res *abc // handle gRPC routes first rather than calling splitPath because '/' characters // are used as part of gRPC paths if grpcHandler := app.grpcQueryRouter.Route(req.Path); grpcHandler != nil { - resp := app.handleQueryGRPC(grpcHandler, *req) + resp := app.handleQueryGRPC(ctx, grpcHandler, *req) return &resp, nil } @@ -623,15 +623,15 @@ func (app *BaseApp) ApplySnapshotChunk(context context.Context, req *abci.Reques } } -func (app *BaseApp) handleQueryGRPC(handler GRPCQueryHandler, req abci.RequestQuery) abci.ResponseQuery { - ctx, err := app.CreateQueryContext(req.Height, req.Prove) +func (app *BaseApp) handleQueryGRPC(ctx context.Context, handler GRPCQueryHandler, req abci.RequestQuery) abci.ResponseQuery { + sdkCtx, err := app.CreateQueryContext(req.Height, req.Prove) if err != nil { return sdkerrors.QueryResultWithDebug(err, app.trace) } // Only Cosmos ABCI gRPC queries may use client-facing pagination semantics. // Historical EVM RPC also calls CreateQueryContext and must remain unmarked. - res, err := handler(ctx.WithIsABCIQuery(true), req) + res, err := handler(app.enrichABCIQueryContext(ctx, sdkCtx), req) if err != nil { res = sdkerrors.QueryResultWithDebug(gRPCErrorToSDKError(err), app.trace) res.Height = req.Height diff --git a/sei-cosmos/baseapp/baseapp.go b/sei-cosmos/baseapp/baseapp.go index 34043eb30b..a1ecc5fecf 100644 --- a/sei-cosmos/baseapp/baseapp.go +++ b/sei-cosmos/baseapp/baseapp.go @@ -184,6 +184,9 @@ type BaseApp struct { concurrencyWorkers int occEnabled bool + queryConfig config.QueryConfig + trustedOriginMatcher *trustedCIDRMatcher + deliverTxHooks []DeliverTxHook execProcessProposalMs int64 @@ -320,6 +323,15 @@ func NewBaseApp( app.concurrencyWorkers = config.DefaultConcurrencyWorkers } + queryCfg, err := readQueryConfig(appOpts) + if err != nil { + panic(err) + } + warnQueryConfig(queryCfg) + matcher := newTrustedCIDRMatcher(queryCfg.TrustedCIDRs) + app.queryConfig = queryCfg + app.trustedOriginMatcher = matcher + return app } diff --git a/sei-cosmos/baseapp/config_fuzz_test.go b/sei-cosmos/baseapp/config_fuzz_test.go index c6a5dda3dc..6b42b714e6 100644 --- a/sei-cosmos/baseapp/config_fuzz_test.go +++ b/sei-cosmos/baseapp/config_fuzz_test.go @@ -584,3 +584,57 @@ func panicsNot(t *testing.T, fn func()) (ok bool) { fn() return true } + +// queryKeys covers the [query] keys readQueryConfig reads during BaseApp construction. +// +// Spelled as literals rather than through FlagQueryTrustedCIDRs so a constant rename fails +// CheckKeyNames rather than moving the row with the reader. Both reads are guarded and checked, +// matching ParseQueryConfig in sei-cosmos/server/config, but this manifest describes this reader. +var queryKeys = []configtest.KeySpec{ + { + Key: "query.trusted-cidrs", Path: "TrustedCIDRs", Cast: configtest.CastStringSlice, Checked: true, + Why: "CIDR allowlist for relaxed query scan limits; empty means fail closed", + }, + { + Key: "query.trusted-scan-limit", Path: "TrustedScanLimit", Cast: configtest.CastUint64, Checked: true, + Why: "max store entries a trusted-origin paginator may scan", + }, +} + +func readBaseAppQuery(opts configtest.AppOpts) (any, error) { + return readQueryConfig(opts) +} + +func FuzzBaseAppQueryConfig(f *testing.F) { + seeds := configtest.NewSeeds(f, fuzzing.ConfigValue) + for i := range len(queryKeys) { + seeds.AddRow(uint(i), fuzzing.KindNil, "", int64(0), false) + seeds.AddRow(uint(i), fuzzing.KindString, "not-a-value", int64(0), false) + seeds.AddRow(uint(i), fuzzing.KindMap, "", int64(0), false) + } + seeds.AddRow(uint(0), fuzzing.KindStringSlice, "127.0.0.1/32", int64(0), false) + seeds.AddRow(uint(1), fuzzing.KindInt64, "", int64(250_000), false) + + configtest.CheckEveryRowHasADiscriminatingSeed(f, "query", readBaseAppQuery, queryKeys, seeds) + + f.Fuzz(func(t *testing.T, keyIdx uint, kind uint8, s string, n int64, b bool) { + spec := configtest.Pick(queryKeys, keyIdx) + configtest.CheckRow(t, "query", readBaseAppQuery, spec, fuzzing.ConfigValue(kind, s, n, b)) + }) +} + +func TestBaseAppQueryAbsentKeysKeepDefaults(t *testing.T) { + configtest.CheckAbsent(t, "query", readBaseAppQuery, config.DefaultQueryConfig()) +} + +func TestBaseAppQueryDefaultsMatchTheRecordedValues(t *testing.T) { + configtest.CheckDefaults(t, "query", config.DefaultQueryConfig()) +} + +func TestBaseAppQueryKeyNamesMatchTheRecordedNames(t *testing.T) { + configtest.CheckKeyNames(t, "query", queryKeys) +} + +func TestBaseAppQueryManifestNamesEveryField(t *testing.T) { + configtest.CheckManifestCoversEveryField(t, "query", config.DefaultQueryConfig(), queryKeys) +} diff --git a/sei-cosmos/baseapp/grpcserver.go b/sei-cosmos/baseapp/grpcserver.go index c7900e0b68..d00a56a6e3 100644 --- a/sei-cosmos/baseapp/grpcserver.go +++ b/sei-cosmos/baseapp/grpcserver.go @@ -59,7 +59,7 @@ func (app *BaseApp) RegisterGRPCServer(server gogogrpc.Server) { // Direct Cosmos gRPC queries may use client-facing pagination semantics. // Other CreateQueryContext consumers remain v6.6-compatible by default. - sdkCtx = sdkCtx.WithIsABCIQuery(true) + sdkCtx = app.enrichABCIQueryContext(grpcCtx, sdkCtx) grpcCtx = context.WithValue(grpcCtx, sdk.SdkContextKey, sdkCtx) md = metadata.Pairs(grpctypes.GRPCBlockHeightHeader, strconv.FormatInt(height, 10)) diff --git a/sei-cosmos/baseapp/query_trust.go b/sei-cosmos/baseapp/query_trust.go new file mode 100644 index 0000000000..1aae5f2cad --- /dev/null +++ b/sei-cosmos/baseapp/query_trust.go @@ -0,0 +1,131 @@ +package baseapp + +import ( + "context" + "fmt" + "net" + "strings" + + srvconfig "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" + servertypes "github.com/sei-protocol/sei-chain/sei-cosmos/server/types" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/types/query" + rpctypes "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/jsonrpc/types" + "github.com/spf13/cast" + "google.golang.org/grpc/peer" +) + +const ( + FlagQueryTrustedCIDRs = "query.trusted-cidrs" + FlagQueryTrustedScanLimit = "query.trusted-scan-limit" +) + +type trustedCIDRMatcher struct { + networks []*net.IPNet +} + +// newTrustedCIDRMatcher returns a matcher for parseable entries in cidrs, skipping the rest. +// warnQueryConfig should run before this so skipped entries are logged. +func newTrustedCIDRMatcher(cidrs []string) *trustedCIDRMatcher { + networks := make([]*net.IPNet, 0, len(cidrs)) + for _, cidr := range cidrs { + _, network, err := net.ParseCIDR(cidr) + if err != nil { + continue + } + networks = append(networks, network) + } + return &trustedCIDRMatcher{networks: networks} +} + +func (m *trustedCIDRMatcher) contains(ipStr string) bool { + if m == nil { + return false + } + ip := net.ParseIP(stripHostPort(ipStr)) + if ip == nil { + return false + } + for _, network := range m.networks { + if network.Contains(ip) { + return true + } + } + return false +} + +func readQueryConfig(appOpts servertypes.AppOptions) (srvconfig.QueryConfig, error) { + cfg := srvconfig.DefaultQueryConfig() + var err error + + if v := appOpts.Get(FlagQueryTrustedCIDRs); v != nil { + if cfg.TrustedCIDRs, err = cast.ToStringSliceE(v); err != nil { + return cfg, fmt.Errorf("invalid %s: %w", FlagQueryTrustedCIDRs, err) + } + } + if v := appOpts.Get(FlagQueryTrustedScanLimit); v != nil { + if cfg.TrustedScanLimit, err = cast.ToUint64E(v); err != nil { + return cfg, fmt.Errorf("invalid %s: %w", FlagQueryTrustedScanLimit, err) + } + } + return cfg, nil +} + +func warnQueryConfig(cfg srvconfig.QueryConfig) { + for _, warning := range srvconfig.ValidateQueryConfig(cfg) { + logger.Warn(warning) + } +} + +func (app *BaseApp) enrichABCIQueryContext(ctx context.Context, sdkCtx sdk.Context) sdk.Context { + sdkCtx = sdkCtx.WithIsABCIQuery(true) + originIP := queryOriginIP(ctx) + trusted := app.trustedOriginMatcher != nil && app.trustedOriginMatcher.contains(originIP) + sdkCtx = sdkCtx.WithIsTrustedQueryOrigin(trusted) + + if trusted { + if app.queryConfig.TrustedScanLimit == 0 { + sdkCtx = sdkCtx.WithQueryScanLimit(false, 0) + } else { + sdkCtx = sdkCtx.WithQueryScanLimit(true, app.queryConfig.TrustedScanLimit) + } + logger.Debug( + "query pagination using trusted scan limit", + "origin", originIP, + "limit", app.queryConfig.TrustedScanLimit, + ) + return sdkCtx + } + + return sdkCtx.WithQueryScanLimit(true, query.MaxScanLimit) +} + +func queryOriginIP(ctx context.Context) string { + if callInfo := rpctypes.GetCallInfo(ctx); callInfo != nil { + if addr := callInfo.RemoteAddr(); addr != "" { + return addr + } + } + if p, ok := peer.FromContext(ctx); ok && p.Addr != nil { + return p.Addr.String() + } + return "" +} + +func stripHostPort(addr string) string { + if addr == "" { + return "" + } + if strings.HasPrefix(addr, "[") { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return strings.Trim(addr, "[]") + } + return host + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return addr + } + return host +} diff --git a/sei-cosmos/baseapp/query_trust_test.go b/sei-cosmos/baseapp/query_trust_test.go new file mode 100644 index 0000000000..f3dd7f81a6 --- /dev/null +++ b/sei-cosmos/baseapp/query_trust_test.go @@ -0,0 +1,81 @@ +package baseapp + +import ( + "net" + "testing" + + srvconfig "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/testutil/configtest" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/peer" +) + +func TestTrustedCIDRMatcher(t *testing.T) { + matcher := newTrustedCIDRMatcher([]string{"127.0.0.1/32", "10.0.0.0/8"}) + require.True(t, matcher.contains("127.0.0.1:54321")) + require.True(t, matcher.contains("10.1.2.3")) + require.False(t, matcher.contains("203.0.113.1")) +} + +func TestTrustedCIDRMatcherSkipsInvalidEntries(t *testing.T) { + matcher := newTrustedCIDRMatcher([]string{"not-a-cidr", "127.0.0.1/32"}) + require.True(t, matcher.contains("127.0.0.1")) + require.False(t, matcher.contains("203.0.113.1")) +} + +func TestQueryOriginIPFromGRPCPeer(t *testing.T) { + addr, err := net.ResolveTCPAddr("tcp", "192.0.2.1:9090") + require.NoError(t, err) + ctx := peer.NewContext(t.Context(), &peer.Peer{Addr: addr}) + require.Equal(t, "192.0.2.1:9090", queryOriginIP(ctx)) +} + +func TestValidateQueryConfigWarnsOnBroadCIDR(t *testing.T) { + warnings := srvconfig.ValidateQueryConfig(srvconfig.QueryConfig{ + TrustedCIDRs: []string{"0.0.0.0/0"}, + }) + require.Len(t, warnings, 1) + require.Contains(t, warnings[0], "overly broad") +} + +func TestStripHostPort(t *testing.T) { + require.Equal(t, "127.0.0.1", stripHostPort("127.0.0.1:9090")) + require.Equal(t, "2001:db8::1", stripHostPort("[2001:db8::1]:9090")) +} + +func TestEnrichABCIQueryContextTrustedOriginUnlimitedScan(t *testing.T) { + app := newTestBaseApp(t, configtest.AppOpts{ + FlagChainID: "sei-test", + FlagQueryTrustedCIDRs: []string{"127.0.0.1/32"}, + FlagQueryTrustedScanLimit: uint64(0), + }) + + addr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:9090") + require.NoError(t, err) + grpcCtx := peer.NewContext(t.Context(), &peer.Peer{Addr: addr}) + + sdkCtx := app.enrichABCIQueryContext(grpcCtx, sdk.Context{}) + require.True(t, sdkCtx.IsABCIQuery()) + require.True(t, sdkCtx.IsTrustedQueryOrigin()) + require.False(t, sdkCtx.EnforceQueryScanLimit()) +} + +func TestEnrichABCIQueryContextTrustedOriginUsesConfiguredLimit(t *testing.T) { + const trustedLimit = uint64(250_000) + + app := newTestBaseApp(t, configtest.AppOpts{ + FlagChainID: "sei-test", + FlagQueryTrustedCIDRs: []string{"10.0.0.0/8"}, + FlagQueryTrustedScanLimit: trustedLimit, + }) + + addr, err := net.ResolveTCPAddr("tcp", "10.1.2.3:9090") + require.NoError(t, err) + grpcCtx := peer.NewContext(t.Context(), &peer.Peer{Addr: addr}) + + sdkCtx := app.enrichABCIQueryContext(grpcCtx, sdk.Context{}) + require.True(t, sdkCtx.IsTrustedQueryOrigin()) + require.True(t, sdkCtx.EnforceQueryScanLimit()) + require.Equal(t, trustedLimit, sdkCtx.QueryScanLimit()) +} diff --git a/sei-cosmos/baseapp/testdata/query.golden b/sei-cosmos/baseapp/testdata/query.golden new file mode 100644 index 0000000000..0506f2529f --- /dev/null +++ b/sei-cosmos/baseapp/testdata/query.golden @@ -0,0 +1,2 @@ +TrustedCIDRs = +TrustedScanLimit = uint64(100000) diff --git a/sei-cosmos/baseapp/testdata/query.keys.golden b/sei-cosmos/baseapp/testdata/query.keys.golden new file mode 100644 index 0000000000..c9de93fb8e --- /dev/null +++ b/sei-cosmos/baseapp/testdata/query.keys.golden @@ -0,0 +1,3 @@ +"query.trusted-cidrs" +"query.trusted-scan-limit" +# keys with a target of their own diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index 8133fe33c9..eaa8851898 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -312,6 +312,7 @@ type Config struct { StateCommit config.StateCommitConfig `mapstructure:"state-commit"` StateStore config.StateStoreConfig `mapstructure:"state-store"` Genesis GenesisConfig `mapstructure:"genesis"` + Query QueryConfig `mapstructure:"query"` } // SetMinGasPrices sets the validator's minimum gas prices. @@ -409,6 +410,7 @@ func DefaultConfig() *Config { StreamImport: false, GenesisStreamFile: "", }, + Query: DefaultQueryConfig(), } } @@ -569,7 +571,7 @@ func GetConfig(v *viper.Viper) (Config, error) { grpcMaxConnectionAge := clampNonNegativeDuration(v.GetDuration("grpc.max-connection-age"), DefaultGRPCMaxConnectionAge) grpcMaxConnectionAgeGrace := clampNonNegativeDuration(v.GetDuration("grpc.max-connection-age-grace"), DefaultGRPCMaxConnectionAgeGrace) - return Config{ + cfg := Config{ BaseConfig: BaseConfig{ MinGasPrices: v.GetString("minimum-gas-prices"), InterBlockCache: v.GetBool("inter-block-cache"), @@ -664,7 +666,15 @@ func GetConfig(v *viper.Viper) (Config, error) { StreamImport: v.GetBool("genesis.stream-import"), GenesisStreamFile: v.GetString("genesis.genesis-stream-file"), }, - }, nil + } + + queryCfg, err := ParseQueryConfig(v) + if err != nil { + return Config{}, err + } + cfg.Query = queryCfg + + return cfg, nil } // ValidateBasic validates the server configuration. diff --git a/sei-cosmos/server/config/config_fuzz_test.go b/sei-cosmos/server/config/config_fuzz_test.go index 48ec29b7a1..dce59b931e 100644 --- a/sei-cosmos/server/config/config_fuzz_test.go +++ b/sei-cosmos/server/config/config_fuzz_test.go @@ -743,6 +743,7 @@ func TestDefaultsMatchTheRecordedValues(t *testing.T) { // lines here instead of finding them among two hundred. Regenerating one of the two records without // the other leaves that other one red. configtest.CheckDefaults(t, "state-sync", DefaultConfig().StateSync) + configtest.CheckDefaults(t, "query", DefaultQueryConfig()) configtest.CheckDefaults(t, "server_config", DefaultConfig(), configtest.DerivedDefault{ @@ -961,6 +962,21 @@ var telemetryKeys = []configtest.KeySpec{ // is driven by targets rather than by a row. var telemetryKeysWithTargetsOfTheirOwn = []configtest.KeyName{"telemetry.global-labels"} +// queryKeys covers the [query] keys ParseQueryConfig reads through GetConfig. +// +// Both reads are guarded (v.IsSet) and checked (cast.ToXE), so an absent key keeps the in-code +// default and a malformed value fails boot with the key named. +var queryKeys = []configtest.KeySpec{ + { + Key: "query.trusted-cidrs", Path: "TrustedCIDRs", Cast: configtest.CastStringSlice, Checked: true, + Why: "CIDR allowlist for relaxed query scan limits; empty means fail closed", + }, + { + Key: "query.trusted-scan-limit", Path: "TrustedScanLimit", Cast: configtest.CastUint64, Checked: true, + Why: "max store entries a trusted-origin paginator may scan", + }, +} + func readRosetta(t testing.TB) func(configtest.AppOpts) (any, error) { return sectionOfGetConfig(t, func(c Config) any { return c.Rosetta }) } @@ -973,6 +989,10 @@ func readTelemetry(t testing.TB) func(configtest.AppOpts) (any, error) { return sectionOfGetConfig(t, func(c Config) any { return c.Telemetry }) } +func readQuery(t testing.TB) func(configtest.AppOpts) (any, error) { + return sectionOfGetConfig(t, func(c Config) any { return c.Query }) +} + // sectionOfGetConfig adapts GetConfig to the reader shape the checks take, for one section. // // One helper rather than a function per section, because every one of these differs only in which @@ -1053,6 +1073,24 @@ func FuzzTelemetryConfig(f *testing.F) { }) } +func FuzzQueryConfig(f *testing.F) { + seeds := configtest.NewSeeds(f, fuzzing.ConfigValue) + for i := range len(queryKeys) { + seeds.AddRow(uint(i), fuzzing.KindNil, "", int64(0), false) + seeds.AddRow(uint(i), fuzzing.KindString, "not-a-value", int64(0), false) + seeds.AddRow(uint(i), fuzzing.KindMap, "", int64(0), false) + } + seeds.AddRow(uint(0), fuzzing.KindStringSlice, "127.0.0.1/32", int64(0), false) + seeds.AddRow(uint(1), fuzzing.KindInt64, "", int64(250_000), false) + + configtest.CheckEveryRowHasADiscriminatingSeed(f, "query", readQuery(f), queryKeys, seeds) + + f.Fuzz(func(t *testing.T, keyIdx uint, kind uint8, s string, n int64, b bool) { + spec := configtest.Pick(queryKeys, keyIdx) + configtest.CheckRow(t, "query", readQuery(t), spec, fuzzing.ConfigValue(kind, s, n, b)) + }) +} + func TestRosettaKeyNamesMatchTheRecordedNames(t *testing.T) { configtest.CheckKeyNames(t, "rosetta", rosettaKeys) } @@ -1065,6 +1103,10 @@ func TestTelemetryKeyNamesMatchTheRecordedNames(t *testing.T) { configtest.CheckKeyNames(t, "telemetry", telemetryKeys, telemetryKeysWithTargetsOfTheirOwn...) } +func TestQueryKeyNamesMatchTheRecordedNames(t *testing.T) { + configtest.CheckKeyNames(t, "query", queryKeys) +} + func TestRosettaManifestNamesEveryField(t *testing.T) { configtest.CheckManifestCoversEveryField(t, "rosetta", DefaultConfig().Rosetta, rosettaKeys) } @@ -1080,6 +1122,15 @@ func TestTelemetryManifestNamesEveryField(t *testing.T) { ) } +func TestQueryManifestNamesEveryField(t *testing.T) { + configtest.CheckManifestCoversEveryField(t, "query", DefaultQueryConfig(), queryKeys) +} + +// TestQueryAbsentKeysKeepDefaults pins the [query] section baseline. +func TestQueryAbsentKeysKeepDefaults(t *testing.T) { + configtest.CheckAbsent(t, "query", readQuery(t), DefaultQueryConfig()) +} + // TestGetConfigAbsentSectionDivergences records every field these sections resolve away from its // declared default when the section is absent from app.toml. // diff --git a/sei-cosmos/server/config/query.go b/sei-cosmos/server/config/query.go new file mode 100644 index 0000000000..2d72718199 --- /dev/null +++ b/sei-cosmos/server/config/query.go @@ -0,0 +1,76 @@ +package config + +import ( + "fmt" + "net" + + "github.com/spf13/cast" + "github.com/spf13/viper" +) + +const ( + DefaultTrustedScanLimit = uint64(100_000) +) + +// QueryConfig holds node-local query pagination settings. +type QueryConfig struct { + // TrustedCIDRs is a CIDR allowlist for relaxed scan limits. Empty means fail closed. + TrustedCIDRs []string `mapstructure:"trusted-cidrs"` + + // TrustedScanLimit is the max store entries a paginator may scan for trusted origins. + // Zero means unlimited. + TrustedScanLimit uint64 `mapstructure:"trusted-scan-limit"` +} + +// DefaultQueryConfig returns the default query configuration. +func DefaultQueryConfig() QueryConfig { + return QueryConfig{ + TrustedCIDRs: nil, + TrustedScanLimit: DefaultTrustedScanLimit, + } +} + +// ParseQueryConfig reads the [query] section from v. +func ParseQueryConfig(v *viper.Viper) (QueryConfig, error) { + cfg := DefaultQueryConfig() + if v == nil { + return cfg, nil + } + + if v.IsSet("query.trusted-cidrs") { + cidrs, err := cast.ToStringSliceE(v.Get("query.trusted-cidrs")) + if err != nil { + return cfg, fmt.Errorf("invalid query.trusted-cidrs: %w", err) + } + cfg.TrustedCIDRs = cidrs + } + + if v.IsSet("query.trusted-scan-limit") { + limit, err := cast.ToUint64E(v.Get("query.trusted-scan-limit")) + if err != nil { + return cfg, fmt.Errorf("invalid query.trusted-scan-limit: %w", err) + } + cfg.TrustedScanLimit = limit + } + + return cfg, nil +} + +// ValidateQueryConfig checks trusted CIDR entries for unsafe patterns. +func ValidateQueryConfig(cfg QueryConfig) []string { + var warnings []string + for _, cidr := range cfg.TrustedCIDRs { + _, network, err := net.ParseCIDR(cidr) + if err != nil { + warnings = append(warnings, fmt.Sprintf("query.trusted-cidrs contains invalid CIDR %q: %v", cidr, err)) + continue + } + if ones, _ := network.Mask.Size(); ones == 0 { + warnings = append(warnings, fmt.Sprintf( + "query.trusted-cidrs contains overly broad entry %q; public RPC callers will receive relaxed scan limits", + cidr, + )) + } + } + return warnings +} diff --git a/sei-cosmos/server/config/query_test.go b/sei-cosmos/server/config/query_test.go new file mode 100644 index 0000000000..a282d23992 --- /dev/null +++ b/sei-cosmos/server/config/query_test.go @@ -0,0 +1,31 @@ +package config + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestParseQueryConfigDefaults(t *testing.T) { + cfg, err := ParseQueryConfig(viper.New()) + require.NoError(t, err) + require.Equal(t, DefaultQueryConfig(), cfg) +} + +func TestParseQueryConfigOverrides(t *testing.T) { + v := viper.New() + v.Set("query.trusted-cidrs", []string{"127.0.0.1/32"}) + v.Set("query.trusted-scan-limit", 250_000) + + cfg, err := ParseQueryConfig(v) + require.NoError(t, err) + require.Equal(t, []string{"127.0.0.1/32"}, cfg.TrustedCIDRs) + require.Equal(t, uint64(250_000), cfg.TrustedScanLimit) +} + +func TestValidateQueryConfigRejectsInvalidCIDR(t *testing.T) { + warnings := ValidateQueryConfig(QueryConfig{TrustedCIDRs: []string{"not-a-cidr"}}) + require.Len(t, warnings, 1) + require.Contains(t, warnings[0], "invalid CIDR") +} diff --git a/sei-cosmos/server/config/testdata/query.golden b/sei-cosmos/server/config/testdata/query.golden new file mode 100644 index 0000000000..0506f2529f --- /dev/null +++ b/sei-cosmos/server/config/testdata/query.golden @@ -0,0 +1,2 @@ +TrustedCIDRs = +TrustedScanLimit = uint64(100000) diff --git a/sei-cosmos/server/config/testdata/query.keys.golden b/sei-cosmos/server/config/testdata/query.keys.golden new file mode 100644 index 0000000000..c9de93fb8e --- /dev/null +++ b/sei-cosmos/server/config/testdata/query.keys.golden @@ -0,0 +1,3 @@ +"query.trusted-cidrs" +"query.trusted-scan-limit" +# keys with a target of their own diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index ac1e300e25..03c1d0aab3 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -142,3 +142,5 @@ StateStore.EVMDBDirectory = string("") StateStore.SeparateEVMSubDBs = bool(false) Genesis.StreamImport = bool(false) Genesis.GenesisStreamFile = string("") +Query.TrustedCIDRs = +Query.TrustedScanLimit = uint64(100000) diff --git a/sei-cosmos/server/config/toml.go b/sei-cosmos/server/config/toml.go index 1c0ab40fd7..cf12c79ce7 100644 --- a/sei-cosmos/server/config/toml.go +++ b/sei-cosmos/server/config/toml.go @@ -85,6 +85,20 @@ snapshot-keep-recent = {{ .StateSync.SnapshotKeepRecent }} # snapshot-directory sets the directory for where state sync snapshots are persisted. # default is empty which will then store under the app home directory same as before. snapshot-directory = "{{ .StateSync.SnapshotDirectory }}" + +############################################################################### +### Query Configuration ### +############################################################################### + +[query] +# trusted-cidrs is a CIDR allowlist for relaxed pagination scan limits. +# Empty means every caller (including localhost) receives the 10k scan cap. +# Never list public ingress, load balancers, or grpc-gateway relays here. +trusted-cidrs = [{{- range $i, $c := .Query.TrustedCIDRs }}{{- if $i }}, {{ end }}"{{ $c }}"{{- end }}] + +# trusted-scan-limit is the max store entries a single paginator call may scan +# for trusted origins. 0 means unlimited. +trusted-scan-limit = {{ .Query.TrustedScanLimit }} ` // AutoManagedConfigTemplate contains configuration sections that are auto-managed diff --git a/sei-cosmos/types/context.go b/sei-cosmos/types/context.go index 7f2d881b96..280cb42539 100644 --- a/sei-cosmos/types/context.go +++ b/sei-cosmos/types/context.go @@ -26,32 +26,35 @@ but please do not over-use it. We try to keep all data structured and standard additions here would be better just to add to the Context struct */ type Context struct { - ctx context.Context - ms MultiStore - nextMs MultiStore // ms of the next height; only used in tracing - nextStoreKeys map[string]struct{} // store key names that should use nextMs - header tmproto.Header - headerHash tmbytes.HexBytes - chainID string - txBytes []byte - txSum [32]byte - voteInfo []abci.VoteInfo - gasMeter GasMeter - gasEstimate uint64 - occEnabled bool - blockGasMeter GasMeter - checkTx bool - recheckTx bool // if recheckTx == true, then checkTx must also be true - abciQuery bool // true only for BaseApp.Query; never transaction/block execution - isGenesis bool - minGasPrice DecCoins - consParams *tmproto.ConsensusParams - eventManager *EventManager - evmEventManager *EVMEventManager - priority int64 // The tx priority, only relevant in CheckTx - hasPriority bool // Whether the tx has a priority set - deliverTxCallback func(Context) // callback to make at the end of DeliverTx. - evmRequiredBalance *big.Int // Required sender balance for this EVM tx, only relevant in CheckTx. + ctx context.Context + ms MultiStore + nextMs MultiStore // ms of the next height; only used in tracing + nextStoreKeys map[string]struct{} // store key names that should use nextMs + header tmproto.Header + headerHash tmbytes.HexBytes + chainID string + txBytes []byte + txSum [32]byte + voteInfo []abci.VoteInfo + gasMeter GasMeter + gasEstimate uint64 + occEnabled bool + blockGasMeter GasMeter + checkTx bool + recheckTx bool // if recheckTx == true, then checkTx must also be true + abciQuery bool // true only for BaseApp.Query; never transaction/block execution + isTrustedQueryOrigin bool + enforceQueryScanLimit bool + queryScanLimit uint64 + isGenesis bool + minGasPrice DecCoins + consParams *tmproto.ConsensusParams + eventManager *EventManager + evmEventManager *EVMEventManager + priority int64 // The tx priority, only relevant in CheckTx + hasPriority bool // Whether the tx has a priority set + deliverTxCallback func(Context) // callback to make at the end of DeliverTx. + evmRequiredBalance *big.Int // Required sender balance for this EVM tx, only relevant in CheckTx. // EVM properties evm bool // EVM transaction flag @@ -139,6 +142,18 @@ func (c Context) IsABCIQuery() bool { return c.abciQuery } +func (c Context) IsTrustedQueryOrigin() bool { + return c.isTrustedQueryOrigin +} + +func (c Context) EnforceQueryScanLimit() bool { + return c.enforceQueryScanLimit +} + +func (c Context) QueryScanLimit() uint64 { + return c.queryScanLimit +} + func (c Context) IsGenesis() bool { return c.isGenesis } @@ -395,6 +410,17 @@ func (c Context) WithIsABCIQuery(isABCIQuery bool) Context { return c } +func (c Context) WithIsTrustedQueryOrigin(isTrustedQueryOrigin bool) Context { + c.isTrustedQueryOrigin = isTrustedQueryOrigin + return c +} + +func (c Context) WithQueryScanLimit(enforce bool, limit uint64) Context { + c.enforceQueryScanLimit = enforce + c.queryScanLimit = limit + return c +} + func (c Context) WithIsGenesis(isGenesis bool) Context { c.isGenesis = isGenesis return c diff --git a/sei-cosmos/types/query/filtered_pagination.go b/sei-cosmos/types/query/filtered_pagination.go index f78c6c6054..6ee6a91742 100644 --- a/sei-cosmos/types/query/filtered_pagination.go +++ b/sei-cosmos/types/query/filtered_pagination.go @@ -1,12 +1,9 @@ package query import ( - "fmt" - "github.com/sei-protocol/sei-chain/sei-cosmos/codec" "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" ) // FilteredPaginate does pagination of all the results in the PrefixStore based on the @@ -18,11 +15,26 @@ import ( // When accumulate is true, the current result should be appended to the result set returned // to the client. func FilteredPaginate( + ctx sdk.Context, + prefixStore types.KVStore, + pageRequest *PageRequest, + onResult func(key []byte, value []byte, accumulate bool) (bool, error), +) (*PageResponse, error) { + return filteredPaginate(prefixStore, pageRequest, onResult, scanLimitParamsFromContext(ctx)) +} + +// FilteredPaginateForContext applies FilteredPaginate on the ABCI query path and +// FilteredPaginateV66 during consensus execution. +func FilteredPaginateForContext( + ctx sdk.Context, prefixStore types.KVStore, pageRequest *PageRequest, onResult func(key []byte, value []byte, accumulate bool) (bool, error), ) (*PageResponse, error) { - return filteredPaginate(prefixStore, pageRequest, onResult, false) + if ctx.IsABCIQuery() { + return FilteredPaginate(ctx, prefixStore, pageRequest, onResult) + } + return FilteredPaginateV66(prefixStore, pageRequest, onResult) } // FilteredPaginateV66 preserves release/v6.6 behavior for EVM precompiles. @@ -36,138 +48,27 @@ func FilteredPaginateV66( pageRequest *PageRequest, onResult func(key []byte, value []byte, accumulate bool) (bool, error), ) (*PageResponse, error) { - return filteredPaginate(prefixStore, pageRequest, onResult, true) + return filteredPaginate(prefixStore, pageRequest, onResult, v66ScanLimitParams()) } func filteredPaginate( prefixStore types.KVStore, pageRequest *PageRequest, onResult func(key []byte, value []byte, accumulate bool) (bool, error), - enforceV66ScanLimit bool, + scanLimit scanLimitParams, ) (*PageResponse, error) { - // if the PageRequest is nil, use default PageRequest - if pageRequest == nil { - pageRequest = &PageRequest{} - } - - offset := pageRequest.Offset - key := pageRequest.Key - limit := pageRequest.Limit - countTotal := pageRequest.CountTotal - reverse := pageRequest.Reverse - - if offset > 0 && key != nil { - return nil, fmt.Errorf("invalid request, either offset or key is expected, got both") - } - - // Note: unlike upstream cosmos-sdk, limit == 0 must NOT implicitly enable - // countTotal; see the note in Paginate. - if limit == 0 { - limit = DefaultLimit - } - - if len(key) != 0 { - iterator := getIterator(prefixStore, key, reverse) - defer func() { _ = iterator.Close() }() - - var ( - numHits uint64 - nextKey []byte - totalIter uint64 - ) - - for ; iterator.Valid(); iterator.Next() { - totalIter++ - if numHits == limit { - nextKey = iterator.Key() - break - } - if enforceV66ScanLimit && totalIter > MaxScanLimit { - return nil, status.Errorf(codes.InvalidArgument, - "scanned more than %d entries without filling the page; use a more specific key prefix or reduce limit", MaxScanLimit) - } - - if iterator.Error() != nil { - return nil, iterator.Error() - } - - hit, err := onResult(iterator.Key(), iterator.Value(), true) - if err != nil { - return nil, err - } - - if hit { - numHits++ - } - } - - return &PageResponse{ - NextKey: nextKey, - }, nil - } - - iterator := getIterator(prefixStore, nil, reverse) - defer func() { _ = iterator.Close() }() - - end := paginationEnd(offset, limit) - var ( - numHits uint64 - nextKey []byte - totalIter uint64 - pageCompleteIter uint64 - ) - - for ; iterator.Valid(); iterator.Next() { - totalIter++ - if enforceV66ScanLimit && numHits < end && totalIter > paginationEnd(offset, MaxScanLimit) { - return nil, status.Errorf(codes.InvalidArgument, - "scanned more than %d entries without filling the page; use key-based pagination instead", MaxScanLimit) - } - if enforceV66ScanLimit && pageCompleteIter > MaxScanLimit { - if !countTotal { - break - } - return nil, status.Errorf(codes.InvalidArgument, - "scanned more than %d entries past the end of the page; use key-based pagination instead", MaxScanLimit) - } - - if iterator.Error() != nil { - return nil, iterator.Error() - } - - accumulate := numHits >= offset && numHits < end - hit, err := onResult(iterator.Key(), iterator.Value(), accumulate) - if err != nil { - return nil, err - } - - if hit { - numHits++ - } - - if numHits >= end { - pageCompleteIter++ - } - - if numHits > end { - // Only the first entry past the end of the page is the next key; - // do not overwrite it while scanning the remainder for countTotal. - if nextKey == nil { - nextKey = iterator.Key() - } - - if !countTotal { - break - } - } + req, err := preparePageRequest(pageRequest, scanLimit) + if err != nil { + return nil, err } - res := &PageResponse{NextKey: nextKey} - if countTotal { - res.Total = numHits + if req.useKey { + return runKeyPath(prefixStore, req, scanLimit, func(key, value []byte) (bool, error) { + return onResult(key, value, true) + }) } - return res, nil + return runOffsetPathFiltered(prefixStore, req, scanLimit, onResult) } // GenericFilteredPaginate does pagination of all the results in the PrefixStore based on the @@ -179,13 +80,30 @@ func filteredPaginate( // The resulting slice (of type F) can be of a different type than the one being iterated through // (type T), so it's possible to do any necessary transformation inside the onResult function. func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( + ctx sdk.Context, cdc codec.BinaryCodec, prefixStore types.KVStore, pageRequest *PageRequest, onResult func(key []byte, value T) (F, error), constructor func() T, ) ([]F, *PageResponse, error) { - return genericFilteredPaginate(cdc, prefixStore, pageRequest, onResult, constructor, false) + return genericFilteredPaginate(cdc, prefixStore, pageRequest, onResult, constructor, scanLimitParamsFromContext(ctx)) +} + +// GenericFilteredPaginateForContext applies GenericFilteredPaginate on the ABCI +// query path and GenericFilteredPaginateV66 during consensus execution. +func GenericFilteredPaginateForContext[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( + ctx sdk.Context, + cdc codec.BinaryCodec, + prefixStore types.KVStore, + pageRequest *PageRequest, + onResult func(key []byte, value T) (F, error), + constructor func() T, +) ([]F, *PageResponse, error) { + if ctx.IsABCIQuery() { + return GenericFilteredPaginate(ctx, cdc, prefixStore, pageRequest, onResult, constructor) + } + return GenericFilteredPaginateV66(cdc, prefixStore, pageRequest, onResult, constructor) } // GenericFilteredPaginateV66 preserves release/v6.6 behavior for EVM @@ -197,7 +115,7 @@ func GenericFilteredPaginateV66[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( onResult func(key []byte, value T) (F, error), constructor func() T, ) ([]F, *PageResponse, error) { - return genericFilteredPaginate(cdc, prefixStore, pageRequest, onResult, constructor, true) + return genericFilteredPaginate(cdc, prefixStore, pageRequest, onResult, constructor, v66ScanLimitParams()) } func genericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( @@ -206,148 +124,44 @@ func genericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler]( pageRequest *PageRequest, onResult func(key []byte, value T) (F, error), constructor func() T, - enforceV66ScanLimit bool, + scanLimit scanLimitParams, ) ([]F, *PageResponse, error) { - // if the PageRequest is nil, use default PageRequest - if pageRequest == nil { - pageRequest = &PageRequest{} + req, err := preparePageRequest(pageRequest, scanLimit) + if err != nil { + return nil, nil, err } - offset := pageRequest.Offset - key := pageRequest.Key - limit := pageRequest.Limit - countTotal := pageRequest.CountTotal - reverse := pageRequest.Reverse var results []F - if offset > 0 && key != nil { - return results, nil, fmt.Errorf("invalid request, either offset or key is expected, got both") - } - - // Note: unlike upstream cosmos-sdk, limit == 0 must NOT implicitly enable - // countTotal; see the note in Paginate. - if limit == 0 { - limit = DefaultLimit - } - - if len(key) != 0 { - iterator := getIterator(prefixStore, key, reverse) - defer func() { _ = iterator.Close() }() - - var ( - numHits uint64 - nextKey []byte - totalIter uint64 - ) - - for ; iterator.Valid(); iterator.Next() { - totalIter++ - if numHits == limit { - nextKey = iterator.Key() - break - } - if enforceV66ScanLimit && totalIter > MaxScanLimit { - return nil, nil, status.Errorf(codes.InvalidArgument, - "scanned more than %d entries without filling the page; use a more specific key prefix or reduce limit", MaxScanLimit) - } - - if iterator.Error() != nil { - return nil, nil, iterator.Error() - } - - protoMsg := constructor() - - err := cdc.Unmarshal(iterator.Value(), protoMsg) - if err != nil { - return nil, nil, err - } - - val, err := onResult(iterator.Key(), protoMsg) - if err != nil { - return nil, nil, err - } - - if val.Size() != 0 { - results = append(results, val) - numHits++ - } - } - - return results, &PageResponse{ - NextKey: nextKey, - }, nil - } - - iterator := getIterator(prefixStore, nil, reverse) - defer func() { _ = iterator.Close() }() - - end := paginationEnd(offset, limit) - var ( - numHits uint64 - nextKey []byte - totalIter uint64 - pageCompleteIter uint64 - ) - - for ; iterator.Valid(); iterator.Next() { - totalIter++ - if enforceV66ScanLimit && numHits < end && totalIter > paginationEnd(offset, MaxScanLimit) { - return nil, nil, status.Errorf(codes.InvalidArgument, - "scanned more than %d entries without filling the page; use key-based pagination instead", MaxScanLimit) - } - if enforceV66ScanLimit && pageCompleteIter > MaxScanLimit { - if !countTotal { - break - } - return nil, nil, status.Errorf(codes.InvalidArgument, - "scanned more than %d entries past the end of the page; use key-based pagination instead", MaxScanLimit) - } - - if iterator.Error() != nil { - return nil, nil, iterator.Error() - } - + appendResult := func(key []byte, value []byte, accumulate bool) (bool, error) { protoMsg := constructor() - - err := cdc.Unmarshal(iterator.Value(), protoMsg) - if err != nil { - return nil, nil, err + if err := cdc.Unmarshal(value, protoMsg); err != nil { + return false, err } - val, err := onResult(iterator.Key(), protoMsg) + val, err := onResult(key, protoMsg) if err != nil { - return nil, nil, err - } - - if val.Size() != 0 { - // Previously this was the "accumulate" flag - if numHits >= offset && numHits < end { - results = append(results, val) - } - numHits++ + return false, err } - if numHits >= end { - pageCompleteIter++ + if val.Size() == 0 { + return false, nil } - if numHits > end { - // Only the first entry past the end of the page is the next key; - // do not overwrite it while scanning the remainder for countTotal. - if nextKey == nil { - nextKey = iterator.Key() - } - - if !countTotal { - break - } + if accumulate { + results = append(results, val) } + return true, nil } - res := &PageResponse{NextKey: nextKey} - if countTotal { - res.Total = numHits + if req.useKey { + pageRes, err := runKeyPath(prefixStore, req, scanLimit, func(key, value []byte) (bool, error) { + hit, err := appendResult(key, value, true) + return hit, err + }) + return results, pageRes, err } - return results, res, nil + pageRes, err := runOffsetPathFiltered(prefixStore, req, scanLimit, appendResult) + return results, pageRes, err } diff --git a/sei-cosmos/types/query/filtered_pagination_test.go b/sei-cosmos/types/query/filtered_pagination_test.go index 87a01efde3..1cb2c9aae3 100644 --- a/sei-cosmos/types/query/filtered_pagination_test.go +++ b/sei-cosmos/types/query/filtered_pagination_test.go @@ -37,21 +37,21 @@ func (s *paginationTestSuite) TestFilteredPaginations() { // verify pagination with limit > total values pageReq := &query.PageRequest{Key: nil, Limit: 5, CountTotal: true} - balances, res, err := execFilterPaginate(store, pageReq, appCodec) + balances, res, err := execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(4, len(balances)) s.T().Log("verify maximum uint64 limit returns all filtered values") pageReq = &query.PageRequest{Limit: query.MaxLimit} - balances, res, err = execFilterPaginate(store, pageReq, appCodec) + balances, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(4, len(balances)) s.Require().Nil(res.NextKey) s.T().Log("verify empty request") - balances, res, err = execFilterPaginate(store, nil, appCodec) + balances, res, err = execFilterPaginate(ctx, store, nil, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(4, len(balances)) @@ -60,7 +60,7 @@ func (s *paginationTestSuite) TestFilteredPaginations() { s.T().Log("verify nextKey is returned if there are more results") pageReq = &query.PageRequest{Key: nil, Limit: 2, CountTotal: true} - balances, res, err = execFilterPaginate(store, pageReq, appCodec) + balances, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(2, len(balances)) @@ -70,12 +70,12 @@ func (s *paginationTestSuite) TestFilteredPaginations() { s.T().Log("verify both key and offset can't be given") pageReq = &query.PageRequest{Key: res.NextKey, Limit: 1, Offset: 2, CountTotal: true} - _, _, err = execFilterPaginate(store, pageReq, appCodec) + _, _, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().Error(err) s.T().Log("use nextKey for query") pageReq = &query.PageRequest{Key: res.NextKey, Limit: 2, CountTotal: true} - balances, res, err = execFilterPaginate(store, pageReq, appCodec) + balances, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(2, len(balances)) @@ -83,7 +83,7 @@ func (s *paginationTestSuite) TestFilteredPaginations() { s.T().Log("verify default limit") pageReq = &query.PageRequest{Key: nil, Limit: 0} - balances, res, err = execFilterPaginate(store, pageReq, appCodec) + balances, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(4, len(balances)) @@ -91,7 +91,7 @@ func (s *paginationTestSuite) TestFilteredPaginations() { s.T().Log("verify with offset") pageReq = &query.PageRequest{Offset: 2, Limit: 2} - balances, res, err = execFilterPaginate(store, pageReq, appCodec) + balances, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().LessOrEqual(len(balances), 2) @@ -120,13 +120,13 @@ func (s *paginationTestSuite) TestReverseFilteredPaginations() { // verify pagination with limit > total values pageReq := &query.PageRequest{Key: nil, Limit: 5, CountTotal: true, Reverse: true} - balns, res, err := execFilterPaginate(store, pageReq, appCodec) + balns, res, err := execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(5, len(balns)) s.T().Log("verify empty request") - balns, res, err = execFilterPaginate(store, nil, appCodec) + balns, res, err = execFilterPaginate(ctx, store, nil, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(10, len(balns)) @@ -135,7 +135,7 @@ func (s *paginationTestSuite) TestReverseFilteredPaginations() { s.T().Log("verify default limit") pageReq = &query.PageRequest{Reverse: true} - balns, res, err = execFilterPaginate(store, pageReq, appCodec) + balns, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(10, len(balns)) @@ -143,7 +143,7 @@ func (s *paginationTestSuite) TestReverseFilteredPaginations() { s.T().Log("verify nextKey is returned if there are more results") pageReq = &query.PageRequest{Limit: 2, CountTotal: true, Reverse: true} - balns, res, err = execFilterPaginate(store, pageReq, appCodec) + balns, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(2, len(balns)) @@ -153,12 +153,12 @@ func (s *paginationTestSuite) TestReverseFilteredPaginations() { s.T().Log("verify both key and offset can't be given") pageReq = &query.PageRequest{Key: res.NextKey, Limit: 1, Offset: 2, Reverse: true} - _, _, err = execFilterPaginate(store, pageReq, appCodec) + _, _, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().Error(err) s.T().Log("use nextKey for query and reverse true") pageReq = &query.PageRequest{Key: res.NextKey, Limit: 2, Reverse: true} - balns, res, err = execFilterPaginate(store, pageReq, appCodec) + balns, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(2, len(balns)) @@ -167,7 +167,7 @@ func (s *paginationTestSuite) TestReverseFilteredPaginations() { s.T().Log("verify last page records, nextKey for query and reverse true") pageReq = &query.PageRequest{Key: res.NextKey, Reverse: true} - balns, res, err = execFilterPaginate(store, pageReq, appCodec) + balns, res, err = execFilterPaginate(ctx, store, pageReq, appCodec) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(6, len(balns)) @@ -187,7 +187,7 @@ func (s *paginationTestSuite) TestFilteredPaginateCountTotalLargeSparseStore() { kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) } - res, err := query.FilteredPaginate(kvStore, &query.PageRequest{Limit: 3, CountTotal: true}, func(key []byte, _ []byte, _ bool) (bool, error) { + res, err := query.FilteredPaginate(ctx, kvStore, &query.PageRequest{Limit: 3, CountTotal: true}, func(key []byte, _ []byte, _ bool) (bool, error) { return string(key) == "00000000" || string(key) == "00005000" || string(key) == "00010000", nil }) s.Require().NoError(err) @@ -209,7 +209,7 @@ func (s *paginationTestSuite) TestFilteredPaginateLimitExceedsHitsInLargeStore() } var hits int - res, err := query.FilteredPaginate(kvStore, &query.PageRequest{Limit: 250}, func(_ []byte, value []byte, accumulate bool) (bool, error) { + res, err := query.FilteredPaginate(ctx, kvStore, &query.PageRequest{Limit: 250}, func(_ []byte, value []byte, accumulate bool) (bool, error) { hit := string(value) == "hit" if hit && accumulate { hits++ @@ -245,7 +245,7 @@ func (s *paginationTestSuite) TestFilteredPaginateSparseFilter() { return true, nil } - res, err := query.FilteredPaginate(kvStore, &query.PageRequest{Limit: 5}, onResult) + res, err := query.FilteredPaginate(ctx, kvStore, &query.PageRequest{Limit: 5}, onResult) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(5, len(hits)) @@ -255,7 +255,7 @@ func (s *paginationTestSuite) TestFilteredPaginateSparseFilter() { s.T().Log("count_total scans the rest of the store") hits = nil - res, err = query.FilteredPaginate(kvStore, &query.PageRequest{Limit: 5, CountTotal: true}, onResult) + res, err = query.FilteredPaginate(ctx, kvStore, &query.PageRequest{Limit: 5, CountTotal: true}, onResult) s.Require().NoError(err) s.Require().NotNil(res) s.Require().Equal(5, len(hits)) @@ -263,12 +263,12 @@ func (s *paginationTestSuite) TestFilteredPaginateSparseFilter() { s.Require().NotNil(res.NextKey) } -func execFilterPaginate(store sdk.KVStore, pageReq *query.PageRequest, appCodec codec.Codec) (balances sdk.Coins, res *query.PageResponse, err error) { +func execFilterPaginate(ctx sdk.Context, store sdk.KVStore, pageReq *query.PageRequest, appCodec codec.Codec) (balances sdk.Coins, res *query.PageResponse, err error) { balancesStore := prefix.NewStore(store, types.BalancesPrefix) accountStore := prefix.NewStore(balancesStore, address.MustLengthPrefix(addr1)) var balResult sdk.Coins - res, err = query.FilteredPaginate(accountStore, pageReq, func(key []byte, value []byte, accumulate bool) (bool, error) { + res, err = query.FilteredPaginate(ctx, accountStore, pageReq, func(key []byte, value []byte, accumulate bool) (bool, error) { var bal sdk.Coin err := appCodec.Unmarshal(value, &bal) if err != nil { diff --git a/sei-cosmos/types/query/paginate_driver.go b/sei-cosmos/types/query/paginate_driver.go new file mode 100644 index 0000000000..c2737e27de --- /dev/null +++ b/sei-cosmos/types/query/paginate_driver.go @@ -0,0 +1,329 @@ +package query + +import ( + "fmt" + "math" + + storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + db "github.com/tendermint/tm-db" +) + +// scanPhase describes where the offset paginator is in the store scan. +type scanPhase int + +const ( + scanPhaseSkip scanPhase = iota + scanPhaseCollect + scanPhasePostPage +) + +type pageRequestNorm struct { + offset uint64 + limit uint64 + end uint64 + countTotal bool + reverse bool + startKey []byte + useKey bool +} + +func normalizePageRequest(pageRequest *PageRequest) (pageRequestNorm, error) { + if pageRequest == nil { + pageRequest = &PageRequest{} + } + + if pageRequest.Offset > 0 && pageRequest.Key != nil { + return pageRequestNorm{}, fmt.Errorf("invalid request, either offset or key is expected, got both") + } + + limit := pageRequest.Limit + // Note: unlike upstream cosmos-sdk, limit == 0 must NOT implicitly enable + // countTotal. EVM precompiles call query handlers during transaction + // execution with Limit: 0; an implicit full-store count would change gas + // consumption and break AppHash and LastResultsHash across versions. + if limit == 0 { + limit = DefaultLimit + } + + return pageRequestNorm{ + offset: pageRequest.Offset, + limit: limit, + end: paginationEnd(pageRequest.Offset, limit), + countTotal: pageRequest.CountTotal, + reverse: pageRequest.Reverse, + startKey: pageRequest.Key, + useKey: len(pageRequest.Key) != 0, + }, nil +} + +func preparePageRequest(pageRequest *PageRequest, scanLimit scanLimitParams) (pageRequestNorm, error) { + req, err := normalizePageRequest(pageRequest) + if err != nil { + return pageRequestNorm{}, err + } + if err := scanLimit.checkRequest(req); err != nil { + return pageRequestNorm{}, err + } + return req, nil +} + +func paginationEnd(offset, limit uint64) uint64 { + if limit > math.MaxUint64-offset { + return math.MaxUint64 + } + return offset + limit +} + +func getIterator(prefixStore storetypes.KVStore, start []byte, reverse bool) db.Iterator { + if reverse { + var end []byte + if start != nil { + itr := prefixStore.Iterator(start, nil) + defer func() { _ = itr.Close() }() + if itr.Valid() { + itr.Next() + end = itr.Key() + } + } + return prefixStore.ReverseIterator(nil, end) + } + return prefixStore.Iterator(start, nil) +} + +func runKeyPath( + store storetypes.KVStore, + req pageRequestNorm, + scanLimit scanLimitParams, + accept func(key, value []byte) (countsTowardLimit bool, err error), +) (*PageResponse, error) { + iterator := getIterator(store, req.startKey, req.reverse) + defer func() { _ = iterator.Close() }() + + var ( + numHits uint64 + nextKey []byte + totalIter uint64 + ) + + for ; iterator.Valid(); iterator.Next() { + if numHits == req.limit { + nextKey = iterator.Key() + break + } + + if scanLimit.enforce { + totalIter++ + if err := scanLimit.checkKeyPath(totalIter); err != nil { + return nil, err + } + } + + if iterator.Error() != nil { + return nil, iterator.Error() + } + + counts, err := accept(iterator.Key(), iterator.Value()) + if err != nil { + return nil, err + } + if counts { + numHits++ + } + } + + return &PageResponse{NextKey: nextKey}, nil +} + +// offsetScanCursor tracks offset-pagination progress through the store. +// Unfiltered scans count every KV entry; filtered scans count only entries +// accepted by the caller's filter. +type offsetScanCursor struct { + req pageRequestNorm + scanLimit scanLimitParams + filtered bool + + scanned uint64 + hits uint64 + nextKey []byte + pageCompleteIter uint64 +} + +func newOffsetScanCursor(req pageRequestNorm, scanLimit scanLimitParams, filtered bool) *offsetScanCursor { + return &offsetScanCursor{ + req: req, + scanLimit: scanLimit, + filtered: filtered, + } +} + +func (c *offsetScanCursor) phase() scanPhase { + if !c.filtered { + switch { + case c.scanned <= c.req.offset: + return scanPhaseSkip + case c.scanned <= c.req.end: + return scanPhaseCollect + default: + return scanPhasePostPage + } + } + + switch { + case c.hits < c.req.offset: + return scanPhaseSkip + case c.hits < c.req.end: + return scanPhaseCollect + default: + return scanPhasePostPage + } +} + +func (c *offsetScanCursor) beginIteration() error { + c.scanned++ + return c.checkScanBudgetBeforePageFilled() +} + +// checkScanBudgetBeforePageFilled caps raw KV scans while the page is still +// being filled. Filtered and unfiltered offset paths share this guard. +func (c *offsetScanCursor) checkScanBudgetBeforePageFilled() error { + if c.scanLimit.enforce && c.phase() != scanPhasePostPage && + c.scanned > paginationEnd(c.req.offset, c.scanLimit.limit) { + return scanLimitError(c.scanLimit.limit, "use key-based pagination instead") + } + return nil +} + +func (c *offsetScanCursor) checkPostPageBudget() (stop bool, err error) { + if c.pageEndReached() { + c.pageCompleteIter++ + } + return c.scanLimit.checkPostPage(c.pageCompleteIter, c.req.countTotal) +} + +// pageEndReached reports whether the current entry should count toward the +// post-page scan budget. Filtered scans use hit count; unfiltered scans enter +// post-page only after the in-page window is complete. +func (c *offsetScanCursor) pageEndReached() bool { + return c.phase() == scanPhasePostPage +} + +func (c *offsetScanCursor) accumulate() bool { + return c.phase() == scanPhaseCollect +} + +func (c *offsetScanCursor) recordFilteredHit(key []byte, matched bool) (stop bool) { + if matched { + c.hits++ + } + if c.hits > c.req.end { + if c.nextKey == nil { + c.nextKey = key + } + return !c.req.countTotal + } + return false +} + +func (c *offsetScanCursor) recordUnfilteredHit(key []byte) (stop bool) { + if c.scanned == c.req.end+1 { + c.nextKey = key + return !c.req.countTotal + } + return false +} + +func (c *offsetScanCursor) total() uint64 { + if c.filtered { + return c.hits + } + return c.scanned +} + +func runOffsetPathUnfiltered( + store storetypes.KVStore, + req pageRequestNorm, + scanLimit scanLimitParams, + onResult func(key, value []byte) error, +) (*PageResponse, error) { + iterator := getIterator(store, nil, req.reverse) + defer func() { _ = iterator.Close() }() + + cursor := newOffsetScanCursor(req, scanLimit, false) + +loop: + for ; iterator.Valid(); iterator.Next() { + if err := cursor.beginIteration(); err != nil { + return nil, err + } + if stop, err := cursor.checkPostPageBudget(); err != nil { + return nil, err + } else if stop { + break + } + + switch cursor.phase() { + case scanPhaseSkip: + continue + case scanPhaseCollect: + if err := onResult(iterator.Key(), iterator.Value()); err != nil { + return nil, err + } + case scanPhasePostPage: + if cursor.recordUnfilteredHit(iterator.Key()) { + break loop + } + } + + if iterator.Error() != nil { + return nil, iterator.Error() + } + } + + res := &PageResponse{NextKey: cursor.nextKey} + if req.countTotal { + res.Total = cursor.total() + } + return res, nil +} + +func runOffsetPathFiltered( + store storetypes.KVStore, + req pageRequestNorm, + scanLimit scanLimitParams, + onResult func(key, value []byte, accumulate bool) (hit bool, err error), +) (*PageResponse, error) { + iterator := getIterator(store, nil, req.reverse) + defer func() { _ = iterator.Close() }() + + cursor := newOffsetScanCursor(req, scanLimit, true) + + for ; iterator.Valid(); iterator.Next() { + if err := cursor.beginIteration(); err != nil { + return nil, err + } + if stop, err := cursor.checkPostPageBudget(); err != nil { + return nil, err + } else if stop { + break + } + + if iterator.Error() != nil { + return nil, iterator.Error() + } + + hit, err := onResult(iterator.Key(), iterator.Value(), cursor.accumulate()) + if err != nil { + return nil, err + } + + if cursor.recordFilteredHit(iterator.Key(), hit) { + break + } + } + + res := &PageResponse{NextKey: cursor.nextKey} + if req.countTotal { + res.Total = cursor.total() + } + return res, nil +} diff --git a/sei-cosmos/types/query/paginate_driver_test.go b/sei-cosmos/types/query/paginate_driver_test.go new file mode 100644 index 0000000000..1f00e21883 --- /dev/null +++ b/sei-cosmos/types/query/paginate_driver_test.go @@ -0,0 +1,47 @@ +package query + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRunOffsetPathUnfilteredSetsNextKeyWithTightPostPageBudget(t *testing.T) { + kvStore := newTestKVStore(t) + for i := 0; i < 3; i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) + } + + req, err := normalizePageRequest(&PageRequest{Limit: 1, CountTotal: false}) + require.NoError(t, err) + + var count int + res, err := runOffsetPathUnfiltered(kvStore, req, scanLimitParams{enforce: true, limit: 1}, func(_, _ []byte) error { + count++ + return nil + }) + require.NoError(t, err) + require.Equal(t, 1, count) + require.NotNil(t, res.NextKey) + require.Equal(t, []byte("00000001"), res.NextKey) +} + +func TestRunOffsetPathUnfilteredBreaksWithoutExtraNextAfterNextKey(t *testing.T) { + kvStore := newTestKVStore(t) + for i := 0; i < 4; i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) + } + + req, err := normalizePageRequest(&PageRequest{Limit: 2, CountTotal: false}) + require.NoError(t, err) + + var seenKeys []string + res, err := runOffsetPathUnfiltered(kvStore, req, scanLimitParams{enforce: false, limit: 0}, func(key, _ []byte) error { + seenKeys = append(seenKeys, string(key)) + return nil + }) + require.NoError(t, err) + require.Equal(t, []string{"00000000", "00000001"}, seenKeys) + require.Equal(t, []byte("00000002"), res.NextKey) +} diff --git a/sei-cosmos/types/query/pagination.go b/sei-cosmos/types/query/pagination.go index 668da055be..cf452b4262 100644 --- a/sei-cosmos/types/query/pagination.go +++ b/sei-cosmos/types/query/pagination.go @@ -1,11 +1,10 @@ package query import ( - "fmt" "math" "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" - db "github.com/tendermint/tm-db" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -18,7 +17,8 @@ const DefaultLimit = 100 // which equals the maximum value that can be stored in uint64 const MaxLimit = uint64(math.MaxUint64) -// MaxScanLimit is retained by the v6.6-compatible precompile paginators. +// MaxScanLimit is the scan cap for untrusted ABCI query origins and the frozen +// consensus-path limit used by the v6.6-compatible paginators. const MaxScanLimit = uint64(10_000) // ParsePagination validate PageRequest and returns page number & limit. @@ -50,123 +50,25 @@ func ParsePagination(pageReq *PageRequest) (page, limit int, err error) { // Paginate does pagination of all the results in the PrefixStore based on the // provided PageRequest. onResult should be used to do actual unmarshaling. func Paginate( + ctx sdk.Context, prefixStore types.KVStore, pageRequest *PageRequest, onResult func(key []byte, value []byte) error, ) (*PageResponse, error) { - - // if the PageRequest is nil, use default PageRequest - if pageRequest == nil { - pageRequest = &PageRequest{} - } - - offset := pageRequest.Offset - key := pageRequest.Key - limit := pageRequest.Limit - countTotal := pageRequest.CountTotal - reverse := pageRequest.Reverse - - if offset > 0 && key != nil { - return nil, fmt.Errorf("invalid request, either offset or key is expected, got both") - } - - // Note: unlike upstream cosmos-sdk, limit == 0 must NOT implicitly enable - // countTotal. EVM precompiles (e.g. precompiles/staking) call query - // handlers during transaction execution with Limit: 0; an implicit - // full-store count would change their gas consumption and therefore - // break AppHash and LastResultsHash across versions. - if limit == 0 { - limit = DefaultLimit - } - - if len(key) != 0 { - iterator := getIterator(prefixStore, key, reverse) - defer func() { _ = iterator.Close() }() - - var count uint64 - var nextKey []byte - - for ; iterator.Valid(); iterator.Next() { - if count == limit { - nextKey = iterator.Key() - break - } - if iterator.Error() != nil { - return nil, iterator.Error() - } - err := onResult(iterator.Key(), iterator.Value()) - if err != nil { - return nil, err - } - count++ - } - - return &PageResponse{ - NextKey: nextKey, - }, nil + scanLimit := scanLimitParamsFromContext(ctx) + req, err := preparePageRequest(pageRequest, scanLimit) + if err != nil { + return nil, err } - iterator := getIterator(prefixStore, nil, reverse) - defer func() { _ = iterator.Close() }() - - end := paginationEnd(offset, limit) - - var count uint64 - var nextKey []byte - - for ; iterator.Valid(); iterator.Next() { - count++ - - if count <= offset { - continue - } - if count <= end { - err := onResult(iterator.Key(), iterator.Value()) - if err != nil { - return nil, err + if req.useKey { + return runKeyPath(prefixStore, req, scanLimit, func(key, value []byte) (bool, error) { + if err := onResult(key, value); err != nil { + return false, err } - } else if count == end+1 { - nextKey = iterator.Key() - - if !countTotal { - break - } - } - if iterator.Error() != nil { - return nil, iterator.Error() - } - } - - res := &PageResponse{NextKey: nextKey} - if countTotal { - res.Total = count + return true, nil + }) } - return res, nil -} - -// paginationEnd returns the index one past the last entry of the requested -// page, saturating at math.MaxUint64 instead of wrapping around when -// offset+limit overflows. -func paginationEnd(offset, limit uint64) uint64 { - if limit > math.MaxUint64-offset { - return math.MaxUint64 - } - return offset + limit -} - -func getIterator(prefixStore types.KVStore, start []byte, reverse bool) db.Iterator { - if reverse { - var end []byte - if start != nil { - itr := prefixStore.Iterator(start, nil) - defer func() { _ = itr.Close() }() - if itr.Valid() { - itr.Next() - end = itr.Key() - } - } - return prefixStore.ReverseIterator(nil, end) - } - return prefixStore.Iterator(start, nil) + return runOffsetPathUnfiltered(prefixStore, req, scanLimit, onResult) } diff --git a/sei-cosmos/types/query/pagination_test.go b/sei-cosmos/types/query/pagination_test.go index 6601911759..1c33244749 100644 --- a/sei-cosmos/types/query/pagination_test.go +++ b/sei-cosmos/types/query/pagination_test.go @@ -314,13 +314,13 @@ func (s *paginationTestSuite) TestPaginateCountTotalLargeStore() { } s.T().Log("count_total scans the whole store and returns an accurate total") - res, err := query.Paginate(kvStore, &query.PageRequest{Limit: 1, CountTotal: true}, func(_, _ []byte) error { return nil }) + res, err := query.Paginate(ctx, kvStore, &query.PageRequest{Limit: 1, CountTotal: true}, func(_, _ []byte) error { return nil }) s.Require().NoError(err) s.Require().Equal(uint64(numItems), res.Total) s.T().Log("a large offset skips entries instead of being rejected") var count int - res, err = query.Paginate(kvStore, &query.PageRequest{Offset: 10_001, Limit: 5}, func(_, _ []byte) error { + res, err = query.Paginate(ctx, kvStore, &query.PageRequest{Offset: 10_001, Limit: 5}, func(_, _ []byte) error { count++ return nil }) @@ -330,7 +330,7 @@ func (s *paginationTestSuite) TestPaginateCountTotalLargeStore() { s.T().Log("a maximum uint64 limit returns everything without overflowing") count = 0 - res, err = query.Paginate(kvStore, &query.PageRequest{Offset: 1, Limit: query.MaxLimit}, func(_, _ []byte) error { + res, err = query.Paginate(ctx, kvStore, &query.PageRequest{Offset: 1, Limit: query.MaxLimit}, func(_, _ []byte) error { count++ return nil }) diff --git a/sei-cosmos/types/query/scan_limit.go b/sei-cosmos/types/query/scan_limit.go new file mode 100644 index 0000000000..3da67d23fa --- /dev/null +++ b/sei-cosmos/types/query/scan_limit.go @@ -0,0 +1,70 @@ +package query + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" +) + +type scanLimitParams struct { + enforce bool + limit uint64 + // boundRequest rejects PageRequest limit and offset above `limit` before + // any store iteration. + boundRequest bool +} + +func v66ScanLimitParams() scanLimitParams { + return scanLimitParams{enforce: true, limit: MaxScanLimit} +} + +func scanLimitParamsFromContext(ctx sdk.Context) scanLimitParams { + if !ctx.IsABCIQuery() { + return scanLimitParams{} + } + if !ctx.EnforceQueryScanLimit() { + return scanLimitParams{} + } + return scanLimitParams{enforce: true, limit: ctx.QueryScanLimit(), boundRequest: true} +} + +// checkRequest rejects limit and offset that cannot be served within `limit`. +func (p scanLimitParams) checkRequest(req pageRequestNorm) error { + if !p.boundRequest || p.limit == 0 { + return nil + } + if req.limit > p.limit { + return status.Errorf(codes.InvalidArgument, + "limit %d exceeds the maximum of %d; use key-based pagination instead", + req.limit, p.limit) + } + if req.offset > p.limit { + return status.Errorf(codes.InvalidArgument, + "offset %d exceeds the maximum of %d; use key-based pagination instead", + req.offset, p.limit) + } + return nil +} + +func (p scanLimitParams) checkKeyPath(totalIter uint64) error { + if p.enforce && totalIter > p.limit { + return scanLimitError(p.limit, "use a more specific key prefix or reduce limit") + } + return nil +} + +func (p scanLimitParams) checkPostPage(pageCompleteIter uint64, countTotal bool) (stop bool, err error) { + if p.enforce && pageCompleteIter > p.limit { + if countTotal { + return false, scanLimitError(p.limit, "use key-based pagination instead") + } + return true, nil + } + return false, nil +} + +func scanLimitError(limit uint64, hint string) error { + return status.Errorf(codes.InvalidArgument, + "scanned more than %d entries without filling the page; %s", limit, hint) +} diff --git a/sei-cosmos/types/query/scan_limit_test.go b/sei-cosmos/types/query/scan_limit_test.go new file mode 100644 index 0000000000..0082da4edf --- /dev/null +++ b/sei-cosmos/types/query/scan_limit_test.go @@ -0,0 +1,261 @@ +package query + +import ( + "fmt" + "testing" + + "github.com/sei-protocol/sei-chain/sei-cosmos/store/prefix" + storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + + "github.com/sei-protocol/sei-chain/sei-cosmos/store" +) + +func newTestKVStore(t *testing.T) sdk.KVStore { + t.Helper() + + db := dbm.NewMemDB() + key := storetypes.NewKVStoreKey("test") + ms := store.NewCommitMultiStore(db) + ms.MountStoreWithDB(key, storetypes.StoreTypeIAVL, db) + require.NoError(t, ms.LoadLatestVersion()) + return prefix.NewStore(ms.GetKVStore(key), []byte("scanlimit/")) +} + +func enforcingABCIContext(t *testing.T) sdk.Context { + t.Helper() + return sdk.Context{}.WithIsABCIQuery(true).WithQueryScanLimit(true, MaxScanLimit) +} + +func TestPaginateEnforcesUntrustedScanLimitOnOffsetPath(t *testing.T) { + kvStore := newTestKVStore(t) + ctx := enforcingABCIContext(t) + + for i := 0; i < int(MaxScanLimit+50); i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) + } + + _, err := Paginate(ctx, kvStore, &PageRequest{Limit: 1, CountTotal: true}, func(_, _ []byte) error { + return nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "scanned more than 10000 entries") +} + +func TestPaginateRejectsUntrustedLimitAboveScanCap(t *testing.T) { + kvStore := newTestKVStore(t) + ctx := enforcingABCIContext(t) + kvStore.Set([]byte("a"), []byte("v")) + + called := false + _, err := Paginate(ctx, kvStore, &PageRequest{Limit: MaxScanLimit + 1}, func(_, _ []byte) error { + called = true + return nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "limit 10001 exceeds the maximum of 10000") + require.False(t, called) +} + +func TestPaginateRejectsUntrustedOffsetAboveScanCap(t *testing.T) { + kvStore := newTestKVStore(t) + ctx := enforcingABCIContext(t) + kvStore.Set([]byte("a"), []byte("v")) + + called := false + _, err := Paginate(ctx, kvStore, &PageRequest{Offset: MaxScanLimit + 1, Limit: 1}, func(_, _ []byte) error { + called = true + return nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "offset 10001 exceeds the maximum of 10000") + require.False(t, called) +} + +func TestFilteredPaginateEnforcesUntrustedScanLimitOnKeyPath(t *testing.T) { + kvStore := newTestKVStore(t) + ctx := enforcingABCIContext(t) + + for i := 0; i < int(MaxScanLimit+50); i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) + } + + _, err := FilteredPaginate(ctx, kvStore, &PageRequest{Key: []byte("00000000"), Limit: 5}, func(_ []byte, value []byte, _ bool) (bool, error) { + return string(value) == "hit", nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "scanned more than 10000 entries") +} + +func TestPaginateTrustedOriginUsesHigherLimit(t *testing.T) { + const ( + storeSize = 25_000 + pageLimit = 20_000 + ) + trustedLimit := uint64(storeSize) + + kvStore := newTestKVStore(t) + for i := 0; i < storeSize; i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) + } + + pageReq := &PageRequest{Limit: pageLimit, CountTotal: false} + + _, err := Paginate(enforcingABCIContext(t), kvStore, pageReq, func(_, _ []byte) error { + return nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "limit 20000 exceeds the maximum of 10000") + + trustedCtx := sdk.Context{}. + WithIsABCIQuery(true). + WithIsTrustedQueryOrigin(true). + WithQueryScanLimit(true, trustedLimit) + + var count int + _, err = Paginate(trustedCtx, kvStore, pageReq, func(_, _ []byte) error { + count++ + return nil + }) + require.NoError(t, err) + require.Equal(t, pageLimit, count) +} + +func TestPaginateTrustedOriginUnlimitedScan(t *testing.T) { + const ( + storeSize = int(MaxScanLimit) + 50 + pageLimit = uint64(storeSize) + ) + + kvStore := newTestKVStore(t) + for i := 0; i < storeSize; i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) + } + + pageReq := &PageRequest{Limit: pageLimit, CountTotal: false} + unlimitedCtx := sdk.Context{}. + WithIsABCIQuery(true). + WithIsTrustedQueryOrigin(true). + WithQueryScanLimit(false, 0) + + var count int + _, err := Paginate(unlimitedCtx, kvStore, pageReq, func(_, _ []byte) error { + count++ + return nil + }) + require.NoError(t, err) + require.Equal(t, storeSize, count) +} + +func TestPaginateTrustedOriginAllowsLimitAndOffsetWithinTrustedCap(t *testing.T) { + const ( + storeSize = 25_000 + trustedLimit = uint64(storeSize) + offset = uint64(20_000) + pageLimit = uint64(1_000) + ) + + kvStore := newTestKVStore(t) + for i := 0; i < storeSize; i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) + } + + untrusted := enforcingABCIContext(t) + _, err := Paginate(untrusted, kvStore, &PageRequest{Offset: offset, Limit: pageLimit}, func(_, _ []byte) error { + return nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "offset 20000 exceeds the maximum of 10000") + + trustedCtx := sdk.Context{}. + WithIsABCIQuery(true). + WithIsTrustedQueryOrigin(true). + WithQueryScanLimit(true, trustedLimit) + + var count int + _, err = Paginate(trustedCtx, kvStore, &PageRequest{Offset: offset, Limit: pageLimit}, func(_, _ []byte) error { + count++ + return nil + }) + require.NoError(t, err) + require.Equal(t, int(pageLimit), count) +} + +func TestFilteredPaginateV66DoesNotRejectOversizedLimit(t *testing.T) { + kvStore := newTestKVStore(t) + for i := 0; i < 5; i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("v")) + } + + var count int + _, err := FilteredPaginateV66(kvStore, &PageRequest{Limit: MaxScanLimit + 1}, func(_, _ []byte, accumulate bool) (bool, error) { + if accumulate { + count++ + } + return true, nil + }) + require.NoError(t, err) + require.Equal(t, 5, count) + + _, err = FilteredPaginateV66(kvStore, &PageRequest{Offset: MaxScanLimit + 1, Limit: 1}, func(_, _ []byte, accumulate bool) (bool, error) { + if accumulate { + t.Fatal("offset past the store must not accumulate") + } + return true, nil + }) + require.NoError(t, err) +} + +func TestFilteredPaginateEnforcesUntrustedScanLimit(t *testing.T) { + kvStore := newTestKVStore(t) + ctx := enforcingABCIContext(t) + + for i := 0; i < 20_000; i++ { + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), []byte("miss")) + } + + _, err := FilteredPaginate(ctx, kvStore, &PageRequest{Limit: 5}, func(_ []byte, value []byte, _ bool) (bool, error) { + return string(value) == "hit", nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "scanned more than 10000 entries") +} + +func TestFilteredPaginateStopsPostPageScanWhenCountTotalFalse(t *testing.T) { + kvStore := newTestKVStore(t) + ctx := enforcingABCIContext(t) + + for i := 0; i < 20_000; i++ { + value := []byte("miss") + if i < 5 { + value = []byte("hit") + } + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), value) + } + + res, err := FilteredPaginate(ctx, kvStore, &PageRequest{Limit: 5, CountTotal: false}, func(_ []byte, value []byte, _ bool) (bool, error) { + return string(value) == "hit", nil + }) + require.NoError(t, err) + require.Nil(t, res.NextKey) +} + +func TestFilteredPaginateV66StopsPostPageScanWhenCountTotalFalse(t *testing.T) { + kvStore := newTestKVStore(t) + + for i := 0; i < 20_000; i++ { + value := []byte("miss") + if i < 5 { + value = []byte("hit") + } + kvStore.Set([]byte(fmt.Sprintf("%08d", i)), value) + } + + res, err := FilteredPaginateV66(kvStore, &PageRequest{Limit: 5, CountTotal: false}, func(_ []byte, value []byte, _ bool) (bool, error) { + return string(value) == "hit", nil + }) + require.NoError(t, err) + require.Nil(t, res.NextKey) +} diff --git a/sei-cosmos/x/auth/keeper/grpc_query.go b/sei-cosmos/x/auth/keeper/grpc_query.go index c1d06d21ea..bbf98a6339 100644 --- a/sei-cosmos/x/auth/keeper/grpc_query.go +++ b/sei-cosmos/x/auth/keeper/grpc_query.go @@ -26,7 +26,7 @@ func (ak AccountKeeper) Accounts(c context.Context, req *types.QueryAccountsRequ accountsStore := prefix.NewStore(store, types.AddressStoreKeyPrefix) var accounts []*codectypes.Any - pageRes, err := query.Paginate(accountsStore, req.Pagination, func(key, value []byte) error { + pageRes, err := query.Paginate(ctx, accountsStore, req.Pagination, func(key, value []byte) error { account := ak.decodeAccount(value) any, err := codectypes.NewAnyWithValue(account) if err != nil { diff --git a/sei-cosmos/x/authz/keeper/grpc_query.go b/sei-cosmos/x/authz/keeper/grpc_query.go index e30aeeb15b..d80aa8867c 100644 --- a/sei-cosmos/x/authz/keeper/grpc_query.go +++ b/sei-cosmos/x/authz/keeper/grpc_query.go @@ -53,12 +53,7 @@ func (k Keeper) Grants(c context.Context, req *authz.QueryGrantsRequest) (*authz key := grantStoreKey(grantee, granter, "") grantsStore := prefix.NewStore(store, key) - paginate := query.GenericFilteredPaginateV66[*authz.Grant, *authz.Grant] - if ctx.IsABCIQuery() { - paginate = query.GenericFilteredPaginate[*authz.Grant, *authz.Grant] - } - - authorizations, pageRes, err := paginate(k.cdc, grantsStore, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.Grant, error) { + onResult := func(key []byte, auth *authz.Grant) (*authz.Grant, error) { auth1 := auth.GetAuthorization() if err != nil { return nil, err @@ -72,9 +67,12 @@ func (k Keeper) Grants(c context.Context, req *authz.QueryGrantsRequest) (*authz Authorization: authorizationAny, Expiration: auth.Expiration, }, nil - }, func() *authz.Grant { - return &authz.Grant{} - }) + } + constructor := func() *authz.Grant { return &authz.Grant{} } + + authorizations, pageRes, err := query.GenericFilteredPaginateForContext( + ctx, k.cdc, grantsStore, req.Pagination, onResult, constructor, + ) if err != nil { return nil, err } @@ -100,12 +98,7 @@ func (k Keeper) GranterGrants(c context.Context, req *authz.QueryGranterGrantsRe store := ctx.KVStore(k.storeKey) authzStore := prefix.NewStore(store, grantStoreKey(nil, granter, "")) - paginate := query.GenericFilteredPaginateV66[*authz.Grant, *authz.GrantAuthorization] - if ctx.IsABCIQuery() { - paginate = query.GenericFilteredPaginate[*authz.Grant, *authz.GrantAuthorization] - } - - grants, pageRes, err := paginate(k.cdc, authzStore, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.GrantAuthorization, error) { + onResult := func(key []byte, auth *authz.Grant) (*authz.GrantAuthorization, error) { auth1 := auth.GetAuthorization() if err != nil { return nil, err @@ -123,10 +116,12 @@ func (k Keeper) GranterGrants(c context.Context, req *authz.QueryGranterGrantsRe Authorization: any, Expiration: auth.Expiration, }, nil + } + constructor := func() *authz.Grant { return &authz.Grant{} } - }, func() *authz.Grant { - return &authz.Grant{} - }) + grants, pageRes, err := query.GenericFilteredPaginateForContext( + ctx, k.cdc, authzStore, req.Pagination, onResult, constructor, + ) if err != nil { return nil, err @@ -152,12 +147,7 @@ func (k Keeper) GranteeGrants(c context.Context, req *authz.QueryGranteeGrantsRe ctx := sdk.UnwrapSDKContext(c) store := prefix.NewStore(ctx.KVStore(k.storeKey), GrantKey) - paginate := query.GenericFilteredPaginateV66[*authz.Grant, *authz.GrantAuthorization] - if ctx.IsABCIQuery() { - paginate = query.GenericFilteredPaginate[*authz.Grant, *authz.GrantAuthorization] - } - - authorizations, pageRes, err := paginate(k.cdc, store, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.GrantAuthorization, error) { + onResult := func(key []byte, auth *authz.Grant) (*authz.GrantAuthorization, error) { auth1 := auth.GetAuthorization() if err != nil { return nil, err @@ -179,9 +169,12 @@ func (k Keeper) GranteeGrants(c context.Context, req *authz.QueryGranteeGrantsRe Granter: granter.String(), Grantee: grantee.String(), }, nil - }, func() *authz.Grant { - return &authz.Grant{} - }) + } + constructor := func() *authz.Grant { return &authz.Grant{} } + + authorizations, pageRes, err := query.GenericFilteredPaginateForContext( + ctx, k.cdc, store, req.Pagination, onResult, constructor, + ) if err != nil { return nil, err } diff --git a/sei-cosmos/x/bank/keeper/grpc_query.go b/sei-cosmos/x/bank/keeper/grpc_query.go index 1685c636bc..d848271d9e 100644 --- a/sei-cosmos/x/bank/keeper/grpc_query.go +++ b/sei-cosmos/x/bank/keeper/grpc_query.go @@ -59,7 +59,7 @@ func (k BaseKeeper) AllBalances(ctx context.Context, req *types.QueryAllBalances balances := sdk.NewCoins() accountStore := k.getAccountStore(sdkCtx, addr) - pageRes, err := query.Paginate(accountStore, req.Pagination, func(_, value []byte) error { + pageRes, err := query.Paginate(sdkCtx, accountStore, req.Pagination, func(_, value []byte) error { var result sdk.Coin err := k.cdc.Unmarshal(value, &result) if err != nil { @@ -94,7 +94,7 @@ func (k BaseKeeper) SpendableBalances(ctx context.Context, req *types.QuerySpend accountStore := k.getAccountStore(sdkCtx, addr) zeroAmt := sdk.ZeroInt() - pageRes, err := query.Paginate(accountStore, req.Pagination, func(key, value []byte) error { + pageRes, err := query.Paginate(sdkCtx, accountStore, req.Pagination, func(key, value []byte) error { balances = append(balances, sdk.NewCoin(string(key), zeroAmt)) return nil }) @@ -161,7 +161,7 @@ func (k BaseKeeper) DenomsMetadata(c context.Context, req *types.QueryDenomsMeta store := prefix.NewStore(ctx.KVStore(k.storeKey), types.DenomMetadataPrefix) metadatas := []types.Metadata{} - pageRes, err := query.Paginate(store, req.Pagination, func(_, value []byte) error { + pageRes, err := query.Paginate(ctx, store, req.Pagination, func(_, value []byte) error { var metadata types.Metadata k.cdc.MustUnmarshal(value, &metadata) diff --git a/sei-cosmos/x/bank/keeper/keeper.go b/sei-cosmos/x/bank/keeper/keeper.go index e5b4869c0e..e2a6d39671 100644 --- a/sei-cosmos/x/bank/keeper/keeper.go +++ b/sei-cosmos/x/bank/keeper/keeper.go @@ -88,7 +88,7 @@ func (k BaseKeeper) GetPaginatedTotalSupply(ctx sdk.Context, pagination *query.P ptr := k.intPool.Get() defer k.intPool.Put(ptr) - pageRes, err := query.Paginate(supplyStore, pagination, func(key, value []byte) error { + pageRes, err := query.Paginate(ctx, supplyStore, pagination, func(key, value []byte) error { if err := ptr.Unmarshal(value); err != nil { return fmt.Errorf("unable to convert amount string to Int %v", err) } diff --git a/sei-cosmos/x/distribution/keeper/grpc_query.go b/sei-cosmos/x/distribution/keeper/grpc_query.go index c9ce7c5d64..3d4b2b16dc 100644 --- a/sei-cosmos/x/distribution/keeper/grpc_query.go +++ b/sei-cosmos/x/distribution/keeper/grpc_query.go @@ -90,12 +90,7 @@ func (k Keeper) ValidatorSlashes(c context.Context, req *types.QueryValidatorSla } slashesStore := prefix.NewStore(store, types.GetValidatorSlashEventPrefix(valAddr)) - paginate := query.FilteredPaginateV66 - if ctx.IsABCIQuery() { - paginate = query.FilteredPaginate - } - - pageRes, err := paginate(slashesStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginateForContext(ctx, slashesStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { var result types.ValidatorSlashEvent err := k.cdc.Unmarshal(value, &result) diff --git a/sei-cosmos/x/evidence/keeper/grpc_query.go b/sei-cosmos/x/evidence/keeper/grpc_query.go index 35287069e7..b265cd4a4f 100644 --- a/sei-cosmos/x/evidence/keeper/grpc_query.go +++ b/sei-cosmos/x/evidence/keeper/grpc_query.go @@ -59,7 +59,7 @@ func (k Keeper) AllEvidence(c context.Context, req *types.QueryAllEvidenceReques store := ctx.KVStore(k.storeKey) evidenceStore := prefix.NewStore(store, types.KeyPrefixEvidence) - pageRes, err := query.Paginate(evidenceStore, req.Pagination, func(key []byte, value []byte) error { + pageRes, err := query.Paginate(ctx, evidenceStore, req.Pagination, func(key []byte, value []byte) error { result, err := k.UnmarshalEvidence(value) if err != nil { return err diff --git a/sei-cosmos/x/gov/keeper/grpc_query.go b/sei-cosmos/x/gov/keeper/grpc_query.go index 2bb3b41050..0ff9bd59c2 100644 --- a/sei-cosmos/x/gov/keeper/grpc_query.go +++ b/sei-cosmos/x/gov/keeper/grpc_query.go @@ -42,12 +42,7 @@ func (q Keeper) Proposals(c context.Context, req *types.QueryProposalsRequest) ( store := ctx.KVStore(q.storeKey) proposalStore := prefix.NewStore(store, types.ProposalsKeyPrefix) - paginate := query.FilteredPaginateV66 - if ctx.IsABCIQuery() { - paginate = query.FilteredPaginate - } - - pageRes, err := paginate(proposalStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginateForContext(ctx, proposalStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { var p types.Proposal if err := q.cdc.Unmarshal(value, &p); err != nil { return false, status.Error(codes.Internal, err.Error()) @@ -142,7 +137,7 @@ func (q Keeper) Votes(c context.Context, req *types.QueryVotesRequest) (*types.Q store := ctx.KVStore(q.storeKey) votesStore := prefix.NewStore(store, types.VotesKey(req.ProposalId)) - pageRes, err := query.Paginate(votesStore, req.Pagination, func(key []byte, value []byte) error { + pageRes, err := query.Paginate(ctx, votesStore, req.Pagination, func(key []byte, value []byte) error { var vote types.Vote if err := q.cdc.Unmarshal(value, &vote); err != nil { return err @@ -232,7 +227,7 @@ func (q Keeper) Deposits(c context.Context, req *types.QueryDepositsRequest) (*t store := ctx.KVStore(q.storeKey) depositStore := prefix.NewStore(store, types.DepositsKey(req.ProposalId)) - pageRes, err := query.Paginate(depositStore, req.Pagination, func(key []byte, value []byte) error { + pageRes, err := query.Paginate(ctx, depositStore, req.Pagination, func(key []byte, value []byte) error { var deposit types.Deposit if err := q.cdc.Unmarshal(value, &deposit); err != nil { return err diff --git a/sei-cosmos/x/slashing/keeper/grpc_query.go b/sei-cosmos/x/slashing/keeper/grpc_query.go index ccea2e26ae..77ca2fb80e 100644 --- a/sei-cosmos/x/slashing/keeper/grpc_query.go +++ b/sei-cosmos/x/slashing/keeper/grpc_query.go @@ -58,7 +58,7 @@ func (k Keeper) SigningInfos(c context.Context, req *types.QuerySigningInfosRequ var signInfos []types.ValidatorSigningInfo sigInfoStore := prefix.NewStore(store, types.ValidatorSigningInfoKeyPrefix) - pageRes, err := query.Paginate(sigInfoStore, req.Pagination, func(key []byte, value []byte) error { + pageRes, err := query.Paginate(ctx, sigInfoStore, req.Pagination, func(key []byte, value []byte) error { var info types.ValidatorSigningInfo err := k.cdc.Unmarshal(value, &info) if err != nil { diff --git a/sei-cosmos/x/staking/keeper/grpc_query.go b/sei-cosmos/x/staking/keeper/grpc_query.go index a52ce76371..39fe3ac606 100644 --- a/sei-cosmos/x/staking/keeper/grpc_query.go +++ b/sei-cosmos/x/staking/keeper/grpc_query.go @@ -37,12 +37,7 @@ func (k Querier) Validators(c context.Context, req *types.QueryValidatorsRequest store := ctx.KVStore(k.storeKey) valStore := prefix.NewStore(store, types.ValidatorsKey) - paginate := query.FilteredPaginateV66 - if ctx.IsABCIQuery() { - paginate = query.FilteredPaginate - } - - pageRes, err := paginate(valStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginateForContext(ctx, valStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { val, err := types.UnmarshalValidator(k.cdc, value) if err != nil { return false, err @@ -107,12 +102,7 @@ func (k Querier) ValidatorDelegations(c context.Context, req *types.QueryValidat // Consensus execution defaults to release/v6.6 behavior. Only contexts // created by BaseApp.Query may use the repaired LCD/gRPC behavior. - paginate := query.FilteredPaginateV66 - if ctx.IsABCIQuery() { - paginate = query.FilteredPaginate - } - - pageRes, err := paginate(valStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginateForContext(ctx, valStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { delegation, err := types.UnmarshalDelegation(k.cdc, value) if err != nil { return false, err @@ -166,7 +156,7 @@ func (k Querier) ValidatorUnbondingDelegations(c context.Context, req *types.Que srcValPrefix := types.GetUBDsByValIndexKey(valAddr) ubdStore := prefix.NewStore(store, srcValPrefix) - pageRes, err := query.Paginate(ubdStore, req.Pagination, func(key []byte, value []byte) error { + pageRes, err := query.Paginate(ctx, ubdStore, req.Pagination, func(key []byte, value []byte) error { storeKey := types.GetUBDKeyFromValIndexKey(append(srcValPrefix, key...)) storeValue := store.Get(storeKey) @@ -282,7 +272,7 @@ func (k Querier) DelegatorDelegations(c context.Context, req *types.QueryDelegat store := ctx.KVStore(k.storeKey) delStore := prefix.NewStore(store, types.GetDelegationsKey(delAddr)) - pageRes, err := query.Paginate(delStore, req.Pagination, func(key []byte, value []byte) error { + pageRes, err := query.Paginate(ctx, delStore, req.Pagination, func(key []byte, value []byte) error { delegation, err := types.UnmarshalDelegation(k.cdc, value) if err != nil { return err @@ -354,7 +344,7 @@ func (k Querier) DelegatorUnbondingDelegations(c context.Context, req *types.Que } unbStore := prefix.NewStore(store, types.GetUBDsKey(delAddr)) - pageRes, err := query.Paginate(unbStore, req.Pagination, func(key []byte, value []byte) error { + pageRes, err := query.Paginate(ctx, unbStore, req.Pagination, func(key []byte, value []byte) error { unbond, err := types.UnmarshalUBD(k.cdc, value) if err != nil { return err @@ -405,9 +395,9 @@ func (k Querier) Redelegations(c context.Context, req *types.QueryRedelegationsR redels, err = queryRedelegation(ctx, k, req) pageRes = &query.PageResponse{} case req.DelegatorAddr == "" && req.SrcValidatorAddr != "" && req.DstValidatorAddr == "": - redels, pageRes, err = queryRedelegationsFromSrcValidator(store, k, req) + redels, pageRes, err = queryRedelegationsFromSrcValidator(ctx, store, k, req) default: - redels, pageRes, err = queryAllRedelegations(store, k, req) + redels, pageRes, err = queryAllRedelegations(ctx, store, k, req) } if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -439,7 +429,7 @@ func (k Querier) DelegatorValidators(c context.Context, req *types.QueryDelegato } delStore := prefix.NewStore(store, types.GetDelegationsKey(delAddr)) - pageRes, err := query.Paginate(delStore, req.Pagination, func(key []byte, value []byte) error { + pageRes, err := query.Paginate(ctx, delStore, req.Pagination, func(key []byte, value []byte) error { delegation, err := types.UnmarshalDelegation(k.cdc, value) if err != nil { return err @@ -513,7 +503,7 @@ func queryRedelegation(ctx sdk.Context, k Querier, req *types.QueryRedelegations return redels, err } -func queryRedelegationsFromSrcValidator(store sdk.KVStore, k Querier, req *types.QueryRedelegationsRequest) (redels types.Redelegations, res *query.PageResponse, err error) { +func queryRedelegationsFromSrcValidator(ctx sdk.Context, store sdk.KVStore, k Querier, req *types.QueryRedelegationsRequest) (redels types.Redelegations, res *query.PageResponse, err error) { valAddr, err := sdk.ValAddressFromBech32(req.SrcValidatorAddr) if err != nil { return nil, nil, err @@ -521,7 +511,7 @@ func queryRedelegationsFromSrcValidator(store sdk.KVStore, k Querier, req *types srcValPrefix := types.GetREDsFromValSrcIndexKey(valAddr) redStore := prefix.NewStore(store, srcValPrefix) - res, err = query.Paginate(redStore, req.Pagination, func(key []byte, value []byte) error { + res, err = query.Paginate(ctx, redStore, req.Pagination, func(key []byte, value []byte) error { storeKey := types.GetREDKeyFromValSrcIndexKey(append(srcValPrefix, key...)) storeValue := store.Get(storeKey) red, err := types.UnmarshalRED(k.cdc, storeValue) @@ -535,14 +525,14 @@ func queryRedelegationsFromSrcValidator(store sdk.KVStore, k Querier, req *types return redels, res, err } -func queryAllRedelegations(store sdk.KVStore, k Querier, req *types.QueryRedelegationsRequest) (redels types.Redelegations, res *query.PageResponse, err error) { +func queryAllRedelegations(ctx sdk.Context, store sdk.KVStore, k Querier, req *types.QueryRedelegationsRequest) (redels types.Redelegations, res *query.PageResponse, err error) { delAddr, err := sdk.AccAddressFromBech32(req.DelegatorAddr) if err != nil { return nil, nil, err } redStore := prefix.NewStore(store, types.GetREDsKey(delAddr)) - res, err = query.Paginate(redStore, req.Pagination, func(key []byte, value []byte) error { + res, err = query.Paginate(ctx, redStore, req.Pagination, func(key []byte, value []byte) error { redelegation, err := types.UnmarshalRED(k.cdc, value) if err != nil { return err diff --git a/sei-ibc-go/modules/apps/transfer/keeper/grpc_query.go b/sei-ibc-go/modules/apps/transfer/keeper/grpc_query.go index 30a355af44..92fae5a3ac 100644 --- a/sei-ibc-go/modules/apps/transfer/keeper/grpc_query.go +++ b/sei-ibc-go/modules/apps/transfer/keeper/grpc_query.go @@ -53,7 +53,7 @@ func (q Keeper) DenomTraces(c context.Context, req *types.QueryDenomTracesReques traces := types.Traces{} store := prefix.NewStore(ctx.KVStore(q.storeKey), types.DenomTraceKey) - pageRes, err := query.Paginate(store, req.Pagination, func(_, value []byte) error { + pageRes, err := query.Paginate(ctx, store, req.Pagination, func(_, value []byte) error { result, err := q.UnmarshalDenomTrace(value) if err != nil { return err diff --git a/sei-ibc-go/modules/core/02-client/keeper/grpc_query.go b/sei-ibc-go/modules/core/02-client/keeper/grpc_query.go index cb3ada8587..9a0c6165da 100644 --- a/sei-ibc-go/modules/core/02-client/keeper/grpc_query.go +++ b/sei-ibc-go/modules/core/02-client/keeper/grpc_query.go @@ -63,7 +63,7 @@ func (q Keeper) ClientStates(c context.Context, req *types.QueryClientStatesRequ clientStates := types.IdentifiedClientStates{} store := prefix.NewStore(ctx.KVStore(q.storeKey), host.KeyClientStorePrefix) - pageRes, err := query.FilteredPaginate(store, req.Pagination, func(key, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginate(ctx, store, req.Pagination, func(key, value []byte, accumulate bool) (bool, error) { keySplit := strings.Split(string(key), "/") if keySplit[len(keySplit)-1] != "clientState" { return false, nil @@ -159,7 +159,7 @@ func (q Keeper) ConsensusStates(c context.Context, req *types.QueryConsensusStat consensusStates := []types.ConsensusStateWithHeight{} store := prefix.NewStore(ctx.KVStore(q.storeKey), host.FullClientKey(req.ClientId, []byte(fmt.Sprintf("%s/", host.KeyConsensusStatePrefix)))) - pageRes, err := query.FilteredPaginate(store, req.Pagination, func(key, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginate(ctx, store, req.Pagination, func(key, value []byte, accumulate bool) (bool, error) { // filter any metadata stored under consensus state key if bytes.Contains(key, []byte("/")) { return false, nil @@ -203,7 +203,7 @@ func (q Keeper) ConsensusStateHeights(c context.Context, req *types.QueryConsens var consensusStateHeights []types.Height store := prefix.NewStore(ctx.KVStore(q.storeKey), host.FullClientKey(req.ClientId, []byte(fmt.Sprintf("%s/", host.KeyConsensusStatePrefix)))) - pageRes, err := query.FilteredPaginate(store, req.Pagination, func(key, _ []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginate(ctx, store, req.Pagination, func(key, _ []byte, accumulate bool) (bool, error) { // filter any metadata stored under consensus state key if bytes.Contains(key, []byte("/")) { return false, nil diff --git a/sei-ibc-go/modules/core/03-connection/keeper/grpc_query.go b/sei-ibc-go/modules/core/03-connection/keeper/grpc_query.go index c513eba658..af16f3fbc3 100644 --- a/sei-ibc-go/modules/core/03-connection/keeper/grpc_query.go +++ b/sei-ibc-go/modules/core/03-connection/keeper/grpc_query.go @@ -53,7 +53,7 @@ func (q Keeper) Connections(c context.Context, req *types.QueryConnectionsReques connections := []*types.IdentifiedConnection{} store := prefix.NewStore(ctx.KVStore(q.storeKey), []byte(host.KeyConnectionPrefix)) - pageRes, err := query.Paginate(store, req.Pagination, func(key, value []byte) error { + pageRes, err := query.Paginate(ctx, store, req.Pagination, func(key, value []byte) error { var result types.ConnectionEnd if err := q.cdc.Unmarshal(value, &result); err != nil { return err diff --git a/sei-ibc-go/modules/core/04-channel/keeper/grpc_query.go b/sei-ibc-go/modules/core/04-channel/keeper/grpc_query.go index b21e14821a..0507aec2af 100644 --- a/sei-ibc-go/modules/core/04-channel/keeper/grpc_query.go +++ b/sei-ibc-go/modules/core/04-channel/keeper/grpc_query.go @@ -54,7 +54,7 @@ func (q Keeper) Channels(c context.Context, req *types.QueryChannelsRequest) (*t channels := []*types.IdentifiedChannel{} store := prefix.NewStore(ctx.KVStore(q.storeKey), []byte(host.KeyChannelEndPrefix)) - pageRes, err := query.Paginate(store, req.Pagination, func(key, value []byte) error { + pageRes, err := query.Paginate(ctx, store, req.Pagination, func(key, value []byte) error { var result types.Channel if err := q.cdc.Unmarshal(value, &result); err != nil { return err @@ -96,7 +96,7 @@ func (q Keeper) ConnectionChannels(c context.Context, req *types.QueryConnection channels := []*types.IdentifiedChannel{} store := prefix.NewStore(ctx.KVStore(q.storeKey), []byte(host.KeyChannelEndPrefix)) - pageRes, err := query.Paginate(store, req.Pagination, func(key, value []byte) error { + pageRes, err := query.Paginate(ctx, store, req.Pagination, func(key, value []byte) error { var result types.Channel if err := q.cdc.Unmarshal(value, &result); err != nil { return err @@ -238,7 +238,7 @@ func (q Keeper) PacketCommitments(c context.Context, req *types.QueryPacketCommi commitments := []*types.PacketState{} store := prefix.NewStore(ctx.KVStore(q.storeKey), []byte(host.PacketCommitmentPrefixPath(req.PortId, req.ChannelId))) - pageRes, err := query.Paginate(store, req.Pagination, func(key, value []byte) error { + pageRes, err := query.Paginate(ctx, store, req.Pagination, func(key, value []byte) error { keySplit := strings.Split(string(key), "/") sequence, err := strconv.ParseUint(keySplit[len(keySplit)-1], 10, 64) @@ -345,7 +345,7 @@ func (q Keeper) PacketAcknowledgements(c context.Context, req *types.QueryPacket }, nil } - pageRes, err := query.Paginate(store, req.Pagination, func(key, value []byte) error { + pageRes, err := query.Paginate(ctx, store, req.Pagination, func(key, value []byte) error { keySplit := strings.Split(string(key), "/") sequence, err := strconv.ParseUint(keySplit[len(keySplit)-1], 10, 64) diff --git a/sei-wasmd/x/wasm/keeper/querier.go b/sei-wasmd/x/wasm/keeper/querier.go index ca3f3e19bd..39d20034d7 100644 --- a/sei-wasmd/x/wasm/keeper/querier.go +++ b/sei-wasmd/x/wasm/keeper/querier.go @@ -63,7 +63,7 @@ func (q grpcQuerier) ContractHistory(c context.Context, req *types.QueryContract r := make([]types.ContractCodeHistoryEntry, 0) prefixStore := prefix.NewStore(ctx.KVStore(q.storeKey), types.GetContractCodeHistoryElementPrefix(contractAddr)) - pageRes, err := query.FilteredPaginate(prefixStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginate(ctx, prefixStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { if accumulate { var e types.ContractCodeHistoryEntry if err := q.cdc.Unmarshal(value, &e); err != nil { @@ -95,7 +95,7 @@ func (q grpcQuerier) ContractsByCode(c context.Context, req *types.QueryContract r := make([]string, 0) prefixStore := prefix.NewStore(ctx.KVStore(q.storeKey), types.GetContractByCodeIDSecondaryIndexPrefix(req.CodeId)) - pageRes, err := query.FilteredPaginate(prefixStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginate(ctx, prefixStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { if accumulate { var contractAddr sdk.AccAddress = key[types.AbsoluteTxPositionLen:] r = append(r, contractAddr.String()) @@ -126,7 +126,7 @@ func (q grpcQuerier) AllContractState(c context.Context, req *types.QueryAllCont r := make([]types.Model, 0) prefixStore := prefix.NewStore(ctx.KVStore(q.storeKey), types.GetContractStorePrefix(contractAddr)) - pageRes, err := query.FilteredPaginate(prefixStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginate(ctx, prefixStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { if accumulate { r = append(r, types.Model{ Key: key, @@ -236,7 +236,7 @@ func (q grpcQuerier) Codes(c context.Context, req *types.QueryCodesRequest) (*ty ctx := sdk.UnwrapSDKContext(c) r := make([]types.CodeInfoResponse, 0) prefixStore := prefix.NewStore(ctx.KVStore(q.storeKey), types.CodeKeyPrefix) - pageRes, err := query.FilteredPaginate(prefixStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginate(ctx, prefixStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) { if accumulate { var c types.CodeInfo if err := q.cdc.Unmarshal(value, &c); err != nil { @@ -302,7 +302,7 @@ func (q grpcQuerier) PinnedCodes(c context.Context, req *types.QueryPinnedCodesR r := make([]uint64, 0) prefixStore := prefix.NewStore(ctx.KVStore(q.storeKey), types.PinnedCodeIndexPrefix) - pageRes, err := query.FilteredPaginate(prefixStore, req.Pagination, func(key []byte, _ []byte, accumulate bool) (bool, error) { + pageRes, err := query.FilteredPaginate(ctx, prefixStore, req.Pagination, func(key []byte, _ []byte, accumulate bool) (bool, error) { if accumulate { r = append(r, sdk.BigEndianToUint64(key)) } diff --git a/x/tokenfactory/keeper/creators.go b/x/tokenfactory/keeper/creators.go index 0e155c3485..0686dd44e4 100644 --- a/x/tokenfactory/keeper/creators.go +++ b/x/tokenfactory/keeper/creators.go @@ -13,7 +13,7 @@ func (k Keeper) addDenomFromCreator(ctx sdk.Context, creator, denom string) { func (k Keeper) getDenomsFromCreator(ctx sdk.Context, creator string, pagination *query.PageRequest) ([]string, *query.PageResponse, error) { store := k.GetCreatorPrefixStore(ctx, creator) var denoms []string - pageRes, err := query.Paginate(store, pagination, func(key []byte, _ []byte) error { + pageRes, err := query.Paginate(ctx, store, pagination, func(key []byte, _ []byte) error { denoms = append(denoms, string(key)) return nil })