diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index 1724161612..df1fe4c7d3 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -4,6 +4,7 @@ import ( "fmt" "runtime" "strings" + "time" storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" "github.com/sei-protocol/sei-chain/sei-cosmos/telemetry" @@ -26,6 +27,49 @@ const ( // DefaultGRPCWebAddress defines the default address to bind the gRPC-web server to. DefaultGRPCWebAddress = "0.0.0.0:9091" + // DefaultGRPCMaxOpenConnections defines the default maximum number of + // simultaneous open connections for the gRPC server. 0 means unlimited. + DefaultGRPCMaxOpenConnections = 1000 + + // DefaultGRPCMaxRecvMsgSize defines the default maximum message size in bytes + // that the gRPC server can receive (4 MB), mirroring gRPC's own default. + DefaultGRPCMaxRecvMsgSize = 4 * 1024 * 1024 + + // DefaultGRPCMaxConnectionIdle is the default duration after which an idle + // connection (one with no in-flight RPCs) is closed via GoAway. It is bounded + // by default to reclaim abandoned connection slots — which matter now that the + // listener is capped by MaxOpenConnections — while staying long enough not to + // churn clients that query on a shorter cadence. The real DoS bound is the + // connection cap; this only reaps dormant connections. 0 means infinity. + DefaultGRPCMaxConnectionIdle = 5 * time.Minute + + // The remaining keepalive defaults below mirror gRPC's own defaults, so they + // are opt-in for operators and do not change behavior unless configured. + + // DefaultGRPCMaxConnectionAge is the default maximum duration a connection may + // exist before it is closed. 0 means infinity. + DefaultGRPCMaxConnectionAge = time.Duration(0) + + // DefaultGRPCMaxConnectionAgeGrace is the default additive period after + // MaxConnectionAge during which the connection is forcibly closed. 0 means infinity. + DefaultGRPCMaxConnectionAgeGrace = time.Duration(0) + + // DefaultGRPCKeepaliveTime is the default interval after which, if the server + // sees no activity, it pings the client to check liveness. + DefaultGRPCKeepaliveTime = 2 * time.Hour + + // DefaultGRPCKeepaliveTimeout is the default duration the server waits for a + // keepalive ping ack before closing the connection. + DefaultGRPCKeepaliveTimeout = 20 * time.Second + + // DefaultGRPCKeepaliveMinTime is the default minimum interval a client must + // wait between keepalive pings; pings more frequent than this are penalized. + DefaultGRPCKeepaliveMinTime = 5 * time.Minute + + // DefaultGRPCKeepalivePermitWithoutStream defines whether the server allows + // keepalive pings even when there are no active streams. + DefaultGRPCKeepalivePermitWithoutStream = false + // DefaultOccEanbled defines whether to use OCC for tx processing DefaultOccEnabled = true ) @@ -168,6 +212,43 @@ type GRPCConfig struct { // Address defines the API server to listen on Address string `mapstructure:"address"` + + // MaxRecvMsgSize defines the maximum message size in bytes the server can + // receive. It bounds per-request memory allocation before the rate limiter + // fires. Defaults to 4 MB. + MaxRecvMsgSize int `mapstructure:"max-recv-msg-size"` + + // MaxOpenConnections defines the maximum number of simultaneous open + // connections. 0 means unlimited. + MaxOpenConnections uint `mapstructure:"max-open-connections"` + + // MaxConnectionIdle is the duration after which an idle connection is closed. + // 0 means infinity. + MaxConnectionIdle time.Duration `mapstructure:"max-connection-idle"` + + // MaxConnectionAge is the maximum duration a connection may exist before it + // is closed (a jitter is added by gRPC). 0 means infinity. + MaxConnectionAge time.Duration `mapstructure:"max-connection-age"` + + // MaxConnectionAgeGrace is an additive period after MaxConnectionAge during + // which the connection is forcibly closed. 0 means infinity. + MaxConnectionAgeGrace time.Duration `mapstructure:"max-connection-age-grace"` + + // KeepaliveTime is the interval after which, if the server sees no activity, + // it pings the client to check liveness. + KeepaliveTime time.Duration `mapstructure:"keepalive-time"` + + // KeepaliveTimeout is the duration the server waits for a keepalive ping ack + // before closing the connection. + KeepaliveTimeout time.Duration `mapstructure:"keepalive-timeout"` + + // KeepaliveMinTime is the minimum interval a client must wait between + // keepalive pings; clients pinging more frequently are penalized. + KeepaliveMinTime time.Duration `mapstructure:"keepalive-min-time"` + + // KeepalivePermitWithoutStream defines whether the server allows keepalive + // pings even when there are no active streams. + KeepalivePermitWithoutStream bool `mapstructure:"keepalive-permit-without-stream"` } // GRPCWebConfig defines configuration for the gRPC-web server. @@ -280,8 +361,17 @@ func DefaultConfig() *Config { RPCMaxBodyBytes: 1000000, }, GRPC: GRPCConfig{ - Enable: true, - Address: DefaultGRPCAddress, + Enable: true, + Address: DefaultGRPCAddress, + MaxRecvMsgSize: DefaultGRPCMaxRecvMsgSize, + MaxOpenConnections: DefaultGRPCMaxOpenConnections, + MaxConnectionIdle: DefaultGRPCMaxConnectionIdle, + MaxConnectionAge: DefaultGRPCMaxConnectionAge, + MaxConnectionAgeGrace: DefaultGRPCMaxConnectionAgeGrace, + KeepaliveTime: DefaultGRPCKeepaliveTime, + KeepaliveTimeout: DefaultGRPCKeepaliveTimeout, + KeepaliveMinTime: DefaultGRPCKeepaliveMinTime, + KeepalivePermitWithoutStream: DefaultGRPCKeepalivePermitWithoutStream, }, Rosetta: RosettaConfig{ Enable: false, @@ -357,6 +447,34 @@ func GetConfig(v *viper.Viper) (Config, error) { flatKVConfig.EnableReadWriteMetrics = v.GetBool("state-commit.flatkv.enable-read-write-metrics") } + // Apply in-code defaults when keys are absent so that nodes upgrading with an + // older app.toml (which lacks these keys) remain bounded rather than running + // with unlimited connections / message sizes. + grpcMaxRecvMsgSize := DefaultGRPCMaxRecvMsgSize + if v.IsSet("grpc.max-recv-msg-size") { + grpcMaxRecvMsgSize = v.GetInt("grpc.max-recv-msg-size") + } + grpcMaxOpenConnections := uint(DefaultGRPCMaxOpenConnections) + if v.IsSet("grpc.max-open-connections") { + grpcMaxOpenConnections = v.GetUint("grpc.max-open-connections") + } + grpcMaxConnectionIdle := DefaultGRPCMaxConnectionIdle + if v.IsSet("grpc.max-connection-idle") { + grpcMaxConnectionIdle = v.GetDuration("grpc.max-connection-idle") + } + grpcKeepaliveTime := DefaultGRPCKeepaliveTime + if v.IsSet("grpc.keepalive-time") { + grpcKeepaliveTime = v.GetDuration("grpc.keepalive-time") + } + grpcKeepaliveTimeout := DefaultGRPCKeepaliveTimeout + if v.IsSet("grpc.keepalive-timeout") { + grpcKeepaliveTimeout = v.GetDuration("grpc.keepalive-timeout") + } + grpcKeepaliveMinTime := DefaultGRPCKeepaliveMinTime + if v.IsSet("grpc.keepalive-min-time") { + grpcKeepaliveMinTime = v.GetDuration("grpc.keepalive-min-time") + } + return Config{ BaseConfig: BaseConfig{ MinGasPrices: v.GetString("minimum-gas-prices"), @@ -400,8 +518,17 @@ func GetConfig(v *viper.Viper) (Config, error) { Offline: v.GetBool("rosetta.offline"), }, GRPC: GRPCConfig{ - Enable: v.GetBool("grpc.enable"), - Address: v.GetString("grpc.address"), + Enable: v.GetBool("grpc.enable"), + Address: v.GetString("grpc.address"), + MaxRecvMsgSize: grpcMaxRecvMsgSize, + MaxOpenConnections: grpcMaxOpenConnections, + MaxConnectionIdle: grpcMaxConnectionIdle, + MaxConnectionAge: v.GetDuration("grpc.max-connection-age"), + MaxConnectionAgeGrace: v.GetDuration("grpc.max-connection-age-grace"), + KeepaliveTime: grpcKeepaliveTime, + KeepaliveTimeout: grpcKeepaliveTimeout, + KeepaliveMinTime: grpcKeepaliveMinTime, + KeepalivePermitWithoutStream: v.GetBool("grpc.keepalive-permit-without-stream"), }, GRPCWeb: GRPCWebConfig{ Enable: v.GetBool("grpc-web.enable"), diff --git a/sei-cosmos/server/config/config_test.go b/sei-cosmos/server/config/config_test.go index 26b984573f..f75176aa61 100644 --- a/sei-cosmos/server/config/config_test.go +++ b/sei-cosmos/server/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "bytes" "testing" + "time" tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config" "github.com/spf13/viper" @@ -68,6 +69,87 @@ func TestDefaultGRPCConfig(t *testing.T) { cfg := DefaultConfig() require.True(t, cfg.GRPC.Enable) require.Equal(t, DefaultGRPCAddress, cfg.GRPC.Address) + require.Equal(t, DefaultGRPCMaxRecvMsgSize, cfg.GRPC.MaxRecvMsgSize) + require.Equal(t, uint(DefaultGRPCMaxOpenConnections), cfg.GRPC.MaxOpenConnections) + require.Equal(t, DefaultGRPCMaxConnectionIdle, cfg.GRPC.MaxConnectionIdle) + require.Equal(t, 5*time.Minute, cfg.GRPC.MaxConnectionIdle) + require.Equal(t, DefaultGRPCMaxConnectionAge, cfg.GRPC.MaxConnectionAge) + require.Equal(t, DefaultGRPCMaxConnectionAgeGrace, cfg.GRPC.MaxConnectionAgeGrace) + require.Equal(t, DefaultGRPCKeepaliveTime, cfg.GRPC.KeepaliveTime) + require.Equal(t, DefaultGRPCKeepaliveTimeout, cfg.GRPC.KeepaliveTimeout) + require.Equal(t, DefaultGRPCKeepaliveMinTime, cfg.GRPC.KeepaliveMinTime) + require.Equal(t, DefaultGRPCKeepalivePermitWithoutStream, cfg.GRPC.KeepalivePermitWithoutStream) +} + +// seedViperWithDefaultConfig renders the default app config template and reads +// it into a fresh viper instance, mirroring how seid loads app.toml. +func seedViperWithDefaultConfig(t *testing.T) *viper.Viper { + t.Helper() + var buf bytes.Buffer + require.NoError(t, configTemplate.Execute(&buf, DefaultConfig())) + v := viper.New() + v.SetConfigType("toml") + require.NoError(t, v.ReadConfig(&buf)) + return v +} + +// TestGetConfigGRPCDefaultsWhenAbsent ensures a node upgrading with an older +// app.toml (which lacks the new gRPC keys) still gets the bounded in-code +// defaults rather than zero/unlimited values. +func TestGetConfigGRPCDefaultsWhenAbsent(t *testing.T) { + // Minimal app.toml that predates the new gRPC keys. global-labels is the + // only key GetConfig hard-requires. + const legacyAppToml = ` +[telemetry] +global-labels = [] + +[grpc] +enable = true +address = "0.0.0.0:9090" +` + v := viper.New() + v.SetConfigType("toml") + require.NoError(t, v.ReadConfig(bytes.NewBufferString(legacyAppToml))) + require.False(t, v.IsSet("grpc.max-recv-msg-size")) + require.False(t, v.IsSet("grpc.max-open-connections")) + require.False(t, v.IsSet("grpc.max-connection-idle")) + + cfg, err := GetConfig(v) + require.NoError(t, err) + require.Equal(t, DefaultGRPCMaxRecvMsgSize, cfg.GRPC.MaxRecvMsgSize) + require.Equal(t, uint(DefaultGRPCMaxOpenConnections), cfg.GRPC.MaxOpenConnections) + // The bounded idle default must survive an older app.toml that omits the key. + require.Equal(t, DefaultGRPCMaxConnectionIdle, cfg.GRPC.MaxConnectionIdle) + require.Equal(t, DefaultGRPCKeepaliveTime, cfg.GRPC.KeepaliveTime) + require.Equal(t, DefaultGRPCKeepaliveTimeout, cfg.GRPC.KeepaliveTimeout) + require.Equal(t, DefaultGRPCKeepaliveMinTime, cfg.GRPC.KeepaliveMinTime) +} + +// TestGetConfigGRPCOverrides ensures operator-provided values override the +// in-code defaults. +func TestGetConfigGRPCOverrides(t *testing.T) { + v := seedViperWithDefaultConfig(t) + v.Set("grpc.max-recv-msg-size", 8*1024*1024) + v.Set("grpc.max-open-connections", 50) + v.Set("grpc.max-connection-idle", "5m") + v.Set("grpc.max-connection-age", "30m") + v.Set("grpc.max-connection-age-grace", "1m") + v.Set("grpc.keepalive-time", "1m") + v.Set("grpc.keepalive-timeout", "10s") + v.Set("grpc.keepalive-min-time", "30s") + v.Set("grpc.keepalive-permit-without-stream", true) + + cfg, err := GetConfig(v) + require.NoError(t, err) + require.Equal(t, 8*1024*1024, cfg.GRPC.MaxRecvMsgSize) + require.Equal(t, uint(50), cfg.GRPC.MaxOpenConnections) + require.Equal(t, 5*time.Minute, cfg.GRPC.MaxConnectionIdle) + require.Equal(t, 30*time.Minute, cfg.GRPC.MaxConnectionAge) + require.Equal(t, time.Minute, cfg.GRPC.MaxConnectionAgeGrace) + require.Equal(t, time.Minute, cfg.GRPC.KeepaliveTime) + require.Equal(t, 10*time.Second, cfg.GRPC.KeepaliveTimeout) + require.Equal(t, 30*time.Second, cfg.GRPC.KeepaliveMinTime) + require.True(t, cfg.GRPC.KeepalivePermitWithoutStream) } func TestDefaultGRPCWebConfig(t *testing.T) { diff --git a/sei-cosmos/server/config/toml.go b/sei-cosmos/server/config/toml.go index ac931084be..b4eb1c986a 100644 --- a/sei-cosmos/server/config/toml.go +++ b/sei-cosmos/server/config/toml.go @@ -189,6 +189,34 @@ enable = {{ .GRPC.Enable }} # Address defines the gRPC server address to bind to. address = "{{ .GRPC.Address }}" +# MaxRecvMsgSize defines the maximum message size in bytes the server can receive. +# It bounds per-request memory allocation before the rate limiter fires. Default 4 MB. +max-recv-msg-size = {{ .GRPC.MaxRecvMsgSize }} + +# MaxOpenConnections defines the maximum number of simultaneous open connections. 0 means unlimited. +max-open-connections = {{ .GRPC.MaxOpenConnections }} + +# MaxConnectionIdle is the duration after which an idle connection is closed (e.g. "5m"). 0 means infinity. +max-connection-idle = "{{ .GRPC.MaxConnectionIdle }}" + +# MaxConnectionAge is the maximum duration a connection may exist before it is closed (e.g. "30m"). 0 means infinity. +max-connection-age = "{{ .GRPC.MaxConnectionAge }}" + +# MaxConnectionAgeGrace is an additive period after max-connection-age during which the connection is forcibly closed. 0 means infinity. +max-connection-age-grace = "{{ .GRPC.MaxConnectionAgeGrace }}" + +# KeepaliveTime is the interval after which, with no activity, the server pings the client to check liveness. +keepalive-time = "{{ .GRPC.KeepaliveTime }}" + +# KeepaliveTimeout is the duration the server waits for a keepalive ping ack before closing the connection. +keepalive-timeout = "{{ .GRPC.KeepaliveTimeout }}" + +# KeepaliveMinTime is the minimum interval a client must wait between keepalive pings; more frequent pings are penalized. +keepalive-min-time = "{{ .GRPC.KeepaliveMinTime }}" + +# KeepalivePermitWithoutStream defines whether the server allows keepalive pings even when there are no active streams. +keepalive-permit-without-stream = {{ .GRPC.KeepalivePermitWithoutStream }} + ############################################################################### ### gRPC Web Configuration (Auto-managed) ### ############################################################################### diff --git a/sei-cosmos/server/grpc/server.go b/sei-cosmos/server/grpc/server.go index cdb4517071..cf24007c15 100644 --- a/sei-cosmos/server/grpc/server.go +++ b/sei-cosmos/server/grpc/server.go @@ -2,21 +2,46 @@ package grpc import ( "fmt" + "math" "net" "time" + "golang.org/x/net/netutil" "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" "github.com/sei-protocol/sei-chain/sei-cosmos/client" + "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" "github.com/sei-protocol/sei-chain/sei-cosmos/server/grpc/gogoreflection" reflection "github.com/sei-protocol/sei-chain/sei-cosmos/server/grpc/reflection/v2alpha1" "github.com/sei-protocol/sei-chain/sei-cosmos/server/types" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" ) -// StartGRPCServer starts a gRPC server on the given address. -func StartGRPCServer(clientCtx client.Context, app types.Application, address string) (*grpc.Server, error) { - grpcSrv := grpc.NewServer(grpc.MaxConcurrentStreams(100)) +// StartGRPCServer starts a gRPC server on the address given by cfg. +func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config.GRPCConfig) (*grpc.Server, error) { + maxRecvMsgSize := cfg.MaxRecvMsgSize + if maxRecvMsgSize <= 0 { + maxRecvMsgSize = config.DefaultGRPCMaxRecvMsgSize + } + + grpcSrv := grpc.NewServer( + grpc.MaxConcurrentStreams(100), + // MaxRecvMsgSize bounds per-request memory allocation before the rate + // limiter fires, preventing an oversized request from exhausting memory. + grpc.MaxRecvMsgSize(maxRecvMsgSize), + grpc.KeepaliveParams(keepalive.ServerParameters{ + MaxConnectionIdle: cfg.MaxConnectionIdle, + MaxConnectionAge: cfg.MaxConnectionAge, + MaxConnectionAgeGrace: cfg.MaxConnectionAgeGrace, + Time: cfg.KeepaliveTime, + Timeout: cfg.KeepaliveTimeout, + }), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: cfg.KeepaliveMinTime, + PermitWithoutStream: cfg.KeepalivePermitWithoutStream, + }), + ) app.RegisterGRPCServer(grpcSrv) // reflection allows consumers to build dynamic clients that can write // to any cosmos-sdk application without relying on application packages at compile time @@ -38,10 +63,17 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, address st // Reflection allows external clients to see what services and methods // the gRPC server exposes. gogoreflection.Register(grpcSrv) - listener, err := net.Listen("tcp", address) + listener, err := net.Listen("tcp", cfg.Address) if err != nil { return nil, err } + if cfg.MaxOpenConnections > 0 { + maxConn := cfg.MaxOpenConnections + if maxConn > math.MaxInt { + maxConn = math.MaxInt + } + listener = netutil.LimitListener(listener, int(maxConn)) //nolint:gosec // G115: clamped to math.MaxInt above + } errCh := make(chan error) go func() { diff --git a/sei-cosmos/server/start.go b/sei-cosmos/server/start.go index 514c922cc7..05e602d107 100644 --- a/sei-cosmos/server/start.go +++ b/sei-cosmos/server/start.go @@ -399,7 +399,7 @@ func startInProcess( } if config.GRPC.Enable { - grpcSrv, err := servergrpc.StartGRPCServer(clientCtx, app, config.GRPC.Address) + grpcSrv, err := servergrpc.StartGRPCServer(clientCtx, app, config.GRPC) if err != nil { return err } diff --git a/sei-cosmos/testutil/network/util.go b/sei-cosmos/testutil/network/util.go index bd6d2c8124..71c589ae63 100644 --- a/sei-cosmos/testutil/network/util.go +++ b/sei-cosmos/testutil/network/util.go @@ -103,7 +103,7 @@ func startInProcess(cfg Config, val *Validator) error { } if val.AppConfig.GRPC.Enable { - grpcSrv, err := servergrpc.StartGRPCServer(val.ClientCtx, app, val.AppConfig.GRPC.Address) + grpcSrv, err := servergrpc.StartGRPCServer(val.ClientCtx, app, val.AppConfig.GRPC) if err != nil { return err }