Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 131 additions & 4 deletions sei-cosmos/server/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"runtime"
"strings"
"time"

storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types"
"github.com/sei-protocol/sei-chain/sei-cosmos/telemetry"
Expand All @@ -26,6 +27,49 @@ const (
// DefaultGRPCWebAddress defines the default address to bind the gRPC-web server to.
DefaultGRPCWebAddress = "0.0.0.0:9091"

// DefaultGRPCMaxOpenConnections defines the default maximum number of
// simultaneous open connections for the gRPC server. 0 means unlimited.
DefaultGRPCMaxOpenConnections = 1000

// DefaultGRPCMaxRecvMsgSize defines the default maximum message size in bytes
// that the gRPC server can receive (4 MB), mirroring gRPC's own default.
DefaultGRPCMaxRecvMsgSize = 4 * 1024 * 1024

// DefaultGRPCMaxConnectionIdle is the default duration after which an idle
// connection (one with no in-flight RPCs) is closed via GoAway. It is bounded
// by default to reclaim abandoned connection slots — which matter now that the
// listener is capped by MaxOpenConnections — while staying long enough not to
// churn clients that query on a shorter cadence. The real DoS bound is the
// connection cap; this only reaps dormant connections. 0 means infinity.
DefaultGRPCMaxConnectionIdle = 5 * time.Minute

// The remaining keepalive defaults below mirror gRPC's own defaults, so they
// are opt-in for operators and do not change behavior unless configured.

// DefaultGRPCMaxConnectionAge is the default maximum duration a connection may
// exist before it is closed. 0 means infinity.
DefaultGRPCMaxConnectionAge = time.Duration(0)

// DefaultGRPCMaxConnectionAgeGrace is the default additive period after
// MaxConnectionAge during which the connection is forcibly closed. 0 means infinity.
DefaultGRPCMaxConnectionAgeGrace = time.Duration(0)

// DefaultGRPCKeepaliveTime is the default interval after which, if the server
// sees no activity, it pings the client to check liveness.
DefaultGRPCKeepaliveTime = 2 * time.Hour

// DefaultGRPCKeepaliveTimeout is the default duration the server waits for a
// keepalive ping ack before closing the connection.
DefaultGRPCKeepaliveTimeout = 20 * time.Second

// DefaultGRPCKeepaliveMinTime is the default minimum interval a client must
// wait between keepalive pings; pings more frequent than this are penalized.
DefaultGRPCKeepaliveMinTime = 5 * time.Minute

// DefaultGRPCKeepalivePermitWithoutStream defines whether the server allows
// keepalive pings even when there are no active streams.
DefaultGRPCKeepalivePermitWithoutStream = false

// DefaultOccEanbled defines whether to use OCC for tx processing
DefaultOccEnabled = true
)
Expand Down Expand Up @@ -168,6 +212,43 @@ type GRPCConfig struct {

// Address defines the API server to listen on
Address string `mapstructure:"address"`

// MaxRecvMsgSize defines the maximum message size in bytes the server can
// receive. It bounds per-request memory allocation before the rate limiter
// fires. Defaults to 4 MB.
MaxRecvMsgSize int `mapstructure:"max-recv-msg-size"`

// MaxOpenConnections defines the maximum number of simultaneous open
// connections. 0 means unlimited.
MaxOpenConnections uint `mapstructure:"max-open-connections"`

// MaxConnectionIdle is the duration after which an idle connection is closed.
// 0 means infinity.
MaxConnectionIdle time.Duration `mapstructure:"max-connection-idle"`

// MaxConnectionAge is the maximum duration a connection may exist before it
// is closed (a jitter is added by gRPC). 0 means infinity.
MaxConnectionAge time.Duration `mapstructure:"max-connection-age"`

// MaxConnectionAgeGrace is an additive period after MaxConnectionAge during
// which the connection is forcibly closed. 0 means infinity.
MaxConnectionAgeGrace time.Duration `mapstructure:"max-connection-age-grace"`

// KeepaliveTime is the interval after which, if the server sees no activity,
// it pings the client to check liveness.
KeepaliveTime time.Duration `mapstructure:"keepalive-time"`

// KeepaliveTimeout is the duration the server waits for a keepalive ping ack
// before closing the connection.
KeepaliveTimeout time.Duration `mapstructure:"keepalive-timeout"`

// KeepaliveMinTime is the minimum interval a client must wait between
// keepalive pings; clients pinging more frequently are penalized.
KeepaliveMinTime time.Duration `mapstructure:"keepalive-min-time"`

// KeepalivePermitWithoutStream defines whether the server allows keepalive
// pings even when there are no active streams.
KeepalivePermitWithoutStream bool `mapstructure:"keepalive-permit-without-stream"`
}

// GRPCWebConfig defines configuration for the gRPC-web server.
Expand Down Expand Up @@ -280,8 +361,17 @@ func DefaultConfig() *Config {
RPCMaxBodyBytes: 1000000,
},
GRPC: GRPCConfig{
Enable: true,
Address: DefaultGRPCAddress,
Enable: true,
Address: DefaultGRPCAddress,
MaxRecvMsgSize: DefaultGRPCMaxRecvMsgSize,
MaxOpenConnections: DefaultGRPCMaxOpenConnections,
MaxConnectionIdle: DefaultGRPCMaxConnectionIdle,
MaxConnectionAge: DefaultGRPCMaxConnectionAge,
MaxConnectionAgeGrace: DefaultGRPCMaxConnectionAgeGrace,
KeepaliveTime: DefaultGRPCKeepaliveTime,
KeepaliveTimeout: DefaultGRPCKeepaliveTimeout,
KeepaliveMinTime: DefaultGRPCKeepaliveMinTime,
KeepalivePermitWithoutStream: DefaultGRPCKeepalivePermitWithoutStream,
},
Rosetta: RosettaConfig{
Enable: false,
Expand Down Expand Up @@ -357,6 +447,34 @@ func GetConfig(v *viper.Viper) (Config, error) {
flatKVConfig.EnableReadWriteMetrics = v.GetBool("state-commit.flatkv.enable-read-write-metrics")
}

// Apply in-code defaults when keys are absent so that nodes upgrading with an
// older app.toml (which lacks these keys) remain bounded rather than running
// with unlimited connections / message sizes.
grpcMaxRecvMsgSize := DefaultGRPCMaxRecvMsgSize
if v.IsSet("grpc.max-recv-msg-size") {
grpcMaxRecvMsgSize = v.GetInt("grpc.max-recv-msg-size")
}
grpcMaxOpenConnections := uint(DefaultGRPCMaxOpenConnections)
if v.IsSet("grpc.max-open-connections") {
grpcMaxOpenConnections = v.GetUint("grpc.max-open-connections")
}
grpcMaxConnectionIdle := DefaultGRPCMaxConnectionIdle
if v.IsSet("grpc.max-connection-idle") {
grpcMaxConnectionIdle = v.GetDuration("grpc.max-connection-idle")
}
grpcKeepaliveTime := DefaultGRPCKeepaliveTime
if v.IsSet("grpc.keepalive-time") {
grpcKeepaliveTime = v.GetDuration("grpc.keepalive-time")
}
grpcKeepaliveTimeout := DefaultGRPCKeepaliveTimeout
if v.IsSet("grpc.keepalive-timeout") {
grpcKeepaliveTimeout = v.GetDuration("grpc.keepalive-timeout")
}
grpcKeepaliveMinTime := DefaultGRPCKeepaliveMinTime
if v.IsSet("grpc.keepalive-min-time") {
grpcKeepaliveMinTime = v.GetDuration("grpc.keepalive-min-time")
}

return Config{
BaseConfig: BaseConfig{
MinGasPrices: v.GetString("minimum-gas-prices"),
Expand Down Expand Up @@ -400,8 +518,17 @@ func GetConfig(v *viper.Viper) (Config, error) {
Offline: v.GetBool("rosetta.offline"),
},
GRPC: GRPCConfig{
Enable: v.GetBool("grpc.enable"),
Address: v.GetString("grpc.address"),
Enable: v.GetBool("grpc.enable"),
Address: v.GetString("grpc.address"),
MaxRecvMsgSize: grpcMaxRecvMsgSize,
MaxOpenConnections: grpcMaxOpenConnections,
MaxConnectionIdle: grpcMaxConnectionIdle,
MaxConnectionAge: v.GetDuration("grpc.max-connection-age"),
MaxConnectionAgeGrace: v.GetDuration("grpc.max-connection-age-grace"),
KeepaliveTime: grpcKeepaliveTime,
KeepaliveTimeout: grpcKeepaliveTimeout,
KeepaliveMinTime: grpcKeepaliveMinTime,
KeepalivePermitWithoutStream: v.GetBool("grpc.keepalive-permit-without-stream"),
},
GRPCWeb: GRPCWebConfig{
Enable: v.GetBool("grpc-web.enable"),
Expand Down
82 changes: 82 additions & 0 deletions sei-cosmos/server/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"bytes"
"testing"
"time"

tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config"
"github.com/spf13/viper"
Expand Down Expand Up @@ -68,6 +69,87 @@ func TestDefaultGRPCConfig(t *testing.T) {
cfg := DefaultConfig()
require.True(t, cfg.GRPC.Enable)
require.Equal(t, DefaultGRPCAddress, cfg.GRPC.Address)
require.Equal(t, DefaultGRPCMaxRecvMsgSize, cfg.GRPC.MaxRecvMsgSize)
require.Equal(t, uint(DefaultGRPCMaxOpenConnections), cfg.GRPC.MaxOpenConnections)
require.Equal(t, DefaultGRPCMaxConnectionIdle, cfg.GRPC.MaxConnectionIdle)
require.Equal(t, 5*time.Minute, cfg.GRPC.MaxConnectionIdle)
require.Equal(t, DefaultGRPCMaxConnectionAge, cfg.GRPC.MaxConnectionAge)
require.Equal(t, DefaultGRPCMaxConnectionAgeGrace, cfg.GRPC.MaxConnectionAgeGrace)
require.Equal(t, DefaultGRPCKeepaliveTime, cfg.GRPC.KeepaliveTime)
require.Equal(t, DefaultGRPCKeepaliveTimeout, cfg.GRPC.KeepaliveTimeout)
require.Equal(t, DefaultGRPCKeepaliveMinTime, cfg.GRPC.KeepaliveMinTime)
require.Equal(t, DefaultGRPCKeepalivePermitWithoutStream, cfg.GRPC.KeepalivePermitWithoutStream)
}

// seedViperWithDefaultConfig renders the default app config template and reads
// it into a fresh viper instance, mirroring how seid loads app.toml.
func seedViperWithDefaultConfig(t *testing.T) *viper.Viper {
t.Helper()
var buf bytes.Buffer
require.NoError(t, configTemplate.Execute(&buf, DefaultConfig()))
v := viper.New()
v.SetConfigType("toml")
require.NoError(t, v.ReadConfig(&buf))
return v
}

// TestGetConfigGRPCDefaultsWhenAbsent ensures a node upgrading with an older
// app.toml (which lacks the new gRPC keys) still gets the bounded in-code
// defaults rather than zero/unlimited values.
func TestGetConfigGRPCDefaultsWhenAbsent(t *testing.T) {
// Minimal app.toml that predates the new gRPC keys. global-labels is the
// only key GetConfig hard-requires.
const legacyAppToml = `
[telemetry]
global-labels = []

[grpc]
enable = true
address = "0.0.0.0:9090"
`
v := viper.New()
v.SetConfigType("toml")
require.NoError(t, v.ReadConfig(bytes.NewBufferString(legacyAppToml)))
require.False(t, v.IsSet("grpc.max-recv-msg-size"))
require.False(t, v.IsSet("grpc.max-open-connections"))
require.False(t, v.IsSet("grpc.max-connection-idle"))

cfg, err := GetConfig(v)
require.NoError(t, err)
require.Equal(t, DefaultGRPCMaxRecvMsgSize, cfg.GRPC.MaxRecvMsgSize)
require.Equal(t, uint(DefaultGRPCMaxOpenConnections), cfg.GRPC.MaxOpenConnections)
// The bounded idle default must survive an older app.toml that omits the key.
require.Equal(t, DefaultGRPCMaxConnectionIdle, cfg.GRPC.MaxConnectionIdle)
require.Equal(t, DefaultGRPCKeepaliveTime, cfg.GRPC.KeepaliveTime)
require.Equal(t, DefaultGRPCKeepaliveTimeout, cfg.GRPC.KeepaliveTimeout)
require.Equal(t, DefaultGRPCKeepaliveMinTime, cfg.GRPC.KeepaliveMinTime)
}

// TestGetConfigGRPCOverrides ensures operator-provided values override the
// in-code defaults.
func TestGetConfigGRPCOverrides(t *testing.T) {
v := seedViperWithDefaultConfig(t)
v.Set("grpc.max-recv-msg-size", 8*1024*1024)
v.Set("grpc.max-open-connections", 50)
v.Set("grpc.max-connection-idle", "5m")
v.Set("grpc.max-connection-age", "30m")
v.Set("grpc.max-connection-age-grace", "1m")
v.Set("grpc.keepalive-time", "1m")
v.Set("grpc.keepalive-timeout", "10s")
v.Set("grpc.keepalive-min-time", "30s")
v.Set("grpc.keepalive-permit-without-stream", true)

cfg, err := GetConfig(v)
require.NoError(t, err)
require.Equal(t, 8*1024*1024, cfg.GRPC.MaxRecvMsgSize)
require.Equal(t, uint(50), cfg.GRPC.MaxOpenConnections)
require.Equal(t, 5*time.Minute, cfg.GRPC.MaxConnectionIdle)
require.Equal(t, 30*time.Minute, cfg.GRPC.MaxConnectionAge)
require.Equal(t, time.Minute, cfg.GRPC.MaxConnectionAgeGrace)
require.Equal(t, time.Minute, cfg.GRPC.KeepaliveTime)
require.Equal(t, 10*time.Second, cfg.GRPC.KeepaliveTimeout)
require.Equal(t, 30*time.Second, cfg.GRPC.KeepaliveMinTime)
require.True(t, cfg.GRPC.KeepalivePermitWithoutStream)
}

func TestDefaultGRPCWebConfig(t *testing.T) {
Expand Down
28 changes: 28 additions & 0 deletions sei-cosmos/server/config/toml.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,34 @@ enable = {{ .GRPC.Enable }}
# Address defines the gRPC server address to bind to.
address = "{{ .GRPC.Address }}"

# MaxRecvMsgSize defines the maximum message size in bytes the server can receive.
# It bounds per-request memory allocation before the rate limiter fires. Default 4 MB.
max-recv-msg-size = {{ .GRPC.MaxRecvMsgSize }}

# MaxOpenConnections defines the maximum number of simultaneous open connections. 0 means unlimited.
max-open-connections = {{ .GRPC.MaxOpenConnections }}

# MaxConnectionIdle is the duration after which an idle connection is closed (e.g. "5m"). 0 means infinity.
max-connection-idle = "{{ .GRPC.MaxConnectionIdle }}"

# MaxConnectionAge is the maximum duration a connection may exist before it is closed (e.g. "30m"). 0 means infinity.
max-connection-age = "{{ .GRPC.MaxConnectionAge }}"

# MaxConnectionAgeGrace is an additive period after max-connection-age during which the connection is forcibly closed. 0 means infinity.
max-connection-age-grace = "{{ .GRPC.MaxConnectionAgeGrace }}"

# KeepaliveTime is the interval after which, with no activity, the server pings the client to check liveness.
keepalive-time = "{{ .GRPC.KeepaliveTime }}"

# KeepaliveTimeout is the duration the server waits for a keepalive ping ack before closing the connection.
keepalive-timeout = "{{ .GRPC.KeepaliveTimeout }}"

# KeepaliveMinTime is the minimum interval a client must wait between keepalive pings; more frequent pings are penalized.
keepalive-min-time = "{{ .GRPC.KeepaliveMinTime }}"

# KeepalivePermitWithoutStream defines whether the server allows keepalive pings even when there are no active streams.
keepalive-permit-without-stream = {{ .GRPC.KeepalivePermitWithoutStream }}

###############################################################################
### gRPC Web Configuration (Auto-managed) ###
###############################################################################
Expand Down
40 changes: 36 additions & 4 deletions sei-cosmos/server/grpc/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,46 @@ package grpc

import (
"fmt"
"math"
"net"
"time"

"golang.org/x/net/netutil"
"google.golang.org/grpc"
"google.golang.org/grpc/keepalive"

"github.com/sei-protocol/sei-chain/sei-cosmos/client"
"github.com/sei-protocol/sei-chain/sei-cosmos/server/config"
"github.com/sei-protocol/sei-chain/sei-cosmos/server/grpc/gogoreflection"
reflection "github.com/sei-protocol/sei-chain/sei-cosmos/server/grpc/reflection/v2alpha1"
"github.com/sei-protocol/sei-chain/sei-cosmos/server/types"
sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
)

// StartGRPCServer starts a gRPC server on the given address.
func StartGRPCServer(clientCtx client.Context, app types.Application, address string) (*grpc.Server, error) {
grpcSrv := grpc.NewServer(grpc.MaxConcurrentStreams(100))
// StartGRPCServer starts a gRPC server on the address given by cfg.
func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config.GRPCConfig) (*grpc.Server, error) {
maxRecvMsgSize := cfg.MaxRecvMsgSize
if maxRecvMsgSize <= 0 {
maxRecvMsgSize = config.DefaultGRPCMaxRecvMsgSize
}

grpcSrv := grpc.NewServer(
grpc.MaxConcurrentStreams(100),
// MaxRecvMsgSize bounds per-request memory allocation before the rate
// limiter fires, preventing an oversized request from exhausting memory.
grpc.MaxRecvMsgSize(maxRecvMsgSize),
grpc.KeepaliveParams(keepalive.ServerParameters{
MaxConnectionIdle: cfg.MaxConnectionIdle,
MaxConnectionAge: cfg.MaxConnectionAge,
MaxConnectionAgeGrace: cfg.MaxConnectionAgeGrace,
Time: cfg.KeepaliveTime,
Timeout: cfg.KeepaliveTimeout,
}),
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
MinTime: cfg.KeepaliveMinTime,
PermitWithoutStream: cfg.KeepalivePermitWithoutStream,
}),
)
app.RegisterGRPCServer(grpcSrv)
// reflection allows consumers to build dynamic clients that can write
// to any cosmos-sdk application without relying on application packages at compile time
Expand All @@ -38,10 +63,17 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, address st
// Reflection allows external clients to see what services and methods
// the gRPC server exposes.
gogoreflection.Register(grpcSrv)
listener, err := net.Listen("tcp", address)
listener, err := net.Listen("tcp", cfg.Address)
if err != nil {
return nil, err
}
if cfg.MaxOpenConnections > 0 {
maxConn := cfg.MaxOpenConnections
if maxConn > math.MaxInt {
maxConn = math.MaxInt
}
listener = netutil.LimitListener(listener, int(maxConn)) //nolint:gosec // G115: clamped to math.MaxInt above
}

errCh := make(chan error)
go func() {
Expand Down
2 changes: 1 addition & 1 deletion sei-cosmos/server/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,7 @@ func startInProcess(
}

if config.GRPC.Enable {
grpcSrv, err := servergrpc.StartGRPCServer(clientCtx, app, config.GRPC.Address)
grpcSrv, err := servergrpc.StartGRPCServer(clientCtx, app, config.GRPC)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion sei-cosmos/testutil/network/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func startInProcess(cfg Config, val *Validator) error {
}

if val.AppConfig.GRPC.Enable {
grpcSrv, err := servergrpc.StartGRPCServer(val.ClientCtx, app, val.AppConfig.GRPC.Address)
grpcSrv, err := servergrpc.StartGRPCServer(val.ClientCtx, app, val.AppConfig.GRPC)
if err != nil {
return err
}
Expand Down
Loading