From 64d0c4876fd2e4fc9b2379322a52b87fb303ae27 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 17 Jun 2026 09:28:17 -0700 Subject: [PATCH 1/9] Added read, write and idle timeout params to grpc web server --- sei-cosmos/server/grpc/grpc_web.go | 3 ++ .../server/grpc/grpc_web_timeout_test.go | 44 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 sei-cosmos/server/grpc/grpc_web_timeout_test.go diff --git a/sei-cosmos/server/grpc/grpc_web.go b/sei-cosmos/server/grpc/grpc_web.go index ab951bb528..cf66a98a71 100644 --- a/sei-cosmos/server/grpc/grpc_web.go +++ b/sei-cosmos/server/grpc/grpc_web.go @@ -28,6 +28,9 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err Addr: config.GRPCWeb.Address, Handler: wrappedServer, ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, } errCh := make(chan error) diff --git a/sei-cosmos/server/grpc/grpc_web_timeout_test.go b/sei-cosmos/server/grpc/grpc_web_timeout_test.go new file mode 100644 index 0000000000..7e3da4e1e2 --- /dev/null +++ b/sei-cosmos/server/grpc/grpc_web_timeout_test.go @@ -0,0 +1,44 @@ +package grpc_test + +import ( + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" + srvgrpc "github.com/sei-protocol/sei-chain/sei-cosmos/server/grpc" +) + +// TestStartGRPCWebTimeouts verifies that StartGRPCWeb sets all HTTP timeout +// fields needed to prevent body-stall DoS attacks. +func TestStartGRPCWebTimeouts(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := ln.Addr().String() + err = ln.Close() + require.NoError(t, err) + + grpcSrv := grpc.NewServer() + cfg := config.Config{ + GRPCWeb: config.GRPCWebConfig{ + Enable: true, + Address: addr, + }, + } + + srv, err := srvgrpc.StartGRPCWeb(grpcSrv, cfg) + require.NoError(t, err) + require.NotNil(t, srv) + defer func() { + err = srv.Shutdown(t.Context()) //nolint:errcheck + require.NoError(t, err) + }() + + require.Equal(t, 10*time.Second, srv.ReadHeaderTimeout, "ReadHeaderTimeout") + require.Equal(t, 30*time.Second, srv.ReadTimeout, "ReadTimeout") + require.Equal(t, 30*time.Second, srv.WriteTimeout, "WriteTimeout") + require.Equal(t, 120*time.Second, srv.IdleTimeout, "IdleTimeout") +} From 512881ded739ed0f9d0d42e47e0ba0d9a6a66dbe Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 17 Jun 2026 10:06:41 -0700 Subject: [PATCH 2/9] Increased write timeout to eliminate risk of slow queries getting truncated --- sei-cosmos/server/grpc/grpc_web.go | 2 +- sei-cosmos/server/grpc/grpc_web_timeout_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sei-cosmos/server/grpc/grpc_web.go b/sei-cosmos/server/grpc/grpc_web.go index cf66a98a71..6905582220 100644 --- a/sei-cosmos/server/grpc/grpc_web.go +++ b/sei-cosmos/server/grpc/grpc_web.go @@ -29,7 +29,7 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err Handler: wrappedServer, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, + WriteTimeout: 2 * time.Minute, IdleTimeout: 120 * time.Second, } diff --git a/sei-cosmos/server/grpc/grpc_web_timeout_test.go b/sei-cosmos/server/grpc/grpc_web_timeout_test.go index 7e3da4e1e2..bdce120cfc 100644 --- a/sei-cosmos/server/grpc/grpc_web_timeout_test.go +++ b/sei-cosmos/server/grpc/grpc_web_timeout_test.go @@ -39,6 +39,6 @@ func TestStartGRPCWebTimeouts(t *testing.T) { require.Equal(t, 10*time.Second, srv.ReadHeaderTimeout, "ReadHeaderTimeout") require.Equal(t, 30*time.Second, srv.ReadTimeout, "ReadTimeout") - require.Equal(t, 30*time.Second, srv.WriteTimeout, "WriteTimeout") + require.Equal(t, 2*time.Minute, srv.WriteTimeout, "WriteTimeout") require.Equal(t, 120*time.Second, srv.IdleTimeout, "IdleTimeout") } From be614e474646e8d168394e2b3b366e341f6fa7d3 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 17 Jun 2026 16:18:29 -0700 Subject: [PATCH 3/9] Reduced idle timeout --- sei-cosmos/server/grpc/grpc_web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-cosmos/server/grpc/grpc_web.go b/sei-cosmos/server/grpc/grpc_web.go index 6905582220..541c56f6b4 100644 --- a/sei-cosmos/server/grpc/grpc_web.go +++ b/sei-cosmos/server/grpc/grpc_web.go @@ -30,7 +30,7 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 2 * time.Minute, - IdleTimeout: 120 * time.Second, + IdleTimeout: 30 * time.Second, } errCh := make(chan error) From cdb10fa29145deaeddb7fc7f0c41827fc75b3444 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 17 Jun 2026 16:31:37 -0700 Subject: [PATCH 4/9] Added max open connections limit --- sei-cosmos/server/config/config.go | 15 ++++++++++----- sei-cosmos/server/grpc/grpc_web.go | 17 +++++++++++++---- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index d7c5e740ea..04bea29128 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -179,6 +179,9 @@ type GRPCWebConfig struct { // EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk) EnableUnsafeCORS bool `mapstructure:"enable-unsafe-cors"` + + // MaxOpenConnections defines the maximum number of simultaneous open connections. 0 means unlimited. + MaxOpenConnections uint `mapstructure:"max-open-connections"` } // StateSyncConfig defines the state sync snapshot configuration. @@ -291,8 +294,9 @@ func DefaultConfig() *Config { Offline: false, }, GRPCWeb: GRPCWebConfig{ - Enable: true, - Address: DefaultGRPCWebAddress, + Enable: true, + Address: DefaultGRPCWebAddress, + MaxOpenConnections: 1000, }, StateSync: StateSyncConfig{ SnapshotInterval: 0, @@ -386,9 +390,10 @@ func GetConfig(v *viper.Viper) (Config, error) { Address: v.GetString("grpc.address"), }, GRPCWeb: GRPCWebConfig{ - Enable: v.GetBool("grpc-web.enable"), - Address: v.GetString("grpc-web.address"), - EnableUnsafeCORS: v.GetBool("grpc-web.enable-unsafe-cors"), + Enable: v.GetBool("grpc-web.enable"), + Address: v.GetString("grpc-web.address"), + EnableUnsafeCORS: v.GetBool("grpc-web.enable-unsafe-cors"), + MaxOpenConnections: v.GetUint("grpc-web.max-open-connections"), }, StateSync: StateSyncConfig{ SnapshotInterval: v.GetUint64("state-sync.snapshot-interval"), diff --git a/sei-cosmos/server/grpc/grpc_web.go b/sei-cosmos/server/grpc/grpc_web.go index 541c56f6b4..c959a16075 100644 --- a/sei-cosmos/server/grpc/grpc_web.go +++ b/sei-cosmos/server/grpc/grpc_web.go @@ -2,10 +2,12 @@ package grpc import ( "fmt" + "net" "net/http" "time" "github.com/improbable-eng/grpc-web/go/grpcweb" + "golang.org/x/net/netutil" "google.golang.org/grpc" "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" @@ -25,7 +27,6 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err wrappedServer := grpcweb.WrapServer(grpcSrv, options...) grpcWebSrv := &http.Server{ - Addr: config.GRPCWeb.Address, Handler: wrappedServer, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, @@ -33,15 +34,23 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err IdleTimeout: 30 * time.Second, } + listener, err := net.Listen("tcp", config.GRPCWeb.Address) + if err != nil { + return nil, fmt.Errorf("[grpc-web] failed to listen on %s: %w", config.GRPCWeb.Address, err) + } + if config.GRPCWeb.MaxOpenConnections > 0 { + listener = netutil.LimitListener(listener, int(config.GRPCWeb.MaxOpenConnections)) + } + errCh := make(chan error) go func() { - if err := grpcWebSrv.ListenAndServe(); err != nil { - errCh <- fmt.Errorf("[grpc] failed to serve: %w", err) + if err = grpcWebSrv.Serve(listener); err != nil { + errCh <- fmt.Errorf("[grpc-web] failed to serve: %w", err) } }() select { - case err := <-errCh: + case err = <-errCh: return nil, err case <-time.After(types.ServerStartTime): // assume server started successfully return grpcWebSrv, nil From 63fcd89040e8ade1c2ed97faa953bbdb44df0a74 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 17 Jun 2026 16:34:44 -0700 Subject: [PATCH 5/9] Fixed test --- sei-cosmos/server/grpc/grpc_web_timeout_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-cosmos/server/grpc/grpc_web_timeout_test.go b/sei-cosmos/server/grpc/grpc_web_timeout_test.go index bdce120cfc..7b2e3437a0 100644 --- a/sei-cosmos/server/grpc/grpc_web_timeout_test.go +++ b/sei-cosmos/server/grpc/grpc_web_timeout_test.go @@ -40,5 +40,5 @@ func TestStartGRPCWebTimeouts(t *testing.T) { require.Equal(t, 10*time.Second, srv.ReadHeaderTimeout, "ReadHeaderTimeout") require.Equal(t, 30*time.Second, srv.ReadTimeout, "ReadTimeout") require.Equal(t, 2*time.Minute, srv.WriteTimeout, "WriteTimeout") - require.Equal(t, 120*time.Second, srv.IdleTimeout, "IdleTimeout") + require.Equal(t, 30*time.Second, srv.IdleTimeout, "IdleTimeout") } From 5aa5c1f00b3f8cba2c415c938e62d526b806d92d Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 17 Jun 2026 16:52:09 -0700 Subject: [PATCH 6/9] Added config for connections, used buffered channel for go routine --- sei-cosmos/server/config/toml.go | 3 +++ sei-cosmos/server/grpc/grpc_web.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sei-cosmos/server/config/toml.go b/sei-cosmos/server/config/toml.go index ac931084be..77fa762542 100644 --- a/sei-cosmos/server/config/toml.go +++ b/sei-cosmos/server/config/toml.go @@ -205,6 +205,9 @@ address = "{{ .GRPCWeb.Address }}" # EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk). enable-unsafe-cors = {{ .GRPCWeb.EnableUnsafeCORS }} +# MaxOpenConnections defines the number of maximum open connections. 0 means unlimited. +max-open-connections = {{ .GRPCWeb.MaxOpenConnections }} + ############################################################################### ### Genesis Configuration (Auto-managed) ### ############################################################################### diff --git a/sei-cosmos/server/grpc/grpc_web.go b/sei-cosmos/server/grpc/grpc_web.go index c959a16075..1977e33c50 100644 --- a/sei-cosmos/server/grpc/grpc_web.go +++ b/sei-cosmos/server/grpc/grpc_web.go @@ -42,9 +42,9 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err listener = netutil.LimitListener(listener, int(config.GRPCWeb.MaxOpenConnections)) } - errCh := make(chan error) + errCh := make(chan error, 1) go func() { - if err = grpcWebSrv.Serve(listener); err != nil { + if err := grpcWebSrv.Serve(listener); err != nil { errCh <- fmt.Errorf("[grpc-web] failed to serve: %w", err) } }() From f77e2ff0e9a9198a4dc0c1f3afc90141c67fe7d5 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 17 Jun 2026 19:01:16 -0700 Subject: [PATCH 7/9] Add max int overflow check --- sei-cosmos/server/grpc/grpc_web.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sei-cosmos/server/grpc/grpc_web.go b/sei-cosmos/server/grpc/grpc_web.go index 1977e33c50..ae6e014367 100644 --- a/sei-cosmos/server/grpc/grpc_web.go +++ b/sei-cosmos/server/grpc/grpc_web.go @@ -2,6 +2,7 @@ package grpc import ( "fmt" + "math" "net" "net/http" "time" @@ -39,7 +40,11 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err return nil, fmt.Errorf("[grpc-web] failed to listen on %s: %w", config.GRPCWeb.Address, err) } if config.GRPCWeb.MaxOpenConnections > 0 { - listener = netutil.LimitListener(listener, int(config.GRPCWeb.MaxOpenConnections)) + maxConn := config.GRPCWeb.MaxOpenConnections + if maxConn > math.MaxInt { + maxConn = math.MaxInt + } + listener = netutil.LimitListener(listener, int(maxConn)) } errCh := make(chan error, 1) From ee9fa0fbecd412d258f25327440c626c7d70dfce Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Thu, 18 Jun 2026 13:28:02 -0700 Subject: [PATCH 8/9] Fixed lint error --- sei-cosmos/server/grpc/grpc_web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sei-cosmos/server/grpc/grpc_web.go b/sei-cosmos/server/grpc/grpc_web.go index ae6e014367..e80e042751 100644 --- a/sei-cosmos/server/grpc/grpc_web.go +++ b/sei-cosmos/server/grpc/grpc_web.go @@ -44,7 +44,7 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err if maxConn > math.MaxInt { maxConn = math.MaxInt } - listener = netutil.LimitListener(listener, int(maxConn)) + listener = netutil.LimitListener(listener, int(maxConn)) //nolint:gosec // G115: clamped to math.MaxInt above } errCh := make(chan error, 1) From e680066ab25b874b3b1fc25d8d8e6a5dfb4cb128 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 23 Jun 2026 14:16:46 -0700 Subject: [PATCH 9/9] added default max open connections --- sei-cosmos/server/config/config.go | 16 +++++++++-- sei-cosmos/server/config/config_test.go | 35 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index 04bea29128..f3719bfcb8 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -25,6 +25,10 @@ const ( // DefaultGRPCWebAddress defines the default address to bind the gRPC-web server to. DefaultGRPCWebAddress = "0.0.0.0:9091" + // DefaultGRPCWebMaxOpenConnections defines the default maximum number of + // simultaneous open connections for the gRPC-web server. + DefaultGRPCWebMaxOpenConnections = 1000 + // DefaultOccEanbled defines whether to use OCC for tx processing DefaultOccEnabled = true ) @@ -296,7 +300,7 @@ func DefaultConfig() *Config { GRPCWeb: GRPCWebConfig{ Enable: true, Address: DefaultGRPCWebAddress, - MaxOpenConnections: 1000, + MaxOpenConnections: DefaultGRPCWebMaxOpenConnections, }, StateSync: StateSyncConfig{ SnapshotInterval: 0, @@ -343,6 +347,14 @@ func GetConfig(v *viper.Viper) (Config, error) { scWriteMode = parsed } + // Apply the in-code default when the key is absent so that nodes upgrading + // with an older app.toml (which lacks this key) are still bounded rather + // than running with unlimited connections. + grpcWebMaxOpenConnections := uint(DefaultGRPCWebMaxOpenConnections) + if v.IsSet("grpc-web.max-open-connections") { + grpcWebMaxOpenConnections = v.GetUint("grpc-web.max-open-connections") + } + return Config{ BaseConfig: BaseConfig{ MinGasPrices: v.GetString("minimum-gas-prices"), @@ -393,7 +405,7 @@ func GetConfig(v *viper.Viper) (Config, error) { Enable: v.GetBool("grpc-web.enable"), Address: v.GetString("grpc-web.address"), EnableUnsafeCORS: v.GetBool("grpc-web.enable-unsafe-cors"), - MaxOpenConnections: v.GetUint("grpc-web.max-open-connections"), + MaxOpenConnections: grpcWebMaxOpenConnections, }, StateSync: StateSyncConfig{ SnapshotInterval: v.GetUint64("state-sync.snapshot-interval"), diff --git a/sei-cosmos/server/config/config_test.go b/sei-cosmos/server/config/config_test.go index dddb3c3e2c..29a0c3c08d 100644 --- a/sei-cosmos/server/config/config_test.go +++ b/sei-cosmos/server/config/config_test.go @@ -73,6 +73,7 @@ func TestDefaultGRPCWebConfig(t *testing.T) { cfg := DefaultConfig() require.True(t, cfg.GRPCWeb.Enable) require.Equal(t, DefaultGRPCWebAddress, cfg.GRPCWeb.Address) + require.Equal(t, uint(DefaultGRPCWebMaxOpenConnections), cfg.GRPCWeb.MaxOpenConnections) } func TestDefaultRosettaConfig(t *testing.T) { @@ -280,6 +281,40 @@ func TestSetAndGetMinGasPrices(t *testing.T) { require.Equal(t, "uatom", parsed[1].Denom) } +func TestGetConfigGRPCWebMaxOpenConnections(t *testing.T) { + baseViper := func() *viper.Viper { + v := viper.New() + v.Set("minimum-gas-prices", DefaultMinGasPrices) + v.Set("telemetry.global-labels", []interface{}{}) + return v + } + + t.Run("missing key falls back to the in-code default", func(t *testing.T) { + // Mirrors a node upgrading with an older app.toml that predates the + // grpc-web.max-open-connections key + cfg, err := GetConfig(baseViper()) + require.NoError(t, err) + require.Equal(t, uint(DefaultGRPCWebMaxOpenConnections), cfg.GRPCWeb.MaxOpenConnections) + }) + + t.Run("explicit zero is preserved as unlimited", func(t *testing.T) { + v := baseViper() + v.Set("grpc-web.max-open-connections", 0) + cfg, err := GetConfig(v) + require.NoError(t, err) + require.Equal(t, uint(0), cfg.GRPCWeb.MaxOpenConnections, + "explicit 0 must remain an opt-in to unlimited connections") + }) + + t.Run("explicit value overrides the default", func(t *testing.T) { + v := baseViper() + v.Set("grpc-web.max-open-connections", 250) + cfg, err := GetConfig(v) + require.NoError(t, err) + require.Equal(t, uint(250), cfg.GRPCWeb.MaxOpenConnections) + }) +} + func TestGetConfigStateCommit(t *testing.T) { v := viper.New()