From 98f528579ac4233f87b9d354e5350677fd214086 Mon Sep 17 00:00:00 2001 From: Tamara Boehm Date: Mon, 1 Jun 2026 12:01:59 +0200 Subject: [PATCH 1/4] feat: Support runtime control of connection rate limiting via socket Co-authored-by: Alexander Nicke Co-authored-by: Dariquest Co-authored-by: M Rizwan Shaik --- acceptance-tests/bosh_helpers.go | 10 + acceptance-tests/rate_limit_test.go | 245 +++++++++++++++++- docs/rate_limiting.md | 54 +++- jobs/haproxy/templates/haproxy.config.erb | 27 +- .../haproxy_config/rate_limit_spec.rb | 65 ++++- 5 files changed, 381 insertions(+), 20 deletions(-) diff --git a/acceptance-tests/bosh_helpers.go b/acceptance-tests/bosh_helpers.go index 8197b47c..415ee0e4 100644 --- a/acceptance-tests/bosh_helpers.go +++ b/acceptance-tests/bosh_helpers.go @@ -322,3 +322,13 @@ func crashHAProxy(haproxyInfo haproxyInfo) { _, _, err := runOnRemote(haproxyInfo.SSHUser, haproxyInfo.PublicIP, haproxyInfo.SSHPrivateKey, "sudo pkill -9 -x haproxy") Expect(err).NotTo(HaveOccurred()) } + +// runHAProxySocketCommand sends a command to the HAProxy Runtime API via the stats socket using socat. +// Returns the trimmed stdout output. +func runHAProxySocketCommand(haproxyInfo haproxyInfo, command string) string { + cmd := fmt.Sprintf(`echo "%s" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock`, command) + stdout, _, err := runOnRemote(haproxyInfo.SSHUser, haproxyInfo.PublicIP, haproxyInfo.SSHPrivateKey, cmd) + Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(stdout) +} + diff --git a/acceptance-tests/rate_limit_test.go b/acceptance-tests/rate_limit_test.go index 3415d967..f123db6f 100644 --- a/acceptance-tests/rate_limit_test.go +++ b/acceptance-tests/rate_limit_test.go @@ -280,9 +280,250 @@ var _ = Describe("Rate-Limiting", func() { Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusTooManyRequests)) } else { - // TCP connection limit reached --> no response - Expect(err).To(HaveOccurred()) + // TCP connection limit reached --> no response + Expect(err).To(HaveOccurred()) } } }) + + It("Connection Based Limiting works via manifest and can be overridden at runtime via socket", func() { + connLimit := 5 + opsfileConnectionsRateLimit := fmt.Sprintf(`--- +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections + value: %d +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size? + value: 100s +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size? + value: 100 +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/block? + value: true +`, connLimit) + haproxyBackendPort := 12000 + haproxyInfo, _ := deployHAProxy(baseManifestVars{ + haproxyBackendPort: haproxyBackendPort, + haproxyBackendServers: []string{"127.0.0.1"}, + deploymentName: deploymentNameForTestNode(), + }, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true) + + closeLocalServer, localPort := startDefaultTestServer() + defer closeLocalServer() + + closeTunnel := setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort) + defer closeTunnel() + + By("Verifying proc.conn_rate_limit is initialised from manifest value") + output := runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_limit") + Expect(output).To(ContainSubstring(fmt.Sprintf("%d", connLimit))) + + By("Verifying proc.conn_rate_block is initialised as true from manifest block: true") + output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") + Expect(output).To(ContainSubstring("1")) + + By("Verifying connections are blocked after exceeding the manifest-configured limit") + testRequestCount := int(float64(connLimit) * 1.5) + firstFailure := -1 + successfulRequestCount := 0 + for i := 0; i < testRequestCount; i++ { + rt := &http.Transport{DisableKeepAlives: true} + client := &http.Client{Transport: rt} + resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP)) + if err == nil && resp.StatusCode == 200 { + resp.Body.Close() + successfulRequestCount++ + continue + } + if err == nil { + resp.Body.Close() + } + if firstFailure == -1 { + firstFailure = i + } + } + Expect(firstFailure).To(Equal(connLimit)) + Expect(successfulRequestCount).To(Equal(connLimit)) + + By("Clearing stick table before overriding limit") + runHAProxySocketCommand(haproxyInfo, "clear table st_tcp_conn_rate") + + By("Overriding the limit at runtime via socket to a higher value") + newLimit := connLimit * 3 + runHAProxySocketCommand(haproxyInfo, fmt.Sprintf("experimental-mode on; set var proc.conn_rate_limit int(%d)", newLimit)) + + By("Verifying the override is reflected via get var") + output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_limit") + Expect(output).To(ContainSubstring(fmt.Sprintf("%d", newLimit))) + + By("Verifying connections are allowed up to the new higher socket-configured limit") + testRequestCount = int(float64(newLimit) * 1.5) + firstFailure = -1 + successfulRequestCount = 0 + for i := 0; i < testRequestCount; i++ { + rt := &http.Transport{DisableKeepAlives: true} + client := &http.Client{Transport: rt} + resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP)) + if err == nil && resp.StatusCode == 200 { + resp.Body.Close() + successfulRequestCount++ + continue + } + if err == nil { + resp.Body.Close() + } + if firstFailure == -1 { + firstFailure = i + } + } + Expect(firstFailure).To(Equal(newLimit)) + Expect(successfulRequestCount).To(Equal(newLimit)) + }) + + It("Connection Based Limiting can be enabled and disabled at runtime via socket with manifest block false", func() { + connLimit := 5 + // block: false in manifest — blocking is enabled/disabled at runtime via socket + opsfileConnectionsRateLimit := fmt.Sprintf(`--- +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections + value: %d +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size? + value: 100s +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size? + value: 100 +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/block? + value: false +`, connLimit) + haproxyBackendPort := 12000 + haproxyInfo, _ := deployHAProxy(baseManifestVars{ + haproxyBackendPort: haproxyBackendPort, + haproxyBackendServers: []string{"127.0.0.1"}, + deploymentName: deploymentNameForTestNode(), + }, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true) + + closeLocalServer, localPort := startDefaultTestServer() + defer closeLocalServer() + + closeTunnel := setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort) + defer closeTunnel() + + By("Verifying proc.conn_rate_block is initialised as false from manifest block: false") + output := runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") + Expect(output).To(ContainSubstring("0")) + + By("Enabling blocking at runtime via socket") + runHAProxySocketCommand(haproxyInfo, "experimental-mode on; set var proc.conn_rate_block bool(true)") + + By("Verifying proc.conn_rate_block is now true") + output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") + Expect(output).To(ContainSubstring("1")) + + By("Verifying connections are blocked after exceeding the limit") + testRequestCount := int(float64(connLimit) * 1.5) + firstFailure := -1 + successfulRequestCount := 0 + for i := 0; i < testRequestCount; i++ { + rt := &http.Transport{DisableKeepAlives: true} + client := &http.Client{Transport: rt} + resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP)) + if err == nil && resp.StatusCode == 200 { + resp.Body.Close() + successfulRequestCount++ + continue + } + if err == nil { + resp.Body.Close() + } + if firstFailure == -1 { + firstFailure = i + } + } + Expect(firstFailure).To(Equal(connLimit)) + Expect(successfulRequestCount).To(Equal(connLimit)) + + By("Disabling blocking at runtime via socket") + runHAProxySocketCommand(haproxyInfo, "experimental-mode on; set var proc.conn_rate_block bool(false)") + + By("Verifying proc.conn_rate_block is now false") + output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") + Expect(output).To(ContainSubstring("0")) + + By("Clearing stick table to reset counters") + runHAProxySocketCommand(haproxyInfo, "clear table st_tcp_conn_rate") + + By("Verifying all connections are now allowed after disabling blocking via socket") + for i := 0; i < testRequestCount; i++ { + rt := &http.Transport{DisableKeepAlives: true} + client := &http.Client{Transport: rt} + resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP)) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + resp.Body.Close() + } + }) + + It("Connection Based Limiting works when limit is set entirely via socket without manifest connections property", func() { + connLimit := 5 + // Only table_size and window_size are set — no connections or block in manifest + // proc.conn_rate_block defaults to false, proc.conn_rate_limit must be set via socket + opsfileConnectionsRateLimit := `--- +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/window_size + value: 100s +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size? + value: 100 +` + haproxyBackendPort := 12000 + haproxyInfo, _ := deployHAProxy(baseManifestVars{ + haproxyBackendPort: haproxyBackendPort, + haproxyBackendServers: []string{"127.0.0.1"}, + deploymentName: deploymentNameForTestNode(), + }, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true) + + closeLocalServer, localPort := startDefaultTestServer() + defer closeLocalServer() + + closeTunnel := setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort) + defer closeTunnel() + + By("Setting conn_rate_limit and enabling blocking entirely via socket") + runHAProxySocketCommand(haproxyInfo, fmt.Sprintf("experimental-mode on; set var proc.conn_rate_limit int(%d)", connLimit)) + runHAProxySocketCommand(haproxyInfo, "experimental-mode on; set var proc.conn_rate_block bool(true)") + + By("Verifying proc.conn_rate_limit is set correctly via socket") + output := runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_limit") + Expect(output).To(ContainSubstring(fmt.Sprintf("%d", connLimit))) + + By("Verifying proc.conn_rate_block is set correctly via socket") + output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") + Expect(output).To(ContainSubstring("1")) + + By("Verifying connections are blocked after exceeding the socket-configured limit") + testRequestCount := int(float64(connLimit) * 1.5) + firstFailure := -1 + successfulRequestCount := 0 + for i := 0; i < testRequestCount; i++ { + rt := &http.Transport{DisableKeepAlives: true} + client := &http.Client{Transport: rt} + resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP)) + if err == nil && resp.StatusCode == 200 { + resp.Body.Close() + successfulRequestCount++ + continue + } + if err == nil { + resp.Body.Close() + } + if firstFailure == -1 { + firstFailure = i + } + } + Expect(firstFailure).To(Equal(connLimit)) + Expect(successfulRequestCount).To(Equal(connLimit)) + }) }) diff --git a/docs/rate_limiting.md b/docs/rate_limiting.md index b6e63ea6..862f6606 100644 --- a/docs/rate_limiting.md +++ b/docs/rate_limiting.md @@ -115,7 +115,59 @@ To get more insight into what is going on inside HAProxy regarding its rate limi ```bash $ echo "show table st_http_req_rate" | socat /var/vcap/sys/run/haproxy/stats.sock - # table: st_http_req_rate, type: ip, size:10485760, used:1 -0x56495f3dc3d0: key=172.18.0.1 use=0 exp=7618 http_req_rate(10000)=10 +0x...: key=:ffff:172.18.0.1 use=0 exp=7618 http_req_rate(10000)=10 + +echo "show table st_tcp_conn_rate" | socat stdio /var/vcap/sys/run/haproxy/stats.sock +# => # table: st_tcp_conn_rate, type: ipv6, size:1048576, used:2 +# => 0x...: key=::ffff:203.0.113.42 use=0 exp=8123 shard=0 conn_rate(10000)=5 +``` + +To find the IP with the highest connection rate, use: + +```bash +echo "show table st_tcp_conn_rate" | socat stdio /var/vcap/sys/run/haproxy/stats.sock | sort -t= -k2 -rn | head -1 ``` > Note: You will likely need `sudo` permission to run socat. + +## Control Connection Rate Limiting via HAProxy Runtime API + +The `connections_rate_limit.block` flag and `connections_rate_limit.connections` threshold are stored as HAProxy process-level variables (`proc.conn_rate_block` and `proc.conn_rate_limit`) and can be changed at runtime without a reload. This requires `ha_proxy.master_cli_enable: true` or `ha_proxy.stats_enable: true`. + +The socket is located at `/var/vcap/sys/run/haproxy/stats.sock`. You will likely need `sudo` to access it. + +> **Note:** The `tcp-request connection reject` rule is always present in the config as long as `connections_rate_limit.table_size` and `connections_rate_limit.window_size` are set. Enforcement is controlled entirely at runtime via `proc.conn_rate_block` and `proc.conn_rate_limit`. Setting `connections_rate_limit.connections` and `connections_rate_limit.block` in the manifest only sets their **initial values** at startup — they can be freely overridden via socket without a reload. + +### Inspect Current Variable Values + +```bash +echo "get var proc.conn_rate_limit" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +# => proc.conn_rate_limit: type=sint value=<100> + +echo "get var proc.conn_rate_block" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +# => proc.conn_rate_block: type=bool value=<1> +``` + +### Enable or Disable Blocking at Runtime + +```bash +# Enable blocking (equivalent to setting block: true in the manifest) +echo "experimental-mode on; set var proc.conn_rate_block bool(true)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock + +# Disable blocking without reloading (equivalent to setting block: false in the manifest) +echo "experimental-mode on; set var proc.conn_rate_block bool(false)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +``` + +### Adjust the Connections Threshold at Runtime + +```bash +# Allow up to 100 connections per window (equivalent to setting connections: 100 in the manifest) +echo "experimental-mode on; set var proc.conn_rate_limit int(100)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +``` + +### Enable Rate Limiting and Set Threshold in One Step + +```bash +echo "experimental-mode on; set var proc.conn_rate_limit int(100); set var proc.conn_rate_block bool(true)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +``` + diff --git a/jobs/haproxy/templates/haproxy.config.erb b/jobs/haproxy/templates/haproxy.config.erb index 16ef574c..be82b8c1 100644 --- a/jobs/haproxy/templates/haproxy.config.erb +++ b/jobs/haproxy/templates/haproxy.config.erb @@ -230,6 +230,15 @@ end abort "Conflicting configuration: enable_redispatch works only with retries > 0" end + # Safety guard: block=true without connections would cause every client with >= 1 connection to be blocked (total lockout) + if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do + if p("ha_proxy.connections_rate_limit.block", false) + if !p("ha_proxy.connections_rate_limit.connections", nil) + abort "Conflicting configuration: connections_rate_limit.connections must be set when connections_rate_limit.block is true (otherwise every client with >= 1 connection would be blocked)" + end + end + end + backend_servers = [] backend_servers_local = [] backend_port = nil @@ -324,6 +333,12 @@ global <%- if backend_match_http_protocol && backends.length == 2 -%> set-var proc.h2_alpn_tag str(h2) <%- end -%> + <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%> + <%- if_p("ha_proxy.connections_rate_limit.connections") do |conn_rate_connections| -%> + set-var proc.conn_rate_limit int(<%= conn_rate_connections %>) + <%- end -%> + set-var proc.conn_rate_block bool(<%= p("ha_proxy.connections_rate_limit.block", false) %>) + <%- end -%> <%- if p("ha_proxy.always_allow_body_http10") %> h1-accept-payload-with-any-method <%- end %> @@ -437,11 +452,7 @@ frontend http-in tcp-request <%= tcp_request_phase %> reject if layer4_block <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%> tcp-request <%= tcp_request_phase %> track-sc0 src table st_tcp_conn_rate - <%- if_p("ha_proxy.connections_rate_limit.block", "ha_proxy.connections_rate_limit.connections") do |block, connections| -%> - <%-if block -%> - tcp-request <%= tcp_request_phase %> reject if { sc_conn_rate(0) gt <%= connections %> } - <%- end -%> - <%- end -%> + tcp-request <%= tcp_request_phase %> reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) } <%- end -%> <%- if_p("ha_proxy.requests_rate_limit.table_size", "ha_proxy.requests_rate_limit.window_size") do -%> http-request track-sc1 src table st_http_req_rate @@ -571,11 +582,7 @@ frontend https-in tcp-request <%= tcp_request_phase %> reject if layer4_block <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%> tcp-request <%= tcp_request_phase %> track-sc0 src table st_tcp_conn_rate - <%- if_p("ha_proxy.connections_rate_limit.block", "ha_proxy.connections_rate_limit.connections") do |block, connections| -%> - <%-if block -%> - tcp-request <%= tcp_request_phase %> reject if { sc_conn_rate(0) gt <%= connections %> } - <%- end -%> - <%- end -%> + tcp-request <%= tcp_request_phase %> reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) } <%- end -%> <%- if_p("ha_proxy.requests_rate_limit.table_size", "ha_proxy.requests_rate_limit.window_size") do -%> http-request track-sc1 src table st_http_req_rate diff --git a/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb b/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb index a3bce227..1110c11a 100644 --- a/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb +++ b/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb @@ -61,6 +61,21 @@ end end + context 'when ha_proxy.connections_rate_limit "window_size" and "table_size" are NOT provided' do + context 'when "connections" and "block" are set in manifest' do + let(:properties) do + default_properties.deep_merge({ + 'connections_rate_limit' => { 'connections' => '5', 'block' => true } + }) + end + + it 'does not set proc.conn_rate_limit or proc.conn_rate_block in global section' do + expect(haproxy_conf['global']).not_to include('set-var proc.conn_rate_limit') + expect(haproxy_conf['global']).not_to include('set-var proc.conn_rate_block') + end + end + end + context 'when ha_proxy.connections_rate_limit properties "window_size", "table_size" are provided' do let(:backend_conn_rate) { haproxy_conf['backend st_tcp_conn_rate'] } @@ -88,6 +103,16 @@ expect(frontend_https).to include('tcp-request connection track-sc0 src table st_tcp_conn_rate') end + it 'always emits the reject rule (even without connections or block set in manifest)' do + expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + end + + it 'always sets proc.conn_rate_block to false in global when block is not configured in manifest' do + expect(haproxy_conf['global']).to include('set-var proc.conn_rate_block bool(false)') + expect(haproxy_conf['global']).not_to include('set-var proc.conn_rate_limit') + end + context 'when proxy protocol used' do let(:properties) do temp_properties.deep_merge({ 'accept_proxy' => true }) @@ -104,23 +129,49 @@ temp_properties.deep_merge({ 'connections_rate_limit' => { 'connections' => '5', 'block' => 'true' } }) end - it 'adds tcp-request connection reject to http-in and https-in frontends' do - expect(frontend_http).to include('tcp-request connection reject if { sc_conn_rate(0) gt 5 }') + it 'adds tcp-request connection reject using process variables to http-in and https-in frontends' do + expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') expect(frontend_http).to include('tcp-request connection track-sc0 src table st_tcp_conn_rate') - expect(frontend_https).to include('tcp-request connection reject if { sc_conn_rate(0) gt 5 }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') expect(frontend_https).to include('tcp-request connection track-sc0 src table st_tcp_conn_rate') end end + context 'when "connections" is provided but "block" is false' do + let(:properties) do + temp_properties.deep_merge({ 'connections_rate_limit' => { 'connections' => '10', 'block' => false } }) + end + + it 'sets proc.conn_rate_limit and proc.conn_rate_block process variables in global section' do + expect(haproxy_conf['global']).to include('set-var proc.conn_rate_limit int(10)') + expect(haproxy_conf['global']).to include('set-var proc.conn_rate_block bool(false)') + end + + it 'still emits reject rule (rejection controlled at runtime via proc.conn_rate_block variable)' do + expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + end + end + + context 'when only "block" is true but "connections" is not set in manifest' do + let(:properties) do + temp_properties.deep_merge({ 'connections_rate_limit' => { 'block' => true } }) + end + + it 'raises a validation error to prevent total lockout (every client with >= 1 connection would be blocked)' do + expect { haproxy_conf }.to raise_error(/connections_rate_limit.connections must be set when connections_rate_limit.block is true/) + end + end + context 'when proxy protocol used and "connections" and "block" are also provided' do let(:properties) do - temp_properties.deep_merge({ 'accept_proxy' => true, 'connections_rate_limit' => { 'connections' => '5', 'block' => 'true' } }) + temp_properties.deep_merge({ 'accept_proxy' => true, 'connections_rate_limit' => { 'connections' => '5', 'block' => true } }) end - it 'adds tcp-request session reject to http-in and https-in frontends' do - expect(frontend_http).to include('tcp-request session reject if { sc_conn_rate(0) gt 5 }') + it 'adds tcp-request session reject using process variables to http-in and https-in frontends' do + expect(frontend_http).to include('tcp-request session reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') expect(frontend_http).to include('tcp-request session track-sc0 src table st_tcp_conn_rate') - expect(frontend_https).to include('tcp-request session reject if { sc_conn_rate(0) gt 5 }') + expect(frontend_https).to include('tcp-request session reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') expect(frontend_https).to include('tcp-request session track-sc0 src table st_tcp_conn_rate') end end From b72c7bd7a6102f35e8729c555ffe13f62da211cf Mon Sep 17 00:00:00 2001 From: Tamara Boehm Date: Tue, 2 Jun 2026 09:00:41 +0200 Subject: [PATCH 2/4] Rework rate limiting tests and enhance vars in ACL tcp-request reject --- acceptance-tests/rate_limit_test.go | 162 +----------------- jobs/haproxy/templates/haproxy.config.erb | 6 +- .../haproxy_config/rate_limit_spec.rb | 16 +- 3 files changed, 20 insertions(+), 164 deletions(-) diff --git a/acceptance-tests/rate_limit_test.go b/acceptance-tests/rate_limit_test.go index f123db6f..c39f3eb2 100644 --- a/acceptance-tests/rate_limit_test.go +++ b/acceptance-tests/rate_limit_test.go @@ -20,16 +20,16 @@ var _ = Describe("Rate-Limiting", func() { value: 10s - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/table_size? - value: 100 + value: 1k - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections value: %d - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size? - value: 100s + value: 10s - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size? - value: 100 + value: 1k `, rateLimit, rateLimit) haproxyBackendPort := 12000 haproxyInfo, _ := deployHAProxy(baseManifestVars{ @@ -118,10 +118,10 @@ var _ = Describe("Rate-Limiting", func() { value: %d - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size? - value: 100s + value: 10s - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size? - value: 100 + value: 1k - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/block? value: true @@ -280,8 +280,8 @@ var _ = Describe("Rate-Limiting", func() { Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusTooManyRequests)) } else { - // TCP connection limit reached --> no response - Expect(err).To(HaveOccurred()) + // TCP connection limit reached --> no response + Expect(err).To(HaveOccurred()) } } }) @@ -294,7 +294,7 @@ var _ = Describe("Rate-Limiting", func() { value: %d - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size? - value: 100s + value: 10s - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size? value: 100 @@ -380,150 +380,4 @@ var _ = Describe("Rate-Limiting", func() { Expect(firstFailure).To(Equal(newLimit)) Expect(successfulRequestCount).To(Equal(newLimit)) }) - - It("Connection Based Limiting can be enabled and disabled at runtime via socket with manifest block false", func() { - connLimit := 5 - // block: false in manifest — blocking is enabled/disabled at runtime via socket - opsfileConnectionsRateLimit := fmt.Sprintf(`--- -- type: replace - path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections - value: %d -- type: replace - path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size? - value: 100s -- type: replace - path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size? - value: 100 -- type: replace - path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/block? - value: false -`, connLimit) - haproxyBackendPort := 12000 - haproxyInfo, _ := deployHAProxy(baseManifestVars{ - haproxyBackendPort: haproxyBackendPort, - haproxyBackendServers: []string{"127.0.0.1"}, - deploymentName: deploymentNameForTestNode(), - }, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true) - - closeLocalServer, localPort := startDefaultTestServer() - defer closeLocalServer() - - closeTunnel := setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort) - defer closeTunnel() - - By("Verifying proc.conn_rate_block is initialised as false from manifest block: false") - output := runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") - Expect(output).To(ContainSubstring("0")) - - By("Enabling blocking at runtime via socket") - runHAProxySocketCommand(haproxyInfo, "experimental-mode on; set var proc.conn_rate_block bool(true)") - - By("Verifying proc.conn_rate_block is now true") - output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") - Expect(output).To(ContainSubstring("1")) - - By("Verifying connections are blocked after exceeding the limit") - testRequestCount := int(float64(connLimit) * 1.5) - firstFailure := -1 - successfulRequestCount := 0 - for i := 0; i < testRequestCount; i++ { - rt := &http.Transport{DisableKeepAlives: true} - client := &http.Client{Transport: rt} - resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP)) - if err == nil && resp.StatusCode == 200 { - resp.Body.Close() - successfulRequestCount++ - continue - } - if err == nil { - resp.Body.Close() - } - if firstFailure == -1 { - firstFailure = i - } - } - Expect(firstFailure).To(Equal(connLimit)) - Expect(successfulRequestCount).To(Equal(connLimit)) - - By("Disabling blocking at runtime via socket") - runHAProxySocketCommand(haproxyInfo, "experimental-mode on; set var proc.conn_rate_block bool(false)") - - By("Verifying proc.conn_rate_block is now false") - output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") - Expect(output).To(ContainSubstring("0")) - - By("Clearing stick table to reset counters") - runHAProxySocketCommand(haproxyInfo, "clear table st_tcp_conn_rate") - - By("Verifying all connections are now allowed after disabling blocking via socket") - for i := 0; i < testRequestCount; i++ { - rt := &http.Transport{DisableKeepAlives: true} - client := &http.Client{Transport: rt} - resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP)) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) - resp.Body.Close() - } - }) - - It("Connection Based Limiting works when limit is set entirely via socket without manifest connections property", func() { - connLimit := 5 - // Only table_size and window_size are set — no connections or block in manifest - // proc.conn_rate_block defaults to false, proc.conn_rate_limit must be set via socket - opsfileConnectionsRateLimit := `--- -- type: replace - path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/window_size - value: 100s -- type: replace - path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size? - value: 100 -` - haproxyBackendPort := 12000 - haproxyInfo, _ := deployHAProxy(baseManifestVars{ - haproxyBackendPort: haproxyBackendPort, - haproxyBackendServers: []string{"127.0.0.1"}, - deploymentName: deploymentNameForTestNode(), - }, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true) - - closeLocalServer, localPort := startDefaultTestServer() - defer closeLocalServer() - - closeTunnel := setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort) - defer closeTunnel() - - By("Setting conn_rate_limit and enabling blocking entirely via socket") - runHAProxySocketCommand(haproxyInfo, fmt.Sprintf("experimental-mode on; set var proc.conn_rate_limit int(%d)", connLimit)) - runHAProxySocketCommand(haproxyInfo, "experimental-mode on; set var proc.conn_rate_block bool(true)") - - By("Verifying proc.conn_rate_limit is set correctly via socket") - output := runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_limit") - Expect(output).To(ContainSubstring(fmt.Sprintf("%d", connLimit))) - - By("Verifying proc.conn_rate_block is set correctly via socket") - output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") - Expect(output).To(ContainSubstring("1")) - - By("Verifying connections are blocked after exceeding the socket-configured limit") - testRequestCount := int(float64(connLimit) * 1.5) - firstFailure := -1 - successfulRequestCount := 0 - for i := 0; i < testRequestCount; i++ { - rt := &http.Transport{DisableKeepAlives: true} - client := &http.Client{Transport: rt} - resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP)) - if err == nil && resp.StatusCode == 200 { - resp.Body.Close() - successfulRequestCount++ - continue - } - if err == nil { - resp.Body.Close() - } - if firstFailure == -1 { - firstFailure = i - } - } - Expect(firstFailure).To(Equal(connLimit)) - Expect(successfulRequestCount).To(Equal(connLimit)) - }) }) diff --git a/jobs/haproxy/templates/haproxy.config.erb b/jobs/haproxy/templates/haproxy.config.erb index be82b8c1..4789af0c 100644 --- a/jobs/haproxy/templates/haproxy.config.erb +++ b/jobs/haproxy/templates/haproxy.config.erb @@ -452,7 +452,8 @@ frontend http-in tcp-request <%= tcp_request_phase %> reject if layer4_block <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%> tcp-request <%= tcp_request_phase %> track-sc0 src table st_tcp_conn_rate - tcp-request <%= tcp_request_phase %> reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) } + # use sub() converter as variable references are only accepted as arguments to converters + tcp-request <%= tcp_request_phase %> reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 } <%- end -%> <%- if_p("ha_proxy.requests_rate_limit.table_size", "ha_proxy.requests_rate_limit.window_size") do -%> http-request track-sc1 src table st_http_req_rate @@ -582,7 +583,8 @@ frontend https-in tcp-request <%= tcp_request_phase %> reject if layer4_block <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%> tcp-request <%= tcp_request_phase %> track-sc0 src table st_tcp_conn_rate - tcp-request <%= tcp_request_phase %> reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) } + # use sub() converter as variable references are only accepted as arguments to converters + tcp-request <%= tcp_request_phase %> reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 } <%- end -%> <%- if_p("ha_proxy.requests_rate_limit.table_size", "ha_proxy.requests_rate_limit.window_size") do -%> http-request track-sc1 src table st_http_req_rate diff --git a/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb b/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb index 1110c11a..9ba9a0da 100644 --- a/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb +++ b/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb @@ -104,8 +104,8 @@ end it 'always emits the reject rule (even without connections or block set in manifest)' do - expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') - expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') end it 'always sets proc.conn_rate_block to false in global when block is not configured in manifest' do @@ -130,9 +130,9 @@ end it 'adds tcp-request connection reject using process variables to http-in and https-in frontends' do - expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') expect(frontend_http).to include('tcp-request connection track-sc0 src table st_tcp_conn_rate') - expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') expect(frontend_https).to include('tcp-request connection track-sc0 src table st_tcp_conn_rate') end end @@ -148,8 +148,8 @@ end it 'still emits reject rule (rejection controlled at runtime via proc.conn_rate_block variable)' do - expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') - expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') end end @@ -169,9 +169,9 @@ end it 'adds tcp-request session reject using process variables to http-in and https-in frontends' do - expect(frontend_http).to include('tcp-request session reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + expect(frontend_http).to include('tcp-request session reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') expect(frontend_http).to include('tcp-request session track-sc0 src table st_tcp_conn_rate') - expect(frontend_https).to include('tcp-request session reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) gt 0 } { sc_conn_rate(0) gt var(proc.conn_rate_limit) }') + expect(frontend_https).to include('tcp-request session reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') expect(frontend_https).to include('tcp-request session track-sc0 src table st_tcp_conn_rate') end end From a0671d584f99158b81b89c806f2b3a3f8c17793d Mon Sep 17 00:00:00 2001 From: Tamara Boehm <34028368+b1tamara@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:37:13 +0200 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Clemens Hoffmann --- docs/rate_limiting.md | 2 +- jobs/haproxy/templates/haproxy.config.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/rate_limiting.md b/docs/rate_limiting.md index 862f6606..48fbff3e 100644 --- a/docs/rate_limiting.md +++ b/docs/rate_limiting.md @@ -134,7 +134,7 @@ echo "show table st_tcp_conn_rate" | socat stdio /var/vcap/sys/run/haproxy/stats The `connections_rate_limit.block` flag and `connections_rate_limit.connections` threshold are stored as HAProxy process-level variables (`proc.conn_rate_block` and `proc.conn_rate_limit`) and can be changed at runtime without a reload. This requires `ha_proxy.master_cli_enable: true` or `ha_proxy.stats_enable: true`. -The socket is located at `/var/vcap/sys/run/haproxy/stats.sock`. You will likely need `sudo` to access it. +The socket is located at `/var/vcap/sys/run/haproxy/stats.sock`. You will likely need `root` permissions to access it. > **Note:** The `tcp-request connection reject` rule is always present in the config as long as `connections_rate_limit.table_size` and `connections_rate_limit.window_size` are set. Enforcement is controlled entirely at runtime via `proc.conn_rate_block` and `proc.conn_rate_limit`. Setting `connections_rate_limit.connections` and `connections_rate_limit.block` in the manifest only sets their **initial values** at startup — they can be freely overridden via socket without a reload. diff --git a/jobs/haproxy/templates/haproxy.config.erb b/jobs/haproxy/templates/haproxy.config.erb index 4789af0c..c6e477e1 100644 --- a/jobs/haproxy/templates/haproxy.config.erb +++ b/jobs/haproxy/templates/haproxy.config.erb @@ -234,7 +234,7 @@ end if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do if p("ha_proxy.connections_rate_limit.block", false) if !p("ha_proxy.connections_rate_limit.connections", nil) - abort "Conflicting configuration: connections_rate_limit.connections must be set when connections_rate_limit.block is true (otherwise every client with >= 1 connection would be blocked)" + abort "connections_rate_limit.connections must be set in the manifest as the initial threshold when block is true; otherwise rate-limiting will be silently disabled until a value is set via the runtime API." end end end From f2f888d2724a58e16499410b70ecc2dc1ced9178 Mon Sep 17 00:00:00 2001 From: Tamara Boehm Date: Thu, 11 Jun 2026 12:33:05 +0200 Subject: [PATCH 4/4] Rename variables and rewrite docu --- acceptance-tests/rate_limit_test.go | 12 +++--- docs/rate_limiting.md | 34 +++++++++++------ jobs/haproxy/templates/haproxy.config.erb | 8 ++-- .../haproxy_config/rate_limit_spec.rb | 38 +++++++++---------- 4 files changed, 51 insertions(+), 41 deletions(-) diff --git a/acceptance-tests/rate_limit_test.go b/acceptance-tests/rate_limit_test.go index c39f3eb2..2400d173 100644 --- a/acceptance-tests/rate_limit_test.go +++ b/acceptance-tests/rate_limit_test.go @@ -315,12 +315,12 @@ var _ = Describe("Rate-Limiting", func() { closeTunnel := setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort) defer closeTunnel() - By("Verifying proc.conn_rate_limit is initialised from manifest value") - output := runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_limit") + By("Verifying proc.connections_rate_limit_connections is initialised from manifest value") + output := runHAProxySocketCommand(haproxyInfo, "get var proc.connections_rate_limit_connections") Expect(output).To(ContainSubstring(fmt.Sprintf("%d", connLimit))) - By("Verifying proc.conn_rate_block is initialised as true from manifest block: true") - output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_block") + By("Verifying proc.connections_rate_limit_block is initialised as true from manifest block: true") + output = runHAProxySocketCommand(haproxyInfo, "get var proc.connections_rate_limit_block") Expect(output).To(ContainSubstring("1")) By("Verifying connections are blocked after exceeding the manifest-configured limit") @@ -351,10 +351,10 @@ var _ = Describe("Rate-Limiting", func() { By("Overriding the limit at runtime via socket to a higher value") newLimit := connLimit * 3 - runHAProxySocketCommand(haproxyInfo, fmt.Sprintf("experimental-mode on; set var proc.conn_rate_limit int(%d)", newLimit)) + runHAProxySocketCommand(haproxyInfo, fmt.Sprintf("experimental-mode on; set var proc.connections_rate_limit_connections int(%d)", newLimit)) By("Verifying the override is reflected via get var") - output = runHAProxySocketCommand(haproxyInfo, "get var proc.conn_rate_limit") + output = runHAProxySocketCommand(haproxyInfo, "get var proc.connections_rate_limit_connections") Expect(output).To(ContainSubstring(fmt.Sprintf("%d", newLimit))) By("Verifying connections are allowed up to the new higher socket-configured limit") diff --git a/docs/rate_limiting.md b/docs/rate_limiting.md index 48fbff3e..2dd4265d 100644 --- a/docs/rate_limiting.md +++ b/docs/rate_limiting.md @@ -110,7 +110,7 @@ frontend http-in ``` ## Querying Current Stick-Table Status -To get more insight into what is going on inside HAProxy regarding its rate limits, you can query the stats socket to get the raw table data: +To get more insight into what is going on inside HAProxy regarding its rate limits, you can query the stats socket at `/var/vcap/sys/run/haproxy/stats.sock` to get the raw table data: ```bash $ echo "show table st_http_req_rate" | socat /var/vcap/sys/run/haproxy/stats.sock - @@ -132,42 +132,52 @@ echo "show table st_tcp_conn_rate" | socat stdio /var/vcap/sys/run/haproxy/stats ## Control Connection Rate Limiting via HAProxy Runtime API -The `connections_rate_limit.block` flag and `connections_rate_limit.connections` threshold are stored as HAProxy process-level variables (`proc.conn_rate_block` and `proc.conn_rate_limit`) and can be changed at runtime without a reload. This requires `ha_proxy.master_cli_enable: true` or `ha_proxy.stats_enable: true`. +Normally, changing rate-limit settings requires updating the manifest and reloading HAProxy. Using the HAProxy Runtime API, blocking can be enabled or disabled, and the connection threshold can be tightened or loosened while HAProxy continues running and serving traffic. This is particularly useful during an active incident, when a rapid reaction is needed. -The socket is located at `/var/vcap/sys/run/haproxy/stats.sock`. You will likely need `root` permissions to access it. +### Prerequisites -> **Note:** The `tcp-request connection reject` rule is always present in the config as long as `connections_rate_limit.table_size` and `connections_rate_limit.window_size` are set. Enforcement is controlled entirely at runtime via `proc.conn_rate_block` and `proc.conn_rate_limit`. Setting `connections_rate_limit.connections` and `connections_rate_limit.block` in the manifest only sets their **initial values** at startup — they can be freely overridden via socket without a reload. +- `ha_proxy.master_cli_enable: true` or `ha_proxy.stats_enable: true` must be set in the manifest to enable the HAProxy Runtime API. +- `ha_proxy.connections_rate_limit.table_size` and `ha_proxy.connections_rate_limit.window_size` must be defined in the manifest to create the stick table and enable connection tracking. +- `root` permissions are required to write to the socket. + +### How Runtime Control Works + +When HAProxy starts, it reads `connections_rate_limit.block` and `connections_rate_limit.connections` from the manifest and stores them as process-level variables inside the running HAProxy process. Updating a variable instantly changes the behavior for all subsequent connections, as every new TCP connection is evaluated against these variables in real time. + +These variables are updated by sending plain-text commands to the HAProxy stats socket. The socket is available as long as HAProxy is running, and any change persists until the next redeploy, at which point the manifest values are restored. + +> Note: the connections threshold is applied per the defined window_size, which is also used for counting connections. For example, if `window_size` is set to `10s` and `connections` is set to `100`, then the threshold of 100 connections applies to every 10-second window. ### Inspect Current Variable Values ```bash -echo "get var proc.conn_rate_limit" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock -# => proc.conn_rate_limit: type=sint value=<100> +echo "get var proc.connections_rate_limit_connections" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +# => proc.connections_rate_limit_connections: type=sint value=<600> -echo "get var proc.conn_rate_block" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock -# => proc.conn_rate_block: type=bool value=<1> +echo "get var proc.connections_rate_limit_block" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +# => proc.connections_rate_limit_block: type=bool value=<1> ``` ### Enable or Disable Blocking at Runtime ```bash # Enable blocking (equivalent to setting block: true in the manifest) -echo "experimental-mode on; set var proc.conn_rate_block bool(true)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +echo "experimental-mode on; set var proc.connections_rate_limit_block bool(true)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock # Disable blocking without reloading (equivalent to setting block: false in the manifest) -echo "experimental-mode on; set var proc.conn_rate_block bool(false)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +echo "experimental-mode on; set var proc.connections_rate_limit_block bool(false)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock ``` ### Adjust the Connections Threshold at Runtime ```bash # Allow up to 100 connections per window (equivalent to setting connections: 100 in the manifest) -echo "experimental-mode on; set var proc.conn_rate_limit int(100)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +echo "experimental-mode on; set var proc.connections_rate_limit_connections int(100)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock ``` ### Enable Rate Limiting and Set Threshold in One Step ```bash -echo "experimental-mode on; set var proc.conn_rate_limit int(100); set var proc.conn_rate_block bool(true)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock +echo "experimental-mode on; set var proc.connections_rate_limit_connections int(100); set var proc.connections_rate_limit_block bool(true)" | sudo socat stdio /var/vcap/sys/run/haproxy/stats.sock ``` diff --git a/jobs/haproxy/templates/haproxy.config.erb b/jobs/haproxy/templates/haproxy.config.erb index c6e477e1..12ef57d7 100644 --- a/jobs/haproxy/templates/haproxy.config.erb +++ b/jobs/haproxy/templates/haproxy.config.erb @@ -335,9 +335,9 @@ global <%- end -%> <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%> <%- if_p("ha_proxy.connections_rate_limit.connections") do |conn_rate_connections| -%> - set-var proc.conn_rate_limit int(<%= conn_rate_connections %>) + set-var proc.connections_rate_limit_connections int(<%= conn_rate_connections %>) <%- end -%> - set-var proc.conn_rate_block bool(<%= p("ha_proxy.connections_rate_limit.block", false) %>) + set-var proc.connections_rate_limit_block bool(<%= p("ha_proxy.connections_rate_limit.block", false) %>) <%- end -%> <%- if p("ha_proxy.always_allow_body_http10") %> h1-accept-payload-with-any-method @@ -453,7 +453,7 @@ frontend http-in <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%> tcp-request <%= tcp_request_phase %> track-sc0 src table st_tcp_conn_rate # use sub() converter as variable references are only accepted as arguments to converters - tcp-request <%= tcp_request_phase %> reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 } + tcp-request <%= tcp_request_phase %> reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 } <%- end -%> <%- if_p("ha_proxy.requests_rate_limit.table_size", "ha_proxy.requests_rate_limit.window_size") do -%> http-request track-sc1 src table st_http_req_rate @@ -584,7 +584,7 @@ frontend https-in <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%> tcp-request <%= tcp_request_phase %> track-sc0 src table st_tcp_conn_rate # use sub() converter as variable references are only accepted as arguments to converters - tcp-request <%= tcp_request_phase %> reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 } + tcp-request <%= tcp_request_phase %> reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 } <%- end -%> <%- if_p("ha_proxy.requests_rate_limit.table_size", "ha_proxy.requests_rate_limit.window_size") do -%> http-request track-sc1 src table st_http_req_rate diff --git a/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb b/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb index 9ba9a0da..f041cf90 100644 --- a/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb +++ b/spec/haproxy/templates/haproxy_config/rate_limit_spec.rb @@ -69,9 +69,9 @@ }) end - it 'does not set proc.conn_rate_limit or proc.conn_rate_block in global section' do - expect(haproxy_conf['global']).not_to include('set-var proc.conn_rate_limit') - expect(haproxy_conf['global']).not_to include('set-var proc.conn_rate_block') + it 'does not set proc.connections_rate_limit_connections or proc.connections_rate_limit_block in global section' do + expect(haproxy_conf['global']).not_to include('set-var proc.connections_rate_limit_connections') + expect(haproxy_conf['global']).not_to include('set-var proc.connections_rate_limit_block') end end end @@ -104,13 +104,13 @@ end it 'always emits the reject rule (even without connections or block set in manifest)' do - expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') - expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') + expect(frontend_http).to include('tcp-request connection reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }') end - it 'always sets proc.conn_rate_block to false in global when block is not configured in manifest' do - expect(haproxy_conf['global']).to include('set-var proc.conn_rate_block bool(false)') - expect(haproxy_conf['global']).not_to include('set-var proc.conn_rate_limit') + it 'always sets proc.connections_rate_limit_block to false in global when block is not configured in manifest' do + expect(haproxy_conf['global']).to include('set-var proc.connections_rate_limit_block bool(false)') + expect(haproxy_conf['global']).not_to include('set-var proc.connections_rate_limit_connections') end context 'when proxy protocol used' do @@ -130,9 +130,9 @@ end it 'adds tcp-request connection reject using process variables to http-in and https-in frontends' do - expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') + expect(frontend_http).to include('tcp-request connection reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }') expect(frontend_http).to include('tcp-request connection track-sc0 src table st_tcp_conn_rate') - expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }') expect(frontend_https).to include('tcp-request connection track-sc0 src table st_tcp_conn_rate') end end @@ -142,14 +142,14 @@ temp_properties.deep_merge({ 'connections_rate_limit' => { 'connections' => '10', 'block' => false } }) end - it 'sets proc.conn_rate_limit and proc.conn_rate_block process variables in global section' do - expect(haproxy_conf['global']).to include('set-var proc.conn_rate_limit int(10)') - expect(haproxy_conf['global']).to include('set-var proc.conn_rate_block bool(false)') + it 'sets proc.connections_rate_limit_connections and proc.connections_rate_limit_block process variables in global section' do + expect(haproxy_conf['global']).to include('set-var proc.connections_rate_limit_connections int(10)') + expect(haproxy_conf['global']).to include('set-var proc.connections_rate_limit_block bool(false)') end - it 'still emits reject rule (rejection controlled at runtime via proc.conn_rate_block variable)' do - expect(frontend_http).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') - expect(frontend_https).to include('tcp-request connection reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') + it 'still emits reject rule (rejection controlled at runtime via proc.connections_rate_limit_block variable)' do + expect(frontend_http).to include('tcp-request connection reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }') + expect(frontend_https).to include('tcp-request connection reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }') end end @@ -159,7 +159,7 @@ end it 'raises a validation error to prevent total lockout (every client with >= 1 connection would be blocked)' do - expect { haproxy_conf }.to raise_error(/connections_rate_limit.connections must be set when connections_rate_limit.block is true/) + expect { haproxy_conf }.to raise_error(/connections_rate_limit.connections must be set in the manifest as the initial threshold when block is true/) end end @@ -169,9 +169,9 @@ end it 'adds tcp-request session reject using process variables to http-in and https-in frontends' do - expect(frontend_http).to include('tcp-request session reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') + expect(frontend_http).to include('tcp-request session reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }') expect(frontend_http).to include('tcp-request session track-sc0 src table st_tcp_conn_rate') - expect(frontend_https).to include('tcp-request session reject if { var(proc.conn_rate_block) -m bool } { var(proc.conn_rate_limit) -m int gt 0 } { sc_conn_rate(0),sub(proc.conn_rate_limit) gt 0 }') + expect(frontend_https).to include('tcp-request session reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }') expect(frontend_https).to include('tcp-request session track-sc0 src table st_tcp_conn_rate') end end