Skip to content

Commit 435e4dc

Browse files
committed
fix: prevent race condition in concurrent Prometheus scrapes
Add CollectAndServe method that holds the mutex during both metric collection and serving. This prevents a race where one request could reset metrics while another is still serving them.
1 parent e3b32d0 commit 435e4dc

2 files changed

Lines changed: 18 additions & 2 deletions

File tree

engine/internal/srv/metrics/collector.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"encoding/json"
1010
"fmt"
11+
"net/http"
1112
"strconv"
1213
"sync"
1314
"time"
@@ -76,6 +77,20 @@ func (c *Collector) Collect(ctx context.Context) {
7677
c.mu.Lock()
7778
defer c.mu.Unlock()
7879

80+
c.collectAll(ctx)
81+
}
82+
83+
// CollectAndServe collects metrics and serves them while holding the lock.
84+
// This prevents race conditions between concurrent Prometheus scrapes.
85+
func (c *Collector) CollectAndServe(ctx context.Context, handler http.Handler, w http.ResponseWriter, r *http.Request) {
86+
c.mu.Lock()
87+
defer c.mu.Unlock()
88+
89+
c.collectAll(ctx)
90+
handler.ServeHTTP(w, r)
91+
}
92+
93+
func (c *Collector) collectAll(ctx context.Context) {
7994
c.metrics.Reset()
8095

8196
c.collectInstanceMetrics()

engine/internal/srv/server.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -304,8 +304,9 @@ func reportLaunching(cfg *srvCfg.Config) {
304304

305305
// metricsHandler returns an HTTP handler that collects and exposes Prometheus metrics.
306306
func (s *Server) metricsHandler() http.Handler {
307+
handler := promhttp.HandlerFor(s.metricsRegistry, promhttp.HandlerOpts{})
308+
307309
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
308-
s.metricsCollector.Collect(r.Context())
309-
promhttp.HandlerFor(s.metricsRegistry, promhttp.HandlerOpts{}).ServeHTTP(w, r)
310+
s.metricsCollector.CollectAndServe(r.Context(), handler, w, r)
310311
})
311312
}

0 commit comments

Comments
 (0)