Changing tracking metric type to histogram from counter

Signed-off-by: Paurush Garg <paurushg@amazon.com>

Author: PaurushGarg

CHANGELOG.md

@@ -46,6 +46,7 @@
 * [ENHANCEMENT] Distributor: Validate metric name before removing empty labels. #7253
 * [ENHANCEMENT] Make cortex_ingester_tsdb_sample_ooo_delta metric per-tenant #7278
 * [ENHANCEMENT] Distributor: Add dimension `nhcb` to keep track of nhcb samples in `cortex_distributor_received_samples_total` and `cortex_distributor_samples_in_total` metrics.
+* [ENHANCEMENT] Ingester: Added `cortex_ingester_ingested_histogram_buckets` metric to track number of histogram buckets ingested per user. #7297
 * [BUGFIX] Distributor: If remote write v2 is disabled, explicitly return HTTP 415 (Unsupported Media Type) for Remote Write V2 requests instead of attempting to parse them as V1. #7238
 * [BUGFIX] Ring: Change DynamoDB KV to retry indefinitely for WatchKey. #7088
 * [BUGFIX] Ruler: Add XFunctions validation support. #7111

pkg/ingester/ingester.go

@@ -1342,7 +1342,6 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 		succeededExemplarsCount                = 0
 		failedExemplarsCount                   = 0
 		startAppend                            = time.Now()
-		succeededHistogramBucketsCount         = 0
 		sampleOutOfBoundsCount                 = 0
 		sampleOutOfOrderCount                  = 0
 		sampleTooOldCount                      = 0
@@ -1508,7 +1507,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 				if ref != 0 {
 					if _, err = app.AppendHistogram(ref, copiedLabels, hp.TimestampMs, h, fh); err == nil {
 						succeededHistogramsCount++
-						succeededHistogramBucketsCount += hp.BucketCount()
+						i.metrics.ingestedHistogramBuckets.WithLabelValues(userID).Observe(float64(hp.BucketCount()))
 						continue
 					}
 				} else {
@@ -1520,7 +1519,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 							newSeries = append(newSeries, copiedLabels)
 						}
 						succeededHistogramsCount++
-						succeededHistogramBucketsCount += hp.BucketCount()
+						i.metrics.ingestedHistogramBuckets.WithLabelValues(userID).Observe(float64(hp.BucketCount()))
 						continue
 					}
 				}
@@ -1615,7 +1614,6 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 	i.metrics.ingestedSamplesFail.Add(float64(failedSamplesCount))
 	i.metrics.ingestedHistograms.Add(float64(succeededHistogramsCount))
 	i.metrics.ingestedHistogramsFail.Add(float64(failedHistogramsCount))
-	i.metrics.ingestedHistogramBuckets.WithLabelValues(userID).Add(float64(succeededHistogramBucketsCount))
 	i.metrics.ingestedExemplars.Add(float64(succeededExemplarsCount))
 	i.metrics.ingestedExemplarsFail.Add(float64(failedExemplarsCount))
 

pkg/ingester/metrics.go

@@ -38,7 +38,7 @@ type ingesterMetrics struct {
 	ingestedHistogramsFail  prometheus.Counter
 	ingestedExemplarsFail   prometheus.Counter
 	ingestedMetadataFail    prometheus.Counter
-	ingestedHistogramBuckets *prometheus.CounterVec
+	ingestedHistogramBuckets *prometheus.HistogramVec
 	oooLabelsTotal          *prometheus.CounterVec
 	queries                 prometheus.Counter
 	queriedSamples          prometheus.Histogram
@@ -131,9 +131,13 @@ func newIngesterMetrics(r prometheus.Registerer,
 			Name: "cortex_ingester_ingested_metadata_failures_total",
 			Help: "The total number of metadata that errored on ingestion.",
 		}),
-		ingestedHistogramBuckets: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
-			Name: "cortex_ingester_ingested_histogram_buckets_total",
-			Help: "The total number of histogram buckets ingested per user.",
+		ingestedHistogramBuckets: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
+			Name:                            "cortex_ingester_ingested_histogram_buckets",
+			Help:                            "The number of ingested native histogram buckets per user.",
+			NativeHistogramBucketFactor:     1.1,
+			NativeHistogramMaxBucketNumber:  100,
+			NativeHistogramMinResetDuration: 1,
+			Buckets:                         prometheus.ExponentialBuckets(1, 2, 10), // 1 to 512 buckets
 		}, []string{"user"}),
 		oooLabelsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
 			Name: "cortex_ingester_out_of_order_labels_total",

pkg/ingester/metrics_test.go

@@ -157,6 +157,8 @@ func TestIngesterMetrics(t *testing.T) {
 			# HELP cortex_ingester_ingested_native_histograms_failures_total The total number of native histograms that errored on ingestion.
 			# TYPE cortex_ingester_ingested_native_histograms_failures_total counter
 			cortex_ingester_ingested_native_histograms_failures_total 0
+			# HELP cortex_ingester_ingested_histogram_buckets The number of ingested native histogram buckets per user.
+			# TYPE cortex_ingester_ingested_histogram_buckets histogram
 			# HELP cortex_ingester_ingestion_rate_samples_per_second Current ingestion rate in samples/sec that ingester is using to limit access.
 			# TYPE cortex_ingester_ingestion_rate_samples_per_second gauge
 			cortex_ingester_ingestion_rate_samples_per_second 0