cortex-jsonnet/cortex-mixin/alerts/compactor.libsonnet at 0b041c6de9f15ced166344f1bed839361bb61cce · cortexproject/cortex-jsonnet · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
{
  groups+: [
    {
      name: 'cortex_compactor_alerts',
      rules: [
        {
          // Alert if the compactor has not successfully cleaned up blocks in the last 6h.
          alert: 'CortexCompactorHasNotSuccessfullyCleanedUpBlocks',
          'for': '1h',
          expr: |||
            (time() - cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds > 60 * 60 * 6)
          |||,
          labels: {
            severity: 'critical',
          },
          annotations: {
            message: |||
              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully cleaned up blocks in the last 6 hours.
            ||| % $._config,
          },
        },
        {
          // Alert if the compactor has not successfully run compaction in the last 24h.
          alert: 'CortexCompactorHasNotSuccessfullyRunCompaction',
          'for': '1h',
          expr: |||
            (time() - cortex_compactor_last_successful_run_timestamp_seconds > 60 * 60 * 24)
            and
            (cortex_compactor_last_successful_run_timestamp_seconds > 0)
          |||,
          labels: {
            severity: 'critical',
          },
          annotations: {
            message: |||
              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.
            ||| % $._config,
          },
        },
        {
          // Alert if the compactor has not successfully run compaction in the last 24h since startup.
          alert: 'CortexCompactorHasNotSuccessfullyRunCompaction',
          'for': '24h',
          expr: |||
            cortex_compactor_last_successful_run_timestamp_seconds == 0
          |||,
          labels: {
            severity: 'critical',
          },
          annotations: {
            message: |||
              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.
            ||| % $._config,
          },
        },
        {
          // Alert if compactor failed to run 2 consecutive compactions.
          alert: 'CortexCompactorHasNotSuccessfullyRunCompaction',
          expr: |||
            increase(cortex_compactor_runs_failed_total[2h]) >= 2
          |||,
          labels: {
            severity: 'critical',
          },
          annotations: {
            message: |||
              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s failed to run 2 consecutive compactions.
            ||| % $._config,
          },
        },
        {
          // Alert if the compactor has not uploaded anything in the last 24h.
          alert: 'CortexCompactorHasNotUploadedBlocks',
          'for': '15m',
          expr: |||
            (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} > 60 * 60 * 24)
            and
            (thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} > 0)
          ||| % $._config.job_names,
          labels: {
            severity: 'critical',
          },
          annotations: {
            message: |||
              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.
            ||| % $._config,
          },
        },
        {
          // Alert if the compactor has not uploaded anything since its start.
          alert: 'CortexCompactorHasNotUploadedBlocks',
          'for': '24h',
          expr: |||
            thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} == 0
          ||| % $._config.job_names,
          labels: {
            severity: 'critical',
          },
          annotations: {
            message: |||
              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.
            ||| % $._config,
          },
        },
        {
          // Alert if compactor are not able to update the visit-marker.
          alert: 'CortexCompactorBlockVisitMarkerIsFailing',
          'for': '2h',
          expr: |||
            sum(increase(cortex_compactor_block_visit_marker_write_failed{job=~".+/%(compactor)s"}[2h]))>0
          ||| % $._config.job_names,
          labels: {
            severity: 'critical'
          },
          annotations: {
            message: |||
              Cortex compactors are not able to update the visit marker, double check logs to see what is happening
            |||
          }
        }
      ],
    },
  ],
}