diff --git a/README.md b/README.md index 14c210042..de0370d1f 100644 --- a/README.md +++ b/README.md @@ -584,7 +584,12 @@ Cutover can be done atomically with multiple PgDog containers because `RELOAD` d 📘 **[Metrics](https://docs.pgdog.dev/features/metrics/)** PgDog exposes both the standard PgBouncer-style admin database, an OpenMetrics endpoint and can push metrics to an OTEL endpoint. The admin database isn't 100% compatible, -so we recommend you use either OpenMetrics or OTEL ingestion for monitoring. Example Datadog configuration and dashboard are [included](examples/datadog). +so we recommend you use either OpenMetrics or OTEL ingestion for monitoring. + +We include two examples: + +- [Datadog configuration and dashboard](examples/datadog) +- [Graphana + Prometheus configuration and dashboard](examples/grafana_prometheus) ## Running PgDog locally diff --git a/examples/grafana_prometheus/README.md b/examples/grafana_prometheus/README.md new file mode 100644 index 000000000..fe0008164 --- /dev/null +++ b/examples/grafana_prometheus/README.md @@ -0,0 +1,32 @@ +# Grafana + Prometheus + +Pushes PgDog metrics to Prometheus over OTLP and visualizes them in Grafana. + +``` +pgdog --OTLP push (every 5s)--> prometheus <-- grafana +``` + +## Push, not scrape + +PgDog's OTEL exporter (`[otel]` block in `pgdog.toml`) POSTs OTLP JSON to +Prometheus's built-in OTLP receiver, enabled with `--web.enable-otlp-receiver` +(see `docker-compose.metrics.yml`). `prometheus.yml` has no scrape jobs — +Prometheus only ingests what PgDog pushes. + +Metric names are prefixed with `pgdog_` (via `namespace` in `[otel]`); OTLP +attributes become Prometheus labels. + +## Grafana provisioning + +`provisioning/grafana/` is mounted into `/etc/grafana/provisioning/`, so the +Prometheus datasource and the PgDog dashboard show up automatically on first +boot. + +## Running + +```sh +docker compose up +``` + +- Prometheus — http://127.0.0.1:9090 +- Grafana — http://127.0.0.1:3000 (admin / admin), _PgDog_ folder diff --git a/examples/grafana_prometheus/docker-compose.metrics.yml b/examples/grafana_prometheus/docker-compose.metrics.yml new file mode 100644 index 000000000..045601b87 --- /dev/null +++ b/examples/grafana_prometheus/docker-compose.metrics.yml @@ -0,0 +1,31 @@ +services: + prometheus: + image: prom/prometheus:latest + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-otlp-receiver + volumes: + - ./provisioning/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + ports: + - 9090:9090 + + grafana: + image: grafana/grafana:latest + depends_on: + - prometheus + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + volumes: + - ./provisioning/grafana/datasources:/etc/grafana/provisioning/datasources + - ./provisioning/grafana/dashboards:/etc/grafana/provisioning/dashboards + - grafana_data:/var/lib/grafana + ports: + - 3000:3000 + +volumes: + prometheus_data: + grafana_data: diff --git a/examples/grafana_prometheus/docker-compose.yml b/examples/grafana_prometheus/docker-compose.yml new file mode 100644 index 000000000..c7e432a71 --- /dev/null +++ b/examples/grafana_prometheus/docker-compose.yml @@ -0,0 +1,45 @@ +include: + - ./docker-compose.metrics.yml + +services: + db0: + image: postgres:17 + environment: + POSTGRES_PASSWORD: postgres + ports: + - 6000:5432 + volumes: + - shard_0:/var/lib/postgresql/data + db1: + image: postgres:17 + environment: + POSTGRES_PASSWORD: postgres + ports: + - 6001:5432 + volumes: + - shard_1:/var/lib/postgresql/data + db2: + image: postgres:17 + environment: + POSTGRES_PASSWORD: postgres + ports: + - 6002:5432 + volumes: + - shard_2:/var/lib/postgresql/data + + pgdog: + image: ghcr.io/pgdogdev/pgdog:main + depends_on: + - db0 + - db1 + - db2 + volumes: + - ./pgdog.toml:/pgdog/pgdog.toml + - ./users.toml:/pgdog/users.toml + ports: + - 6432:6432 + +volumes: + shard_0: + shard_1: + shard_2: diff --git a/examples/grafana_prometheus/pgdog.toml b/examples/grafana_prometheus/pgdog.toml new file mode 100644 index 000000000..f8f365523 --- /dev/null +++ b/examples/grafana_prometheus/pgdog.toml @@ -0,0 +1,29 @@ +# +# PgDog configuration. +# + +[general] +host = "0.0.0.0" + +[otel] +endpoint = "http://prometheus:9090/api/v1/otlp/v1/metrics" +namespace = "pgdog_" +push_interval = 5000 + +[[databases]] +name = "postgres" +host = "db0" +port = 5432 +shard = 0 + +[[databases]] +name = "postgres" +host = "db1" +port = 5432 +shard = 1 + +[[databases]] +name = "postgres" +host = "db2" +port = 5432 +shard = 2 diff --git a/examples/grafana_prometheus/provisioning/grafana/dashboards/dashboards.yml b/examples/grafana_prometheus/provisioning/grafana/dashboards/dashboards.yml new file mode 100644 index 000000000..883e708fe --- /dev/null +++ b/examples/grafana_prometheus/provisioning/grafana/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: PgDog + orgId: 1 + folder: PgDog + type: file + disableDeletion: false + editable: true + updateIntervalSeconds: 30 + options: + path: /etc/grafana/provisioning/dashboards diff --git a/examples/grafana_prometheus/provisioning/grafana/dashboards/pgdog.json b/examples/grafana_prometheus/provisioning/grafana/dashboards/pgdog.json new file mode 100644 index 000000000..51b252af0 --- /dev/null +++ b/examples/grafana_prometheus/provisioning/grafana/dashboards/pgdog.json @@ -0,0 +1,262 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "type": "gauge", + "title": "Locked %", + "description": "Percentage of connected clients currently locked (pinned) to a server.", + "id": 9, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(pgdog_clients_locked_ratio) / clamp_min(sum(pgdog_clients_ratio), 1)", + "legendFormat": "locked", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.25 }, + { "color": "red", "value": 0.5 } + ] + } + } + } + }, + { + "type": "stat", + "title": "Connected clients", + "id": 1, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(pgdog_clients_ratio)", + "legendFormat": "clients", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "type": "stat", + "title": "Clients waiting for a server", + "id": 2, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(pgdog_cl_waiting_ratio)", + "legendFormat": "waiting", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + } + } + }, + { + "type": "stat", + "title": "Max wait (seconds)", + "id": 3, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "max(pgdog_maxwait_seconds)", + "legendFormat": "maxwait", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { "unit": "s" } + } + }, + { + "type": "stat", + "title": "Query cache hit ratio", + "id": 4, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(pgdog_query_cache_hits_total[1m])) / clamp_min(sum(rate(pgdog_query_cache_hits_total[1m])) + sum(rate(pgdog_query_cache_misses_total[1m])), 1)", + "legendFormat": "hit ratio", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { "unit": "percentunit", "min": 0, "max": 1 } + } + }, + { + "type": "timeseries", + "title": "Server connections by shard", + "id": 5, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (shard) (pgdog_sv_active_ratio)", + "legendFormat": "shard {{shard}} active", + "refId": "A" + }, + { + "expr": "sum by (shard) (pgdog_sv_idle_ratio)", + "legendFormat": "shard {{shard}} idle", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "stacking": { "mode": "none" } + } + } + } + }, + { + "type": "timeseries", + "title": "Queries per second", + "id": 6, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (shard) (rate(pgdog_query_count_total[1m]))", + "legendFormat": "shard {{shard}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "qps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10 + } + } + } + }, + { + "type": "timeseries", + "title": "Transactions per second", + "id": 7, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (shard) (rate(pgdog_xact_count_total[1m]))", + "legendFormat": "shard {{shard}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10 + } + } + } + }, + { + "type": "timeseries", + "title": "Query cache hits vs misses", + "id": 8, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(pgdog_query_cache_hits_total[1m]))", + "legendFormat": "hits", + "refId": "A" + }, + { + "expr": "sum(rate(pgdog_query_cache_misses_total[1m]))", + "legendFormat": "misses", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10 + } + } + } + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["pgdog"], + "templating": { "list": [] }, + "time": { "from": "now-15m", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "PgDog", + "uid": "pgdog-overview", + "version": 1, + "weekStart": "" +} diff --git a/examples/grafana_prometheus/provisioning/grafana/datasources/prometheus.yml b/examples/grafana_prometheus/provisioning/grafana/datasources/prometheus.yml new file mode 100644 index 000000000..0b304bc91 --- /dev/null +++ b/examples/grafana_prometheus/provisioning/grafana/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/examples/grafana_prometheus/provisioning/prometheus/prometheus.yml b/examples/grafana_prometheus/provisioning/prometheus/prometheus.yml new file mode 100644 index 000000000..f2514e095 --- /dev/null +++ b/examples/grafana_prometheus/provisioning/prometheus/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +# Metrics are pushed to Prometheus by PgDog via OTLP +# (see --web.enable-otlp-receiver on the prometheus service). +scrape_configs: [] diff --git a/examples/grafana_prometheus/users.toml b/examples/grafana_prometheus/users.toml new file mode 100644 index 000000000..e01e21817 --- /dev/null +++ b/examples/grafana_prometheus/users.toml @@ -0,0 +1,8 @@ +# +# Users and passwords. +# + +[[users]] +name = "postgres" +database = "postgres" +password = "postgres"