From fd769183de2eb527e9c2c0b6411d8adc48e6f586 Mon Sep 17 00:00:00 2001 From: Kennan Hunter Date: Fri, 31 Jul 2026 00:05:22 -0400 Subject: [PATCH 1/3] add grafana+prometheus example --- Cargo.lock | 9 + Cargo.toml | 1 + examples/grafana_prometheus/README.md | 36 +++ .../docker-compose.metrics.yml | 31 +++ .../grafana_prometheus/docker-compose.yml | 56 ++++ examples/grafana_prometheus/pgdog.toml | 35 +++ .../grafana/dashboards/dashboards.yml | 12 + .../grafana/dashboards/pgdog.json | 262 ++++++++++++++++++ .../grafana/datasources/prometheus.yml | 10 + .../provisioning/prometheus/prometheus.yml | 7 + .../grafana_prometheus/synthetic/Cargo.toml | 9 + .../grafana_prometheus/synthetic/Dockerfile | 13 + .../grafana_prometheus/synthetic/src/main.rs | 205 ++++++++++++++ examples/grafana_prometheus/users.toml | 8 + 14 files changed, 694 insertions(+) create mode 100644 examples/grafana_prometheus/README.md create mode 100644 examples/grafana_prometheus/docker-compose.metrics.yml create mode 100644 examples/grafana_prometheus/docker-compose.yml create mode 100644 examples/grafana_prometheus/pgdog.toml create mode 100644 examples/grafana_prometheus/provisioning/grafana/dashboards/dashboards.yml create mode 100644 examples/grafana_prometheus/provisioning/grafana/dashboards/pgdog.json create mode 100644 examples/grafana_prometheus/provisioning/grafana/datasources/prometheus.yml create mode 100644 examples/grafana_prometheus/provisioning/prometheus/prometheus.yml create mode 100644 examples/grafana_prometheus/synthetic/Cargo.toml create mode 100644 examples/grafana_prometheus/synthetic/Dockerfile create mode 100644 examples/grafana_prometheus/synthetic/src/main.rs create mode 100644 examples/grafana_prometheus/users.toml diff --git a/Cargo.lock b/Cargo.lock index 8402aecd3..42c623acb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5134,6 +5134,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "synthetic" +version = "0.1.0" +dependencies = [ + "rand 0.8.6", + "tokio", + "tokio-postgres", +] + [[package]] name = "system-configuration" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 3db828a23..4cdcfa22d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" exclude = ["fuzz"] members = [ "examples/demo", + "examples/grafana_prometheus/synthetic", "integration/rust", "pgdog", "pgdog-config", diff --git a/examples/grafana_prometheus/README.md b/examples/grafana_prometheus/README.md new file mode 100644 index 000000000..621b1bcf7 --- /dev/null +++ b/examples/grafana_prometheus/README.md @@ -0,0 +1,36 @@ +# Grafana + Prometheus + +Pushes PgDog metrics to Prometheus over OTLP and visualizes them in Grafana. + +``` +pgdog --OTLP push (every 5s)--> prometheus <-- grafana +``` + +## Push, not scrape + +PgDog's OTEL exporter (`[otel]` block in `pgdog.toml`) POSTs OTLP JSON to +Prometheus's built-in OTLP receiver, enabled with `--web.enable-otlp-receiver` +(see `docker-compose.metrics.yml`). `prometheus.yml` has no scrape jobs — +Prometheus only ingests what PgDog pushes. + +Metric names are prefixed with `pgdog_` (via `namespace` in `[otel]`); OTLP +attributes become Prometheus labels. + +## Grafana provisioning + +`provisioning/grafana/` is mounted into `/etc/grafana/provisioning/`, so the +Prometheus datasource and the PgDog dashboard show up automatically on first +boot. + +## Running + +```sh +docker compose up +``` + +- Prometheus — http://127.0.0.1:9090 +- Grafana — http://127.0.0.1:3000 (admin / admin), _PgDog_ folder + +## Synthetic + +We have included a synthetic workload for you to experiment with. It emulates a highly concurrent, lock heavy, read and write workload. This is also a useful reference for how to containerize and link a workload to pgdog as seen in [docker-compose.yml](./docker-compose.yml) and [the synthetic dockerfile](./synthetic/Dockerfile). diff --git a/examples/grafana_prometheus/docker-compose.metrics.yml b/examples/grafana_prometheus/docker-compose.metrics.yml new file mode 100644 index 000000000..045601b87 --- /dev/null +++ b/examples/grafana_prometheus/docker-compose.metrics.yml @@ -0,0 +1,31 @@ +services: + prometheus: + image: prom/prometheus:latest + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-otlp-receiver + volumes: + - ./provisioning/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + ports: + - 9090:9090 + + grafana: + image: grafana/grafana:latest + depends_on: + - prometheus + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + volumes: + - ./provisioning/grafana/datasources:/etc/grafana/provisioning/datasources + - ./provisioning/grafana/dashboards:/etc/grafana/provisioning/dashboards + - grafana_data:/var/lib/grafana + ports: + - 3000:3000 + +volumes: + prometheus_data: + grafana_data: diff --git a/examples/grafana_prometheus/docker-compose.yml b/examples/grafana_prometheus/docker-compose.yml new file mode 100644 index 000000000..a63f4e4b2 --- /dev/null +++ b/examples/grafana_prometheus/docker-compose.yml @@ -0,0 +1,56 @@ +include: + - ./docker-compose.metrics.yml + +services: + db0: + image: postgres:17 + environment: + POSTGRES_PASSWORD: postgres + ports: + - 6000:5432 + volumes: + - shard_0:/var/lib/postgresql/data + db1: + image: postgres:17 + environment: + POSTGRES_PASSWORD: postgres + ports: + - 6001:5432 + volumes: + - shard_1:/var/lib/postgresql/data + db2: + image: postgres:17 + environment: + POSTGRES_PASSWORD: postgres + ports: + - 6002:5432 + volumes: + - shard_2:/var/lib/postgresql/data + + pgdog: + image: ghcr.io/pgdogdev/pgdog:main + depends_on: + - db0 + - db1 + - db2 + volumes: + - ./pgdog.toml:/pgdog/pgdog.toml + - ./users.toml:/pgdog/users.toml + ports: + - 6432:6432 + + synthetic: + build: ./synthetic + depends_on: + - pgdog + environment: + PG_URL: postgres://postgres:postgres@pgdog:6432/postgres + CONCURRENCY: "10" + LOCK_RATE: "0.05" + TX_RATE: "0.05" + KEY_SPACE: "10000" + +volumes: + shard_0: + shard_1: + shard_2: diff --git a/examples/grafana_prometheus/pgdog.toml b/examples/grafana_prometheus/pgdog.toml new file mode 100644 index 000000000..c52bb3e50 --- /dev/null +++ b/examples/grafana_prometheus/pgdog.toml @@ -0,0 +1,35 @@ +# +# PgDog configuration. +# + +[general] +host = "0.0.0.0" + +[otel] +endpoint = "http://prometheus:9090/api/v1/otlp/v1/metrics" +namespace = "pgdog_" +push_interval = 5000 + +[[databases]] +name = "postgres" +host = "db0" +port = 5432 +shard = 0 + +[[databases]] +name = "postgres" +host = "db1" +port = 5432 +shard = 1 + +[[databases]] +name = "postgres" +host = "db2" +port = 5432 +shard = 2 + +[[sharded_tables]] +name = "kv" +column = "id" +data_type = "bigint" +database = "postgres" diff --git a/examples/grafana_prometheus/provisioning/grafana/dashboards/dashboards.yml b/examples/grafana_prometheus/provisioning/grafana/dashboards/dashboards.yml new file mode 100644 index 000000000..883e708fe --- /dev/null +++ b/examples/grafana_prometheus/provisioning/grafana/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: PgDog + orgId: 1 + folder: PgDog + type: file + disableDeletion: false + editable: true + updateIntervalSeconds: 30 + options: + path: /etc/grafana/provisioning/dashboards diff --git a/examples/grafana_prometheus/provisioning/grafana/dashboards/pgdog.json b/examples/grafana_prometheus/provisioning/grafana/dashboards/pgdog.json new file mode 100644 index 000000000..51b252af0 --- /dev/null +++ b/examples/grafana_prometheus/provisioning/grafana/dashboards/pgdog.json @@ -0,0 +1,262 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "type": "gauge", + "title": "Locked %", + "description": "Percentage of connected clients currently locked (pinned) to a server.", + "id": 9, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(pgdog_clients_locked_ratio) / clamp_min(sum(pgdog_clients_ratio), 1)", + "legendFormat": "locked", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.25 }, + { "color": "red", "value": 0.5 } + ] + } + } + } + }, + { + "type": "stat", + "title": "Connected clients", + "id": 1, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(pgdog_clients_ratio)", + "legendFormat": "clients", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "type": "stat", + "title": "Clients waiting for a server", + "id": 2, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(pgdog_cl_waiting_ratio)", + "legendFormat": "waiting", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + } + } + }, + { + "type": "stat", + "title": "Max wait (seconds)", + "id": 3, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "max(pgdog_maxwait_seconds)", + "legendFormat": "maxwait", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { "unit": "s" } + } + }, + { + "type": "stat", + "title": "Query cache hit ratio", + "id": 4, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(pgdog_query_cache_hits_total[1m])) / clamp_min(sum(rate(pgdog_query_cache_hits_total[1m])) + sum(rate(pgdog_query_cache_misses_total[1m])), 1)", + "legendFormat": "hit ratio", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { "unit": "percentunit", "min": 0, "max": 1 } + } + }, + { + "type": "timeseries", + "title": "Server connections by shard", + "id": 5, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (shard) (pgdog_sv_active_ratio)", + "legendFormat": "shard {{shard}} active", + "refId": "A" + }, + { + "expr": "sum by (shard) (pgdog_sv_idle_ratio)", + "legendFormat": "shard {{shard}} idle", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "stacking": { "mode": "none" } + } + } + } + }, + { + "type": "timeseries", + "title": "Queries per second", + "id": 6, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (shard) (rate(pgdog_query_count_total[1m]))", + "legendFormat": "shard {{shard}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "qps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10 + } + } + } + }, + { + "type": "timeseries", + "title": "Transactions per second", + "id": 7, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (shard) (rate(pgdog_xact_count_total[1m]))", + "legendFormat": "shard {{shard}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10 + } + } + } + }, + { + "type": "timeseries", + "title": "Query cache hits vs misses", + "id": 8, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(pgdog_query_cache_hits_total[1m]))", + "legendFormat": "hits", + "refId": "A" + }, + { + "expr": "sum(rate(pgdog_query_cache_misses_total[1m]))", + "legendFormat": "misses", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10 + } + } + } + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["pgdog"], + "templating": { "list": [] }, + "time": { "from": "now-15m", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "PgDog", + "uid": "pgdog-overview", + "version": 1, + "weekStart": "" +} diff --git a/examples/grafana_prometheus/provisioning/grafana/datasources/prometheus.yml b/examples/grafana_prometheus/provisioning/grafana/datasources/prometheus.yml new file mode 100644 index 000000000..0b304bc91 --- /dev/null +++ b/examples/grafana_prometheus/provisioning/grafana/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/examples/grafana_prometheus/provisioning/prometheus/prometheus.yml b/examples/grafana_prometheus/provisioning/prometheus/prometheus.yml new file mode 100644 index 000000000..f2514e095 --- /dev/null +++ b/examples/grafana_prometheus/provisioning/prometheus/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +# Metrics are pushed to Prometheus by PgDog via OTLP +# (see --web.enable-otlp-receiver on the prometheus service). +scrape_configs: [] diff --git a/examples/grafana_prometheus/synthetic/Cargo.toml b/examples/grafana_prometheus/synthetic/Cargo.toml new file mode 100644 index 000000000..d71c2b206 --- /dev/null +++ b/examples/grafana_prometheus/synthetic/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "synthetic" +version = "0.1.0" +edition = "2024" + +[dependencies] +tokio = { version = "1", features = ["full"] } +tokio-postgres = "0.7" +rand = "0.8" diff --git a/examples/grafana_prometheus/synthetic/Dockerfile b/examples/grafana_prometheus/synthetic/Dockerfile new file mode 100644 index 000000000..e8c4a9cb4 --- /dev/null +++ b/examples/grafana_prometheus/synthetic/Dockerfile @@ -0,0 +1,13 @@ +FROM rust:1-slim AS builder +WORKDIR /app +# We copy like this to not invalidate cache on Dockerfile changes. +# You can get similar behavior with `COPY . .` paired with a .dockerignore +COPY Cargo.toml ./ +COPY src ./src +RUN cargo build --release + +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/target/release/synthetic /usr/local/bin/synthetic +CMD ["synthetic"] diff --git a/examples/grafana_prometheus/synthetic/src/main.rs b/examples/grafana_prometheus/synthetic/src/main.rs new file mode 100644 index 000000000..ed171195c --- /dev/null +++ b/examples/grafana_prometheus/synthetic/src/main.rs @@ -0,0 +1,205 @@ +use std::env; +use std::str::FromStr; +use std::time::Duration; + +use rand::Rng; +use tokio::signal; +use tokio::task::JoinSet; +use tokio::time::sleep; +use tokio_postgres::{Client, NoTls}; + +fn env_var(name: &str, default: T) -> T { + env::var(name) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +#[tokio::main] +async fn main() { + let pg_url = env::var("PG_URL") + .unwrap_or_else(|_| "postgres://postgres:postgres@pgdog:6432/postgres".into()); + let concurrency: usize = env_var("CONCURRENCY", 10usize); + let lock_rate: f64 = env_var("LOCK_RATE", 0.05); + let tx_rate: f64 = env_var("TX_RATE", 0.05); + let key_space: i64 = env_var("KEY_SPACE", 10_000i64); + + wait_for_pgdog(&pg_url).await; + init(&pg_url).await; + println!( + "synthetic: {} workers, lock_rate={}, tx_rate={}, key_space={} -> {}", + concurrency, lock_rate, tx_rate, key_space, pg_url + ); + + let mut workers = JoinSet::new(); + for id in 0..concurrency { + let url = pg_url.clone(); + workers.spawn(async move { + worker(id, url, lock_rate, tx_rate, key_space).await; + }); + } + + tokio::select! { + _ = drain(&mut workers) => {} + _ = signal::ctrl_c() => { + println!("\nctrl-c received, shutting down"); + workers.shutdown().await; + } + } +} + +async fn drain(workers: &mut JoinSet<()>) { + while workers.join_next().await.is_some() {} +} + +async fn connect(url: &str) -> Client { + let (client, conn) = tokio_postgres::connect(url, NoTls) + .await + .expect("connect to pgdog"); + tokio::spawn(async move { + if let Err(err) = conn.await { + eprintln!("connection error: {err}"); + } + }); + client +} + +async fn wait_for_pgdog(url: &str) { + for attempt in 0..60 { + if let Ok((client, conn)) = tokio_postgres::connect(url, NoTls).await { + let handle = tokio::spawn(conn); + if client.simple_query("SELECT 1").await.is_ok() { + drop(client); + let _ = handle.await; + return; + } + } + if attempt == 0 { + println!("waiting for pgdog..."); + } + sleep(Duration::from_secs(1)).await; + } + panic!("pgdog never became ready"); +} + +async fn init(url: &str) { + let client = connect(url).await; + client + .batch_execute( + "CREATE TABLE IF NOT EXISTS kv ( + id bigint PRIMARY KEY, + value text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() + )", + ) + .await + .expect("create kv table"); +} + +async fn worker(id: usize, url: String, lock_rate: f64, tx_rate: f64, key_space: i64) { + let mut client = connect(&url).await; + loop { + let roll: f64 = rand::thread_rng().r#gen(); + let result = if roll < lock_rate { + advisory_lock_once(&client, key_space).await + } else if roll < lock_rate + tx_rate { + transaction_once(&client, key_space).await + } else if roll < lock_rate + tx_rate + 0.4 { + upsert_once(&client, key_space).await + } else { + read_once(&client, key_space).await + }; + if let Err(err) = result { + eprintln!("worker {id}: {err:?}"); + sleep(Duration::from_millis(500)).await; + client = connect(&url).await; + } + } +} + +fn random_id(key_space: i64) -> i64 { + rand::thread_rng().gen_range(0..key_space) +} + +fn random_value() -> String { + format!("{:016x}", rand::thread_rng().r#gen::()) +} + +async fn read_once(client: &Client, key_space: i64) -> Result<(), tokio_postgres::Error> { + let id = random_id(key_space); + client + .simple_query(&format!("SELECT value FROM kv WHERE id = {id}")) + .await?; + Ok(()) +} + +async fn upsert_once(client: &Client, key_space: i64) -> Result<(), tokio_postgres::Error> { + let id = random_id(key_space); + let value = random_value(); + client + .simple_query(&format!( + "INSERT INTO kv (id, value) VALUES ({id}, '{value}') \ + ON CONFLICT (id) DO UPDATE \ + SET value = EXCLUDED.value, updated_at = now()" + )) + .await?; + Ok(()) +} + +async fn advisory_lock_once(client: &Client, key_space: i64) -> Result<(), tokio_postgres::Error> { + let key = random_id(key_space); + + // Session-level advisory lock. PgDog pins the client to a single server + // for as long as the lock is held (see regex_parser.rs — `pg_advisory_lock` + // triggers pinning), which is what surfaces on the Locked % gauge. + client + .simple_query(&format!("SELECT pg_advisory_lock({key})")) + .await?; + + let work = advisory_lock_work(client).await; + + // Always release the lock, even if the work errored. If the release itself + // fails, the server-side session will still clean up on disconnect. + let _ = client + .simple_query(&format!("SELECT pg_advisory_unlock({key})")) + .await; + + work +} + +async fn advisory_lock_work(client: &Client) -> Result<(), tokio_postgres::Error> { + let hold_ms = 15 + rand::thread_rng().gen_range(0..25); + + sleep(Duration::from_millis(hold_ms)).await; + + // Shard-neutral query — the client is pinned to whichever shard the + // advisory lock landed on. + client.simple_query("SELECT 1").await?; + + Ok(()) +} + +async fn transaction_once(client: &Client, key_space: i64) -> Result<(), tokio_postgres::Error> { + let id = random_id(key_space); + let value = random_value(); + + client.simple_query("BEGIN").await?; + client + .simple_query(&format!( + "INSERT INTO kv (id, value) VALUES ({id}, '{value}') \ + ON CONFLICT (id) DO UPDATE \ + SET value = EXCLUDED.value, updated_at = now()" + )) + .await?; + + let hold_ms = 15 + rand::thread_rng().gen_range(0..25); + + sleep(Duration::from_millis(hold_ms)).await; + + client + .simple_query(&format!("SELECT value FROM kv WHERE id = {id}")) + .await?; + client.simple_query("COMMIT").await?; + + Ok(()) +} diff --git a/examples/grafana_prometheus/users.toml b/examples/grafana_prometheus/users.toml new file mode 100644 index 000000000..e01e21817 --- /dev/null +++ b/examples/grafana_prometheus/users.toml @@ -0,0 +1,8 @@ +# +# Users and passwords. +# + +[[users]] +name = "postgres" +database = "postgres" +password = "postgres" From 83a4bf0534b1523e5ac18d9731159eaae0a46fe1 Mon Sep 17 00:00:00 2001 From: Kennan Hunter Date: Mon, 3 Aug 2026 00:29:20 -0400 Subject: [PATCH 2/3] remove synthetic load from grafana prometheus example --- Cargo.lock | 9 - Cargo.toml | 1 - examples/grafana_prometheus/README.md | 4 - .../grafana_prometheus/docker-compose.yml | 11 - examples/grafana_prometheus/pgdog.toml | 6 - .../grafana_prometheus/synthetic/Cargo.toml | 9 - .../grafana_prometheus/synthetic/Dockerfile | 13 -- .../grafana_prometheus/synthetic/src/main.rs | 205 ------------------ 8 files changed, 258 deletions(-) delete mode 100644 examples/grafana_prometheus/synthetic/Cargo.toml delete mode 100644 examples/grafana_prometheus/synthetic/Dockerfile delete mode 100644 examples/grafana_prometheus/synthetic/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 42c623acb..8402aecd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5134,15 +5134,6 @@ dependencies = [ "syn 2.0.118", ] -[[package]] -name = "synthetic" -version = "0.1.0" -dependencies = [ - "rand 0.8.6", - "tokio", - "tokio-postgres", -] - [[package]] name = "system-configuration" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 4cdcfa22d..3db828a23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,6 @@ resolver = "2" exclude = ["fuzz"] members = [ "examples/demo", - "examples/grafana_prometheus/synthetic", "integration/rust", "pgdog", "pgdog-config", diff --git a/examples/grafana_prometheus/README.md b/examples/grafana_prometheus/README.md index 621b1bcf7..fe0008164 100644 --- a/examples/grafana_prometheus/README.md +++ b/examples/grafana_prometheus/README.md @@ -30,7 +30,3 @@ docker compose up - Prometheus — http://127.0.0.1:9090 - Grafana — http://127.0.0.1:3000 (admin / admin), _PgDog_ folder - -## Synthetic - -We have included a synthetic workload for you to experiment with. It emulates a highly concurrent, lock heavy, read and write workload. This is also a useful reference for how to containerize and link a workload to pgdog as seen in [docker-compose.yml](./docker-compose.yml) and [the synthetic dockerfile](./synthetic/Dockerfile). diff --git a/examples/grafana_prometheus/docker-compose.yml b/examples/grafana_prometheus/docker-compose.yml index a63f4e4b2..c7e432a71 100644 --- a/examples/grafana_prometheus/docker-compose.yml +++ b/examples/grafana_prometheus/docker-compose.yml @@ -39,17 +39,6 @@ services: ports: - 6432:6432 - synthetic: - build: ./synthetic - depends_on: - - pgdog - environment: - PG_URL: postgres://postgres:postgres@pgdog:6432/postgres - CONCURRENCY: "10" - LOCK_RATE: "0.05" - TX_RATE: "0.05" - KEY_SPACE: "10000" - volumes: shard_0: shard_1: diff --git a/examples/grafana_prometheus/pgdog.toml b/examples/grafana_prometheus/pgdog.toml index c52bb3e50..f8f365523 100644 --- a/examples/grafana_prometheus/pgdog.toml +++ b/examples/grafana_prometheus/pgdog.toml @@ -27,9 +27,3 @@ name = "postgres" host = "db2" port = 5432 shard = 2 - -[[sharded_tables]] -name = "kv" -column = "id" -data_type = "bigint" -database = "postgres" diff --git a/examples/grafana_prometheus/synthetic/Cargo.toml b/examples/grafana_prometheus/synthetic/Cargo.toml deleted file mode 100644 index d71c2b206..000000000 --- a/examples/grafana_prometheus/synthetic/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "synthetic" -version = "0.1.0" -edition = "2024" - -[dependencies] -tokio = { version = "1", features = ["full"] } -tokio-postgres = "0.7" -rand = "0.8" diff --git a/examples/grafana_prometheus/synthetic/Dockerfile b/examples/grafana_prometheus/synthetic/Dockerfile deleted file mode 100644 index e8c4a9cb4..000000000 --- a/examples/grafana_prometheus/synthetic/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM rust:1-slim AS builder -WORKDIR /app -# We copy like this to not invalidate cache on Dockerfile changes. -# You can get similar behavior with `COPY . .` paired with a .dockerignore -COPY Cargo.toml ./ -COPY src ./src -RUN cargo build --release - -FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \ - && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/target/release/synthetic /usr/local/bin/synthetic -CMD ["synthetic"] diff --git a/examples/grafana_prometheus/synthetic/src/main.rs b/examples/grafana_prometheus/synthetic/src/main.rs deleted file mode 100644 index ed171195c..000000000 --- a/examples/grafana_prometheus/synthetic/src/main.rs +++ /dev/null @@ -1,205 +0,0 @@ -use std::env; -use std::str::FromStr; -use std::time::Duration; - -use rand::Rng; -use tokio::signal; -use tokio::task::JoinSet; -use tokio::time::sleep; -use tokio_postgres::{Client, NoTls}; - -fn env_var(name: &str, default: T) -> T { - env::var(name) - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(default) -} - -#[tokio::main] -async fn main() { - let pg_url = env::var("PG_URL") - .unwrap_or_else(|_| "postgres://postgres:postgres@pgdog:6432/postgres".into()); - let concurrency: usize = env_var("CONCURRENCY", 10usize); - let lock_rate: f64 = env_var("LOCK_RATE", 0.05); - let tx_rate: f64 = env_var("TX_RATE", 0.05); - let key_space: i64 = env_var("KEY_SPACE", 10_000i64); - - wait_for_pgdog(&pg_url).await; - init(&pg_url).await; - println!( - "synthetic: {} workers, lock_rate={}, tx_rate={}, key_space={} -> {}", - concurrency, lock_rate, tx_rate, key_space, pg_url - ); - - let mut workers = JoinSet::new(); - for id in 0..concurrency { - let url = pg_url.clone(); - workers.spawn(async move { - worker(id, url, lock_rate, tx_rate, key_space).await; - }); - } - - tokio::select! { - _ = drain(&mut workers) => {} - _ = signal::ctrl_c() => { - println!("\nctrl-c received, shutting down"); - workers.shutdown().await; - } - } -} - -async fn drain(workers: &mut JoinSet<()>) { - while workers.join_next().await.is_some() {} -} - -async fn connect(url: &str) -> Client { - let (client, conn) = tokio_postgres::connect(url, NoTls) - .await - .expect("connect to pgdog"); - tokio::spawn(async move { - if let Err(err) = conn.await { - eprintln!("connection error: {err}"); - } - }); - client -} - -async fn wait_for_pgdog(url: &str) { - for attempt in 0..60 { - if let Ok((client, conn)) = tokio_postgres::connect(url, NoTls).await { - let handle = tokio::spawn(conn); - if client.simple_query("SELECT 1").await.is_ok() { - drop(client); - let _ = handle.await; - return; - } - } - if attempt == 0 { - println!("waiting for pgdog..."); - } - sleep(Duration::from_secs(1)).await; - } - panic!("pgdog never became ready"); -} - -async fn init(url: &str) { - let client = connect(url).await; - client - .batch_execute( - "CREATE TABLE IF NOT EXISTS kv ( - id bigint PRIMARY KEY, - value text NOT NULL, - updated_at timestamptz NOT NULL DEFAULT now() - )", - ) - .await - .expect("create kv table"); -} - -async fn worker(id: usize, url: String, lock_rate: f64, tx_rate: f64, key_space: i64) { - let mut client = connect(&url).await; - loop { - let roll: f64 = rand::thread_rng().r#gen(); - let result = if roll < lock_rate { - advisory_lock_once(&client, key_space).await - } else if roll < lock_rate + tx_rate { - transaction_once(&client, key_space).await - } else if roll < lock_rate + tx_rate + 0.4 { - upsert_once(&client, key_space).await - } else { - read_once(&client, key_space).await - }; - if let Err(err) = result { - eprintln!("worker {id}: {err:?}"); - sleep(Duration::from_millis(500)).await; - client = connect(&url).await; - } - } -} - -fn random_id(key_space: i64) -> i64 { - rand::thread_rng().gen_range(0..key_space) -} - -fn random_value() -> String { - format!("{:016x}", rand::thread_rng().r#gen::()) -} - -async fn read_once(client: &Client, key_space: i64) -> Result<(), tokio_postgres::Error> { - let id = random_id(key_space); - client - .simple_query(&format!("SELECT value FROM kv WHERE id = {id}")) - .await?; - Ok(()) -} - -async fn upsert_once(client: &Client, key_space: i64) -> Result<(), tokio_postgres::Error> { - let id = random_id(key_space); - let value = random_value(); - client - .simple_query(&format!( - "INSERT INTO kv (id, value) VALUES ({id}, '{value}') \ - ON CONFLICT (id) DO UPDATE \ - SET value = EXCLUDED.value, updated_at = now()" - )) - .await?; - Ok(()) -} - -async fn advisory_lock_once(client: &Client, key_space: i64) -> Result<(), tokio_postgres::Error> { - let key = random_id(key_space); - - // Session-level advisory lock. PgDog pins the client to a single server - // for as long as the lock is held (see regex_parser.rs — `pg_advisory_lock` - // triggers pinning), which is what surfaces on the Locked % gauge. - client - .simple_query(&format!("SELECT pg_advisory_lock({key})")) - .await?; - - let work = advisory_lock_work(client).await; - - // Always release the lock, even if the work errored. If the release itself - // fails, the server-side session will still clean up on disconnect. - let _ = client - .simple_query(&format!("SELECT pg_advisory_unlock({key})")) - .await; - - work -} - -async fn advisory_lock_work(client: &Client) -> Result<(), tokio_postgres::Error> { - let hold_ms = 15 + rand::thread_rng().gen_range(0..25); - - sleep(Duration::from_millis(hold_ms)).await; - - // Shard-neutral query — the client is pinned to whichever shard the - // advisory lock landed on. - client.simple_query("SELECT 1").await?; - - Ok(()) -} - -async fn transaction_once(client: &Client, key_space: i64) -> Result<(), tokio_postgres::Error> { - let id = random_id(key_space); - let value = random_value(); - - client.simple_query("BEGIN").await?; - client - .simple_query(&format!( - "INSERT INTO kv (id, value) VALUES ({id}, '{value}') \ - ON CONFLICT (id) DO UPDATE \ - SET value = EXCLUDED.value, updated_at = now()" - )) - .await?; - - let hold_ms = 15 + rand::thread_rng().gen_range(0..25); - - sleep(Duration::from_millis(hold_ms)).await; - - client - .simple_query(&format!("SELECT value FROM kv WHERE id = {id}")) - .await?; - client.simple_query("COMMIT").await?; - - Ok(()) -} From 3e1f3d2343b935bedf4f244e4fc0b8348d0679b8 Mon Sep 17 00:00:00 2001 From: Kennan Hunter Date: Mon, 3 Aug 2026 11:55:52 -0400 Subject: [PATCH 3/3] update readme with new example --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 14c210042..de0370d1f 100644 --- a/README.md +++ b/README.md @@ -584,7 +584,12 @@ Cutover can be done atomically with multiple PgDog containers because `RELOAD` d 📘 **[Metrics](https://docs.pgdog.dev/features/metrics/)** PgDog exposes both the standard PgBouncer-style admin database, an OpenMetrics endpoint and can push metrics to an OTEL endpoint. The admin database isn't 100% compatible, -so we recommend you use either OpenMetrics or OTEL ingestion for monitoring. Example Datadog configuration and dashboard are [included](examples/datadog). +so we recommend you use either OpenMetrics or OTEL ingestion for monitoring. + +We include two examples: + +- [Datadog configuration and dashboard](examples/datadog) +- [Graphana + Prometheus configuration and dashboard](examples/grafana_prometheus) ## Running PgDog locally