From 408ad97eb79bf8c700c956717d3fce8fe2ddbee2 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:25:35 +0200 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20pm=20bench=20audit=20=E2=80=94=20ven?= =?UTF-8?q?ue=20coverage,=20copy=20accuracy,=20ramp=20exclusion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmarks/pm-api-latency.yml | 26 +++++++++---------- benchmarks/pm-rate-limits.yml | 10 +++---- benchmarks/pm-ws-latency.yml | 7 +++-- harnesses/pm-rate-limits/cmd/script/config.go | 2 +- src/lib/pm-stats.ts | 4 +++ src/lib/pm-venue-context.ts | 2 ++ 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/benchmarks/pm-api-latency.yml b/benchmarks/pm-api-latency.yml index 6dea781f..3db329b5 100644 --- a/benchmarks/pm-api-latency.yml +++ b/benchmarks/pm-api-latency.yml @@ -5,7 +5,7 @@ number: "038" title: Best prediction market API, ranked by live latency and uptime seo_title: "Best prediction market API 2026" seo_description: "Is the Polymarket API up? Live uptime and latency for Polymarket, Kalshi, Limitless, Manifold, Myriad, PredictIt and Smarkets price endpoints." -subtitle: Warm price endpoint latency of seven prediction market venue APIs from three regions, plus a live uptime panel that answers whether each API is up right now. +subtitle: Warm price endpoint latency of eight prediction market venue APIs from three regions, plus a live uptime panel that answers whether each API is up right now. category: Trading status: live @@ -17,11 +17,11 @@ seo_intro: | When the Polymarket API stops answering, the only pages that notice are downdetector clones aggregating complaints. None of them actually probe the API. This benchmark does. A harness in three regions requests the - price endpoint of Polymarket, Kalshi, Limitless, Manifold and Myriad - every 5 seconds over warm connections, records every round trip, and - publishes two things venue status pages do not: a latency leaderboard - ranking the best prediction market API by measured p50, and a per venue - uptime panel computed from the same probes. If you are here to check + price endpoint of Polymarket, Kalshi, Limitless, Manifold, Myriad, + PredictIt, Smarkets and Metaculus every 5 seconds over warm connections, + records every round trip, and publishes two things venue status pages + do not: a latency leaderboard ranking the best prediction market API + by measured p50, and a per venue uptime panel computed from the same probes. If you are here to check whether the Polymarket API is up right now, the uptime panel below is a direct measurement, refreshed continuously, not a complaint counter. If you are choosing which venue API to build on, the leaderboard ranks @@ -46,7 +46,7 @@ abstract: | fastest right now. methodology: - - "Price endpoint per venue. Polymarket: CLOB `/midpoint`. Kalshi: `/markets/{ticker}`. Limitless: `/markets/{slug}`. Manifold: `/v0/market/{id}`. Myriad: `/markets/{slug}`. This is the hot path of real integrations: one market, one quote, polled in a loop." + - "Price endpoint per venue. Polymarket: CLOB `/midpoint`. Kalshi: `/markets/{ticker}`. Limitless: `/markets/{slug}`. Manifold: `/v0/market/{id}`. Myriad: `/markets/{slug}`. PredictIt: `/api/marketdata/markets/{id}` (15s interval, 5 req/min limit). Smarkets: `/v3/markets/{id}/`. Metaculus: `/api2/questions/{id}/`. This is the hot path of real integrations: one market, one quote, polled in a loop." - "Warm probes every 5s (7s on Manifold, whose API sits behind max-age=5 plus stale-while-revalidate=10) from us-east, eu-west and sgp over a keep alive pool. A separate cold connect probe once per minute records the TCP plus TLS handshake a brand new client pays." - "Every sample carries a cache flag read from cf-cache-status, x-cache and age headers. Latency aggregates exclude cache hits: a CDN hit measures the nearest edge, not the venue's API, and including Manifold's residual cache hits would flatter it by design." - "Uptime is the average of a per venue, per region health gauge (pmapi_health) computed by the harness from probe outcomes. 100 percent means every probe cycle in the window succeeded from that region. The 24h and 7d panels below are the live answer to whether each API is up." @@ -59,7 +59,7 @@ methodology: findings: - "{{best_name}} currently has the lowest warm price endpoint latency at {{best_p50}} p50 over the last 24h, making it the fastest prediction market API in the cohort by direct measurement." - - "All five venue APIs typically sit above 99 percent measured uptime; the gaps that do appear are short and regional, which is exactly what a complaint based status page cannot tell you. The uptime panels show the live 24h and 7d figures per venue." + - "All eight venue APIs typically sit above 99 percent measured uptime; the gaps that do appear are short and regional, which is exactly what a complaint based status page cannot tell you. The uptime panels show the live 24h and 7d figures per venue." - "{{name:manifold}} would look faster than it is if cache hits counted: its whole API sits behind a 5 second CDN cache. With cache hits excluded its origin answers at {{p50:manifold}} p50, an honest number a trading bot polling fresh quotes will actually see." - "{{name:myriad}} serves every region from a single US East origin, so its cohort worst p50 of {{p50:myriad}} is mostly geography. Switch the region dimension to us-east to see the API itself rather than the speed of light." - "{{name:polymarket}} answers its midpoint endpoint at {{p50:polymarket}} p50. Because we probe it every 5 seconds from three regions, the uptime panel doubles as a Polymarket API status check that updates continuously instead of waiting for user reports." @@ -242,13 +242,13 @@ providers: tag: "CFTC no-action letter, US political markets only, probed at 15s due to 5 req/min rate limit" formula: "p50 of warm price requests at 15s intervals against /api/marketdata/markets/{id}, successful requests only, 24h window. Rate limit constraint means fewer samples than other venues." queries: - p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="predictit",source="direct",class="price",conn="warm"}[24h])) by (le)) - p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="predictit",source="direct",class="price",conn="warm"}[24h])) by (le)) - p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="predictit",source="direct",class="price",conn="warm"}[24h])) by (le)) - mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="predictit",source="direct",class="price",conn="warm"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="predictit",source="direct",class="price",conn="warm"}[24h])) + p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="predictit",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="predictit",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="predictit",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="predictit",source="direct",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="predictit",source="direct",class="price",conn="warm",cache!="hit"}[24h])) success: clamp_max(sum(rate(pmapi_requests_total{venue="predictit",source="direct",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="predictit",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) sample_size: sum(increase(pmapi_requests_total{venue="predictit",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])) - series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="predictit",source="direct",class="price",conn="warm"}[1h])) by (le)) + series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="predictit",source="direct",class="price",conn="warm",cache!="hit"}[1h])) by (le)) - slug: smarkets name: Smarkets diff --git a/benchmarks/pm-rate-limits.yml b/benchmarks/pm-rate-limits.yml index 5ee0d3d7..b453116a 100644 --- a/benchmarks/pm-rate-limits.yml +++ b/benchmarks/pm-rate-limits.yml @@ -4,8 +4,8 @@ slug: pm-rate-limits number: "037" title: Prediction market API rate limits, tested with a daily ramp seo_title: "Prediction market API rate limits 2026" -seo_description: "Prediction market API rate limits measured live: daily request ramp against Polymarket, Kalshi, Limitless, Manifold, Myriad." -subtitle: Warm latency on book, price and list endpoints of five prediction market venue APIs, plus a daily rate limit ramp that records added latency and throttle onset per request tier. +seo_description: "Prediction market API rate limits measured live: daily request ramp against Polymarket, Kalshi, Limitless, Manifold, Myriad, PredictIt, Smarkets and Metaculus." +subtitle: Warm latency on book, price and list endpoints of eight prediction market venue APIs, plus a daily rate limit ramp that records added latency and throttle onset per request tier. category: Trading status: live @@ -49,13 +49,13 @@ abstract: | second window. methodology: - - "Endpoint classes per venue. Polymarket: CLOB `/book`, `/midpoint`, Gamma `/markets`. Kalshi: `/markets/{ticker}/orderbook`, `/markets/{ticker}`, `/markets`. Limitless: `/markets/{slug}/orderbook`, `/markets/{slug}`, `/markets/active`. Manifold: `/v0/bets`, `/v0/market/{id}`, `/v0/markets`. Myriad: `/markets/{slug}`, `/markets`. Myriad exposes no order book endpoint; that absence is reported, not worked around." + - "Endpoint classes per venue. Polymarket: CLOB `/book`, `/midpoint`, Gamma `/markets`. Kalshi: `/markets/{ticker}/orderbook`, `/markets/{ticker}`, `/markets`. Limitless: `/markets/{slug}/orderbook`, `/markets/{slug}`, `/markets/active`. Manifold: `/v0/bets`, `/v0/market/{id}`, `/v0/markets`. Myriad: `/markets/{slug}`, `/markets`. PredictIt: `/api/marketdata/markets/{id}`, `/api/marketdata/all` (no book endpoint, 15s interval). Smarkets: `/v3/markets/{id}/quotes/`, `/v3/markets/{id}/`, `/v3/events/`. Metaculus: `/api2/questions/{id}/`, `/api2/questions/` (no book or WebSocket). Myriad and Metaculus expose no order book endpoint; that absence is reported, not worked around." - "Pinned market per venue: most liquid, near the money (price between 0.15 and 0.85), expiring more than 24h out. Re pinned daily at 00:00 UTC and immediately on probe_invalid. A degenerate 0.99 book has a latency profile that does not represent real usage." - "Warm probes every 5s (7s on Manifold, whose entire API sits behind max-age=5 plus stale-while-revalidate=10; tighter spacing would only measure the cache). Cold connect probe once per minute with keep alives disabled, TCP plus TLS handshake recorded separately." - "Every sample records a cache flag from cf-cache-status, x-cache and age headers. Kalshi's market list is served from CloudFront with max-age=15, so its list latency measures the edge, not the API. Edge served samples (cache=hit) are excluded from every latency aggregate and reported through the cache label, so no venue ranks on its CDN." - "Outcome classification: ok, timeout (8s), http_4xx, http_5xx, throttled (429), probe_invalid, net_error. Latency aggregates include successful requests only; failures count toward the success rate. probe_invalid covers stale pins, e.g. Limitless serves a CDN cached 400 for four hours once a market expires. Those are our fault, never the venue's." - "Daily ramp: 60s per tier at rising requests per 10s window against the book endpoint, one venue at a time, one region at a time on disjoint UTC hours (02:00 us-east, 04:00 eu-west, 06:00 sgp)." - - "Ramp tier clamps: Polymarket and Kalshi 25/50/100 per 10s (at most 7 percent of Polymarket's documented 1500 per 10s book budget), Manifold 15/30/60 (500 req/min per IP documented), Limitless 10/20/40 (limits undocumented, so conservative). Kalshi stops at the first 429 since its token bucket is documented. Myriad is excluded: its keyless budget is 30 req per 10s and ramping it would only measure our own quota." + - "Ramp tier clamps: Polymarket and Kalshi 25/50/100 per 10s (at most 7 percent of Polymarket's documented 1500 per 10s book budget), Manifold 15/30/60 (500 req/min per IP documented), Limitless 10/20/40 (limits undocumented, so conservative), Smarkets 10/25/50 (limits undocumented), Metaculus 5/10/20 (limits undocumented, forecasting API). Kalshi stops at the first 429 since its token bucket is documented. Myriad is excluded: its keyless budget is 30 req per 10s and ramping it would only measure our own quota. PredictIt is excluded: its documented 5 req/min limit means any ramp tier above 4 req/10s exceeds the budget." - "Ramp abort guard: the run stops the moment throttled plus 5xx responses exceed 1 percent of any 10 second window. The headline ramp metric is p50 latency during the tier minus the warm baseline p50 of the previous hour in the same region, because venues behind Cloudflare queue bursts without ever returning 429, and added latency is the only honest throttle signal there." - "WebSocket panel covers venues with a public market WebSocket, which today means Polymarket only. Kalshi requires authentication for WS, Manifold and Myriad have none, Limitless uses socket.io. Absence is shown, not penalized." - "Myriad's origin is a single region US East deployment behind Heroku. Latency from eu-west and sgp is geography, reported as measured." @@ -143,7 +143,7 @@ metric_panels: - id: ramp_added_latency label: Ramp added latency description: "Worst tier of the daily ramp: p50 latency during the burst minus the warm baseline p50 of the previous hour, same region. Venues that queue under load instead of returning 429 show up here." - metric: 1000 * max(max_over_time(pmapi_ramp_added_latency_seconds{venue!="predictit",venue!="myriad",venue!="metaculus"}[24h])) + metric: 1000 * max(max_over_time(pmapi_ramp_added_latency_seconds{venue!="predictit",venue!="myriad"}[24h])) label_key: venue unit: ms higher_is_better: false diff --git a/benchmarks/pm-ws-latency.yml b/benchmarks/pm-ws-latency.yml index c2774f2e..d5da136f 100644 --- a/benchmarks/pm-ws-latency.yml +++ b/benchmarks/pm-ws-latency.yml @@ -5,7 +5,7 @@ number: "114" title: Polymarket WebSocket latency, live connection and update speed from 3 regions seo_title: "Polymarket WebSocket latency 2026: connection time and update speed" seo_description: "How fast does the Polymarket WebSocket connect and deliver market updates? Live p50 from us-east, eu-west and Singapore measured continuously by OpenChainBench." -subtitle: Connect-to-snapshot latency, inter-message update rate, and trade publication lag on the Polymarket and Kalshi WebSocket feeds, probed from three regions 24 hours a day. +subtitle: Connect-to-snapshot latency, inter-message update rate, and trade publication lag on the Polymarket WebSocket feed (ranked) and Kalshi WebSocket (measured, not ranked), probed from three regions 24 hours a day. category: Trading status: live @@ -48,7 +48,7 @@ methodology: - "Keepalive: a background goroutine sends the literal text PING every 10 seconds and the venue responds with PONG. Sessions that stop receiving data are detected by the reconnect logic on the next PING timeout. Disconnect rate is exported as a counter." - "Pin: the harness targets the same market used by the pm-rate-limits bench (highest 24h volume binary market with last trade price between 0.15 and 0.85), refreshed daily at 00:00 UTC. The WS client re-subscribes automatically when the pin changes." - "Metrics: pmapi_ws_connect_to_snapshot_seconds histogram and pmapi_ws_update_interarrival_seconds histogram, both labeled by venue, region and source. The same pm-rate-limits harness process that probes REST endpoints runs the WS session, so both REST and WS data share one Railway deployment per region. Note on Kalshi: its ticker channel sends incremental price updates only (no state snapshot on subscribe), so the connect-to-snapshot metric measures time to the first price change rather than time to initial data, which depends on market activity." - - "Other venues: Kalshi WS (wss://external-api-ws.kalshi.com) requires RSA-PSS signed headers from a US KYC account and returns 403 from non-US IPs, making it not viable for a public three-region benchmark. Limitless uses socket.io over HTTP long-polling, not standard WebSocket. Manifold, Myriad, PredictIt and Smarkets have no public WebSocket endpoint." + - "Other venues: Kalshi WS (wss://api.elections.kalshi.com/trade-api/ws/v2) requires RSA-PSS signed headers from a US KYC account and returns 403 from non-US IPs, so it is measured from us-east only and excluded from the rank matrix. Limitless uses socket.io over HTTP long-polling, not standard WebSocket. Manifold, Myriad, PredictIt, Smarkets and Metaculus have no public WebSocket endpoint." findings: - "Polymarket's CLOB WebSocket connects and delivers the first orderbook snapshot in {{p50:polymarket}} p50 (24h, all regions). This is the startup cost a new subscriber pays to go from a fresh dial to live market data." @@ -170,3 +170,6 @@ per_chain_explainer: - slug: smarkets h2: Smarkets body: "Smarkets has no public WebSocket endpoint. The REST API at api.smarkets.com/v3 provides order book access without authentication. Smarkets REST latency is tracked in pm-api-latency." + - slug: metaculus + h2: Metaculus + body: "Metaculus has no WebSocket endpoint. It is a public forecasting platform accessed entirely via REST (/api2/questions/). For REST latency and rate limit measurements, see pm-api-latency and pm-rate-limits." diff --git a/harnesses/pm-rate-limits/cmd/script/config.go b/harnesses/pm-rate-limits/cmd/script/config.go index 107293db..6ca3a65e 100644 --- a/harnesses/pm-rate-limits/cmd/script/config.go +++ b/harnesses/pm-rate-limits/cmd/script/config.go @@ -8,7 +8,7 @@ import ( // userAgent identifies every probe per the OCB methodology page. Venues can // contact us or block the UA selectively instead of banning a bare Go client. -const userAgent = "OpenChainBench/1.0 (+https://openchainbench.com/methodology; contact@mobula.io)" +const userAgent = "OpenChainBench/1.0 (+https://openchainbench.com/methodology; contact@openchainbench.com)" // Config groups the env-driven knobs the harness needs at startup. Direct // venue probes don't need any of these; the aggregator goroutines stay diff --git a/src/lib/pm-stats.ts b/src/lib/pm-stats.ts index 26dd8f60..69a162d6 100644 --- a/src/lib/pm-stats.ts +++ b/src/lib/pm-stats.ts @@ -79,6 +79,10 @@ const PM_VENUES: VenueSeed[] = [ { slug: "kalshi", name: "Kalshi", type: "offchain" }, { slug: "limitless", name: "Limitless", type: "onchain", chain: "base" }, { slug: "myriad", name: "Myriad", type: "onchain", chain: "abstract" }, + { slug: "manifold", name: "Manifold", type: "offchain" }, + { slug: "predictit", name: "PredictIt", type: "offchain" }, + { slug: "smarkets", name: "Smarkets", type: "offchain" }, + { slug: "metaculus", name: "Metaculus", type: "offchain" }, ]; const PM_DATA_FEEDS: DataFeedSeed[] = [ diff --git a/src/lib/pm-venue-context.ts b/src/lib/pm-venue-context.ts index cdec7fb6..9f134760 100644 --- a/src/lib/pm-venue-context.ts +++ b/src/lib/pm-venue-context.ts @@ -25,6 +25,8 @@ export const PM_VENUE_META: Record limitless: { url: "https://limitless.exchange", chainLabel: "Base" }, myriad: { url: "https://myriad.markets", chainLabel: "Abstract L2" }, manifold: { url: "https://manifold.markets", chainLabel: "Offchain" }, + predictit: { url: "https://www.predictit.org", chainLabel: "Offchain US" }, + smarkets: { url: "https://smarkets.com", chainLabel: "Offchain UK" }, metaculus: { url: "https://www.metaculus.com", chainLabel: "Offchain" }, }; From b1b03b30044e7bbe78fce3e7c8e213b5ed47f498 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:26:39 +0200 Subject: [PATCH 2/2] fix(data-api): make pivot bench cells clickable --- src/components/data-api-providers-pivot.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/components/data-api-providers-pivot.tsx b/src/components/data-api-providers-pivot.tsx index a4ef9659..3a179ee0 100644 --- a/src/components/data-api-providers-pivot.tsx +++ b/src/components/data-api-providers-pivot.tsx @@ -235,8 +235,9 @@ function GroupCell({ : "var(--color-ink-soft)"; return ( -
{/* Global rank + value */} @@ -264,7 +265,7 @@ function GroupCell({ />
)} - + ); }