diff --git a/.claude/skills/sanity-check/SKILL.md b/.claude/skills/sanity-check/SKILL.md index f1f2d9a9e5..88445ae723 100644 --- a/.claude/skills/sanity-check/SKILL.md +++ b/.claude/skills/sanity-check/SKILL.md @@ -15,7 +15,10 @@ Run a quick end-to-end sanity check of a published rivetkit version: copy the he ## Inputs 1. **Version or tag** (required) — explicit pkg-pr-new preview, npm dist-tag, or semver. If not provided in the user's message, ask for it. -2. **Additional test behavior** (optional) — e.g., "also verify workflows persist" or "check that KV works." If provided, extend `src/index.ts` + `test.mjs` using the menu in "Extending with custom tests" below. +2. **Engine mode** (optional, defaults to bundled) — the hello-world example's `registry.start()` only spawns the bundled engine binary when `RIVET_RUN_ENGINE=1` is set. Without it, the runtime boots in serverful mode and immediately fails trying to connect to a non-existent engine at `:6420` (`get local datacenters` → `Connection refused`). + - **bundled engine** (default): export `RIVET_RUN_ENGINE=1` before `node test.mjs`. Pairs the published `@rivetkit/engine-cli` binary with the published `rivetkit` runtime — what you want for end-to-end version verification. + - **external engine**: only use if the user explicitly says they have their own engine already running on `:6420` (e.g. `self-host/compose/dev`). Do not set the env var. +3. **Additional test behavior** (optional) — e.g., "also verify workflows persist" or "check that KV works." If provided, extend `src/index.ts` + `test.mjs` using the menu in "Extending with custom tests" below. ## Usage - `/sanity-check ` — run in a temp directory on the host @@ -151,22 +154,25 @@ try { ### 3. Install + run -**Default (host):** +**Default (host, bundled engine):** ```bash cd "$SANITY_DIR" npm install -node test.mjs +RIVET_RUN_ENGINE=1 node test.mjs ``` +Drop `RIVET_RUN_ENGINE=1` only if the user explicitly said they're running an external engine on `:6420`. + If you need to inspect a failure after the fact, tee the output: ```bash -node test.mjs 2>&1 | tee /tmp/sanity-check.log +RIVET_RUN_ENGINE=1 node test.mjs 2>&1 | tee /tmp/sanity-check.log echo "exit=$?" ``` **Docker mode:** ```bash docker run --rm \ + -e RIVET_RUN_ENGINE=1 \ -v "$SANITY_DIR":/app \ -w /app \ node:22 \ diff --git a/.github/workflows/rivet-deploy-kitchen-sink.yml b/.github/workflows/rivet-deploy-kitchen-sink.yml new file mode 100644 index 0000000000..3e316cbf64 --- /dev/null +++ b/.github/workflows/rivet-deploy-kitchen-sink.yml @@ -0,0 +1,97 @@ +name: Rivet Deploy (kitchen-sink) + +on: + pull_request: + types: [opened, synchronize, reopened, closed] + paths: + - "examples/kitchen-sink/**" + - "rivetkit-typescript/packages/**" + - "engine/sdks/typescript/**" + - "shared/typescript/**" + - "docker/build/linux-x64-gnu.Dockerfile" + - ".github/workflows/rivet-deploy-kitchen-sink.yml" + push: + branches: [main] + paths: + - "examples/kitchen-sink/**" + - "rivetkit-typescript/packages/**" + - "engine/sdks/typescript/**" + - "shared/typescript/**" + - "docker/build/linux-x64-gnu.Dockerfile" + - ".github/workflows/rivet-deploy-kitchen-sink.yml" + workflow_dispatch: + +concurrency: + group: rivet-deploy-kitchen-sink-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + rivet-deploy: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build rivetkit-napi .node (linux-x64-gnu) + env: + DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} + run: | + set -euo pipefail + # The DEPOT_TOKEN enables sccache; the Dockerfile gracefully falls + # back to a no-cache build if it isn't set. + DEPOT_TOKEN_FILE=$(mktemp) + printf "%s" "${DEPOT_TOKEN:-}" > "${DEPOT_TOKEN_FILE}" + docker buildx build \ + --file docker/build/linux-x64-gnu.Dockerfile \ + --build-arg BUILD_TARGET=rivetkit-napi \ + --build-arg BUILD_MODE=release \ + --secret id=DEPOT_TOKEN,src="${DEPOT_TOKEN_FILE}" \ + --load \ + --tag rivetkit-napi-builder:local \ + . + rm -f "${DEPOT_TOKEN_FILE}" + container_id=$(docker create rivetkit-napi-builder:local) + docker cp "${container_id}:/artifacts/rivetkit-napi.linux-x64-gnu.node" \ + ./examples/kitchen-sink/rivetkit-napi.linux-x64-gnu.node + docker rm "${container_id}" >/dev/null + ls -lh ./examples/kitchen-sink/rivetkit-napi.linux-x64-gnu.node + - name: Collect workspace manifest stubs + run: | + set -euo pipefail + # pnpm deploy resolves the full workspace topology even when filtered, + # so it needs every workspace package.json that isn't already in the + # Dockerfile's COPY set. Collect them under a single dir that the + # Dockerfile copies AFTER pnpm install/build (so install isn't + # tempted to walk them). + rm -rf .docker-ks-manifests + mkdir -p .docker-ks-manifests + find . -type f -name 'package.json' \ + -not -path './node_modules/*' \ + -not -path '*/node_modules/*' \ + -not -path './target/*' \ + -not -path './.git/*' \ + -not -path './.docker-ks-manifests/*' \ + -not -path './napi-artifacts/*' \ + -not -path './package.json' \ + -not -path './examples/kitchen-sink/*' \ + -not -path './rivetkit-typescript/packages/*' \ + -not -path './engine/sdks/typescript/*' \ + -not -path './shared/typescript/*' \ + -print0 | + while IFS= read -r -d '' src; do + rel="${src#./}" + dest=".docker-ks-manifests/${rel}" + mkdir -p "$(dirname "${dest}")" + cp "${src}" "${dest}" + done + echo "manifest stubs:" + find .docker-ks-manifests -name package.json | sort + - uses: rivet-dev/deploy-action@v1.1.0 + with: + rivet-token: ${{ secrets.KITCHEN_SINK_RIVET_CLOUD_TOKEN }} + dockerfile-path: examples/kitchen-sink/Dockerfile + docker-build-path: . + managed-pool-config: '{"environment":{"PORT":"8080"}}' diff --git a/.gitignore b/.gitignore index d067ff887d..368c5fc595 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,7 @@ examples/*/public/ # Ralph/Codex generated command stream logs scripts/ralph/codex-streams/ + +# Kitchen-sink Rivet Cloud deploy build artifacts +.docker-ks-manifests/ +examples/kitchen-sink/rivetkit-napi.*.node diff --git a/CLAUDE.md b/CLAUDE.md index bafa03a414..0af823dd7a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -77,6 +77,10 @@ docker-compose up -d - We use Graphite for stacked PRs. Diff against the parent branch (`gt ls` to see the stack), not `main`. - To revert a file to the version before this branch's changes, checkout from the first child branch (below in the stack), not from `main` or the parent. Child branches contain the pre-this-branch state of files modified by branches further down the stack. +### jj + +- Always invoke `jj diff` with `--git --color=never`. The jj-default format compresses unchanged context with `...` ellipses and uses a ` :` line-number prefix that visually fuses with adjacent tokens, causing misreads (e.g. `tracing::debug!` → `tracing::info!` looked like a `debug!` → `debuginfo!` rename). The unified `diff --git` format has unambiguous `-`/`+` markers at column 0 and standard `@@` hunk headers. + **Never push to `main` unless explicitly specified by the user.** ## Frontend Visual Changes diff --git a/Cargo.lock b/Cargo.lock index 78fb26e0ff..b84c506c39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5652,6 +5652,7 @@ dependencies = [ "parking_lot", "rand 0.8.5", "rivet-envoy-protocol", + "rivet-metrics", "rivet-util-serde", "rustls 0.23.29", "scc", diff --git a/engine/packages/depot-client/src/database.rs b/engine/packages/depot-client/src/database.rs index 996784b6c6..98613a6dc0 100644 --- a/engine/packages/depot-client/src/database.rs +++ b/engine/packages/depot-client/src/database.rs @@ -32,9 +32,10 @@ pub async fn open_database_from_transport( ) -> Result { let vfs_name = vfs_name_for_actor_database(&actor_id, generation); let config = VfsConfig::default(); - let initial_pages = fetch_initial_pages_for_registration(transport.clone(), &actor_id, &config) - .await - .map_err(|e| anyhow!("failed to preload sqlite pages: {e}"))?; + let initial_pages = + fetch_initial_pages_for_registration(transport.clone(), &actor_id, generation, &config) + .await + .map_err(|e| anyhow!("failed to preload sqlite pages: {e}"))?; let vfs = Arc::new( SqliteVfs::register_with_transport_and_initial_pages( &vfs_name, diff --git a/engine/packages/depot-client/src/optimization_flags.rs b/engine/packages/depot-client/src/optimization_flags.rs index c93398e61f..7c341f852c 100644 --- a/engine/packages/depot-client/src/optimization_flags.rs +++ b/engine/packages/depot-client/src/optimization_flags.rs @@ -23,11 +23,12 @@ pub const VFS_PAGE_CACHE_CAPACITY_PAGES_ENV: &str = "RIVETKIT_SQLITE_OPT_VFS_PAGE_CACHE_CAPACITY_PAGES"; pub const VFS_PROTECTED_CACHE_PAGES_ENV: &str = "RIVETKIT_SQLITE_OPT_VFS_PROTECTED_CACHE_PAGES"; pub const VFS_STAGING_CACHE_TTL_MS_ENV: &str = "RIVETKIT_SQLITE_OPT_VFS_STAGING_CACHE_TTL_MS"; +pub const VFS_RETAIN_READ_CACHE_ENV: &str = "RIVETKIT_SQLITE_OPT_VFS_RETAIN_READ_CACHE"; pub const DEFAULT_STARTUP_PRELOAD_MAX_BYTES: usize = 1024 * 1024; -pub const MAX_STARTUP_PRELOAD_MAX_BYTES: usize = 8 * 1024 * 1024; +pub const MAX_STARTUP_PRELOAD_MAX_BYTES: usize = 64 * 1024 * 1024; pub const DEFAULT_STARTUP_PRELOAD_FIRST_PAGE_COUNT: u32 = 1; -pub const MAX_STARTUP_PRELOAD_FIRST_PAGE_COUNT: u32 = 256; +pub const MAX_STARTUP_PRELOAD_FIRST_PAGE_COUNT: u32 = 16_384; pub const DEFAULT_VFS_PAGE_CACHE_CAPACITY_PAGES: u64 = 50_000; pub const MAX_VFS_PAGE_CACHE_CAPACITY_PAGES: u64 = 500_000; pub const DEFAULT_VFS_PROTECTED_CACHE_PAGES: usize = 512; @@ -106,6 +107,7 @@ pub struct SqliteOptimizationFlags { pub vfs_page_cache_capacity_pages: u64, pub vfs_protected_cache_pages: usize, pub vfs_staging_cache_ttl_ms: u64, + pub vfs_retain_read_cache: bool, } impl Default for SqliteOptimizationFlags { @@ -133,6 +135,7 @@ impl Default for SqliteOptimizationFlags { vfs_page_cache_capacity_pages: DEFAULT_VFS_PAGE_CACHE_CAPACITY_PAGES, vfs_protected_cache_pages: DEFAULT_VFS_PROTECTED_CACHE_PAGES, vfs_staging_cache_ttl_ms: DEFAULT_VFS_STAGING_CACHE_TTL_MS, + vfs_retain_read_cache: false, } } } @@ -206,6 +209,9 @@ impl SqliteOptimizationFlags { DEFAULT_VFS_STAGING_CACHE_TTL_MS, MAX_VFS_STAGING_CACHE_TTL_MS, ), + vfs_retain_read_cache: disabled_by_default( + read_env(VFS_RETAIN_READ_CACHE_ENV).as_deref(), + ), } } } @@ -228,6 +234,20 @@ fn enabled_by_default(value: Option<&str>) -> bool { _ => true, } } + +fn disabled_by_default(value: Option<&str>) -> bool { + match value.map(|value| value.trim().to_ascii_lowercase()) { + Some(value) + if matches!( + value.as_str(), + "1" | "true" | "on" | "yes" | "enabled" | "enable" + ) => + { + true + } + _ => false, + } +} fn usize_bounded_by_default(value: Option<&str>, default: usize, max: usize) -> usize { value .and_then(|value| value.trim().parse::().ok()) @@ -318,6 +338,7 @@ mod tests { VFS_PAGE_CACHE_CAPACITY_PAGES_ENV => Some("0".to_string()), VFS_PROTECTED_CACHE_PAGES_ENV => Some("0".to_string()), VFS_STAGING_CACHE_TTL_MS_ENV => Some("0".to_string()), + VFS_RETAIN_READ_CACHE_ENV => Some("true".to_string()), _ => None, }); @@ -339,6 +360,7 @@ mod tests { assert_eq!(flags.vfs_page_cache_capacity_pages, 0); assert_eq!(flags.vfs_protected_cache_pages, 0); assert_eq!(flags.vfs_staging_cache_ttl_ms, 0); + assert!(flags.vfs_retain_read_cache); } #[test] diff --git a/engine/packages/depot-client/src/vfs.rs b/engine/packages/depot-client/src/vfs.rs index f9ac4dcd41..73bca982af 100644 --- a/engine/packages/depot-client/src/vfs.rs +++ b/engine/packages/depot-client/src/vfs.rs @@ -150,8 +150,11 @@ pub struct VfsConfig { pub cache_hit_predictor_training: bool, pub recent_page_hints: bool, pub adaptive_read_ahead: bool, + pub retain_read_cache: bool, #[cfg(test)] pub assert_batch_atomic: bool, + #[cfg(test)] + pub advertise_batch_atomic: bool, } impl Default for VfsConfig { @@ -203,8 +206,11 @@ impl VfsConfig { cache_hit_predictor_training: flags.cache_hit_predictor_training, recent_page_hints: flags.recent_page_hints, adaptive_read_ahead: flags.adaptive_read_ahead, + retain_read_cache: flags.vfs_retain_read_cache, #[cfg(test)] assert_batch_atomic: true, + #[cfg(test)] + advertise_batch_atomic: true, } } } @@ -331,6 +337,7 @@ enum CommitWait { pub struct VfsContext { actor_id: String, + generation: Option, runtime: Handle, transport: SqliteTransportHandle, config: VfsConfig, @@ -869,6 +876,21 @@ impl VfsState { .or_else(|| self.page_cache.get(&pgno)) } + fn has_readable_page(&self, config: &VfsConfig, pgno: u32) -> bool { + if self.write_buffer.dirty.contains_key(&pgno) { + return true; + } + if !can_read_cached_page(config, pgno) { + return false; + } + self.committed_page_cache.contains_key(&pgno) + || self + .protected_page_cache + .read_sync(&pgno, |_, _| true) + .unwrap_or(false) + || self.page_cache.contains_key(&pgno) + } + fn cache_committed_page(&mut self, config: &VfsConfig, pgno: u32, bytes: Vec) { if config.staging_cache_ttl_ms == 0 || !config.page_cache_mode.caches_any_pages() { return; @@ -958,6 +980,7 @@ fn can_read_cached_page(config: &VfsConfig, pgno: u32) -> bool { impl VfsContext { fn new( actor_id: String, + generation: Option, runtime: Handle, transport: SqliteTransportHandle, config: VfsConfig, @@ -974,6 +997,7 @@ impl VfsContext { Ok(Self { actor_id, + generation, runtime, transport, config: config.clone(), @@ -1260,6 +1284,7 @@ impl VfsContext { seed_pgno, prediction_budget, predicted_pgnos, + skipped_cached_predicted_pages, db_size_pages, expected_head_txid, ) = { @@ -1278,6 +1303,7 @@ impl VfsContext { let seed_pgno = read_ahead_plan.seed_pgno; let mut prediction_budget = 0; let mut predicted_pgnos = Vec::new(); + let mut skipped_cached_predicted_pages = 0; if prefetch { let page_budget = (read_ahead_plan.max_bytes / state.page_size.max(1)).max(1); prediction_budget = page_budget.saturating_sub(to_fetch.len()); @@ -1291,6 +1317,10 @@ impl VfsContext { if resolved.contains_key(&predicted) || to_fetch.contains(&predicted) { continue; } + if state.has_readable_page(&self.config, predicted) { + skipped_cached_predicted_pages += 1; + continue; + } to_fetch.push(predicted); } } @@ -1303,6 +1333,7 @@ impl VfsContext { seed_pgno, prediction_budget, predicted_pgnos, + skipped_cached_predicted_pages, state.db_size_pages, state.head_txid, ) @@ -1322,7 +1353,9 @@ impl VfsContext { page_size as u64, ); } - tracing::debug!( + tracing::info!( + actor_id = %self.actor_id, + generation = ?self.generation, requested_pages = ?target_pgnos, missing_pages = ?missing, read_ahead_mode = ?read_ahead_mode, @@ -1330,10 +1363,13 @@ impl VfsContext { read_ahead_max_bytes, prediction_budget, predicted_pages = ?predicted_pgnos, + skipped_cached_predicted_pages, prefetch_pages = prefetch_count, total_fetch_pages = to_fetch.len(), total_fetch_bytes = to_fetch.len().saturating_mul(page_size), seed_pgno, + db_size_pages, + expected_head_txid, "vfs get_pages fetch" ); } @@ -1536,7 +1572,7 @@ impl VfsContext { if let Some(metrics) = &self.metrics { metrics.record_commit(); } - tracing::debug!( + tracing::info!( dirty_pages = request.dirty_pages.len(), path = ?outcome.path, requested_db_size_pages = request.new_db_size_pages, @@ -1705,6 +1741,37 @@ impl VfsContext { } } +impl Drop for VfsContext { + fn drop(&mut self) { + let state = self.state.read(); + let page_cache_entries = state + .page_cache + .entry_count() + .saturating_add(state.committed_page_cache.entry_count()) + .saturating_add(state.protected_page_cache.len() as u64); + let page_cache_weighted_size = state + .page_cache + .weighted_size() + .saturating_add(state.protected_page_cache.len() as u64); + tracing::debug!( + actor_id = %self.actor_id, + generation = ?self.generation, + resolve_pages_calls = self.resolve_pages_total.load(Ordering::Relaxed), + resolve_pages_cache_hits = self.resolve_pages_cache_hits.load(Ordering::Relaxed), + get_pages_round_trips = self.resolve_pages_fetches.load(Ordering::Relaxed), + pages_fetched_total = self.pages_fetched_total.load(Ordering::Relaxed), + prefetch_pages_total = self.prefetch_pages_total.load(Ordering::Relaxed), + commit_count = self.commit_total.load(Ordering::Relaxed), + db_size_pages = state.db_size_pages, + head_txid = state.head_txid, + page_cache_entries, + page_cache_weighted_size, + page_cache_capacity_pages = self.config.cache_capacity_pages, + "sqlite vfs close summary" + ); + } +} + fn cleanup_batch_atomic_probe(db: *mut sqlite3) { if let Err(err) = sqlite_exec(db, "DROP TABLE IF EXISTS __rivet_batch_probe;") { tracing::warn!(%err, "failed to clean up sqlite batch atomic probe table"); @@ -1775,7 +1842,7 @@ pub(crate) async fn fetch_initial_main_page_for_registration( transport: SqliteTransportHandle, actor_id: &str, ) -> std::result::Result>, String> { - fetch_initial_pages(transport, actor_id.to_string(), 1) + fetch_initial_pages(transport, actor_id.to_string(), 0, 1) .await .map(|pages| { pages @@ -1789,13 +1856,14 @@ pub(crate) async fn fetch_initial_main_page_for_registration( pub(crate) async fn fetch_initial_pages_for_registration( transport: SqliteTransportHandle, actor_id: &str, + generation: u64, config: &VfsConfig, ) -> std::result::Result { if !config.startup_preload_first_pages || !config.page_cache_mode.caches_startup_preloaded_pages() || config.startup_preload_max_bytes < DEFAULT_PAGE_SIZE { - return fetch_initial_pages(transport, actor_id.to_string(), 1).await; + return fetch_initial_pages(transport, actor_id.to_string(), generation, 1).await; } let page_count_from_bytes = config.startup_preload_max_bytes / DEFAULT_PAGE_SIZE; @@ -1803,14 +1871,22 @@ pub(crate) async fn fetch_initial_pages_for_registration( .startup_preload_first_page_count .min(page_count_from_bytes as u32) .max(1); - fetch_initial_pages(transport, actor_id.to_string(), page_count).await + fetch_initial_pages(transport, actor_id.to_string(), generation, page_count).await } async fn fetch_initial_pages( transport: SqliteTransportHandle, actor_id: String, + generation: u64, page_count: u32, ) -> std::result::Result { + tracing::info!( + actor_id = %actor_id, + generation, + page_count, + "sqlite initial page preload request" + ); + let request_actor_id = actor_id.clone(); let response = transport .get_pages(protocol::SqliteGetPagesRequest { @@ -1822,14 +1898,23 @@ async fn fetch_initial_pages( .await; match response { - Ok(protocol::SqliteGetPagesResponse::SqliteGetPagesOk(ok)) => Ok(InitialPages { - pages: ok + Ok(protocol::SqliteGetPagesResponse::SqliteGetPagesOk(ok)) => { + let head_txid = ok.head_txid; + let pages: Vec<_> = ok .pages .into_iter() .filter_map(|page| page.bytes.map(|bytes| (page.pgno, bytes))) - .collect(), - head_txid: ok.head_txid, - }), + .collect(); + tracing::info!( + actor_id = %actor_id, + generation, + page_count, + loaded_pages = pages.len(), + head_txid, + "sqlite initial page preload result" + ); + Ok(InitialPages { pages, head_txid }) + } Ok(protocol::SqliteGetPagesResponse::SqliteErrorResponse(error)) => { if !is_initial_main_page_missing(&error.message) { return Err(format!( @@ -1837,8 +1922,10 @@ async fn fetch_initial_pages( error.message )); } - tracing::debug!( - actor_id, + tracing::info!( + actor_id = %actor_id, + generation, + page_count, error = %error.message, "sqlite initial page fetch did not find persisted data" ); @@ -2301,7 +2388,9 @@ unsafe extern "C" fn io_read( buf[dest_offset..dest_offset + copy_len] .copy_from_slice(&bytes[page_offset..page_offset + copy_len]); } - ctx.state.read().evict_target_read_pages(&requested_pages); + if !ctx.config.retain_read_cache { + ctx.state.read().evict_target_read_pages(&requested_pages); + } if i_offset as usize + i_amt as usize > file_size { return SQLITE_IOERR_SHORT_READ; @@ -2608,6 +2697,10 @@ unsafe extern "C" fn io_device_characteristics(p_file: *mut sqlite3_file) -> c_i if get_aux_state(file).is_some() { 0 } else { + #[cfg(test)] + if !(*file.ctx).config.advertise_batch_atomic { + return 0; + } SQLITE_IOCAP_BATCH_ATOMIC } }) @@ -2908,8 +3001,12 @@ impl SqliteVfs { io_methods.xSectorSize = Some(io_sector_size); io_methods.xDeviceCharacteristics = Some(io_device_characteristics); + let generation = name + .rsplit_once("-g") + .and_then(|(_, generation)| generation.parse::().ok()); let mut ctx = Box::new(VfsContext::new( actor_id, + generation, runtime, transport, config, diff --git a/engine/packages/depot-client/tests/inline/vfs.rs b/engine/packages/depot-client/tests/inline/vfs.rs index 1deb13e0ab..2bbd5e162c 100644 --- a/engine/packages/depot-client/tests/inline/vfs.rs +++ b/engine/packages/depot-client/tests/inline/vfs.rs @@ -3,10 +3,11 @@ mod vfs_support; pub(super) use vfs_support::{ DirectDepotTransport, DirectMirrorTransport, DirectStorage, DirectStorageStats, - storage_dirty_page, + sqlite_error_response, storage_dirty_page, }; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::mem::ManuallyDrop; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Barrier, mpsc}; use std::thread; use std::time::Duration; @@ -17,7 +18,7 @@ use parking_lot::Mutex as SyncMutex; use rivet_envoy_protocol as protocol; use tempfile::TempDir; use tokio::runtime::Builder; -use tokio::sync::OnceCell; +use tokio::sync::{Notify, OnceCell}; use crate::optimization_flags::{ DEFAULT_STARTUP_PRELOAD_MAX_BYTES, DEFAULT_VFS_PAGE_CACHE_CAPACITY_PAGES, @@ -57,6 +58,7 @@ fn vfs_config_wires_optimization_flags() { vfs_page_cache_capacity_pages: DEFAULT_VFS_PAGE_CACHE_CAPACITY_PAGES / 2, vfs_protected_cache_pages: DEFAULT_VFS_PROTECTED_CACHE_PAGES / 2, vfs_staging_cache_ttl_ms: DEFAULT_VFS_STAGING_CACHE_TTL_MS / 2, + vfs_retain_read_cache: true, }; let config = VfsConfig::from_optimization_flags(flags); @@ -80,6 +82,7 @@ fn vfs_config_wires_optimization_flags() { assert_eq!(config.startup_preload_first_page_count, 7); assert!(!config.preload_hints_on_open); assert!(!config.preload_hint_early_pages); + assert!(config.retain_read_cache); assert_eq!(config.recent_hint_page_budget, 0); assert!(config.recent_hint_range_budget > 0); } @@ -119,6 +122,79 @@ impl SqliteTransport for RecordingInitialPagesTransport { } } +struct CappedAtomicCommitTransport { + inner: DirectDepotTransport, + max_dirty_bytes: usize, + rejected: AtomicBool, + max_seen_dirty_bytes: AtomicUsize, + max_seen_dirty_pages: AtomicUsize, +} + +impl CappedAtomicCommitTransport { + fn new(engine: Arc, max_dirty_bytes: usize) -> Self { + Self { + inner: DirectDepotTransport::new(engine), + max_dirty_bytes, + rejected: AtomicBool::new(false), + max_seen_dirty_bytes: AtomicUsize::new(0), + max_seen_dirty_pages: AtomicUsize::new(0), + } + } + + fn rejected(&self) -> bool { + self.rejected.load(Ordering::SeqCst) + } + + fn max_seen_dirty_bytes(&self) -> usize { + self.max_seen_dirty_bytes.load(Ordering::SeqCst) + } + + fn max_seen_dirty_pages(&self) -> usize { + self.max_seen_dirty_pages.load(Ordering::SeqCst) + } +} + +#[async_trait] +impl SqliteTransport for CappedAtomicCommitTransport { + async fn get_pages( + &self, + request: protocol::SqliteGetPagesRequest, + ) -> anyhow::Result { + self.inner.get_pages(request).await + } + + async fn commit( + &self, + request: protocol::SqliteCommitRequest, + ) -> anyhow::Result { + let dirty_bytes = request + .dirty_pages + .iter() + .map(|page| page.bytes.len()) + .sum::(); + self.max_seen_dirty_bytes + .fetch_max(dirty_bytes, Ordering::SeqCst); + self.max_seen_dirty_pages + .fetch_max(request.dirty_pages.len(), Ordering::SeqCst); + + if dirty_bytes > self.max_dirty_bytes { + self.rejected.store(true, Ordering::SeqCst); + return Ok(protocol::SqliteCommitResponse::SqliteErrorResponse( + protocol::SqliteErrorResponse { + group: "sqlite".to_string(), + code: "transaction_too_large".to_string(), + message: format!( + "simulated atomic commit cap exceeded: dirty_bytes={dirty_bytes} max_dirty_bytes={}", + self.max_dirty_bytes + ), + }, + )); + } + + self.inner.commit(request).await + } +} + struct MissingDbTransport; #[async_trait] @@ -144,6 +220,68 @@ impl SqliteTransport for MissingDbTransport { } } +struct DelayCapturedGetPagesTransport { + inner: Arc, + delay_pgno: u32, + enabled: AtomicBool, + delayed: AtomicBool, + captured: Arc, + release: Arc, +} + +impl DelayCapturedGetPagesTransport { + fn new(inner: Arc, delay_pgno: u32) -> Self { + Self { + inner, + delay_pgno, + enabled: AtomicBool::new(false), + delayed: AtomicBool::new(false), + captured: Arc::new(Notify::new()), + release: Arc::new(Notify::new()), + } + } + + fn enable(&self) { + self.enabled.store(true, Ordering::SeqCst); + } + + async fn wait_captured(&self) { + self.captured.notified().await; + } + + fn release(&self) { + self.release.notify_one(); + } +} + +#[async_trait] +impl SqliteTransport for DelayCapturedGetPagesTransport { + async fn get_pages( + &self, + request: protocol::SqliteGetPagesRequest, + ) -> anyhow::Result { + let should_delay = self.enabled.load(Ordering::SeqCst) + && (self.delay_pgno == 0 || request.pgnos.contains(&self.delay_pgno)) + && !self.delayed.swap(true, Ordering::SeqCst); + if !should_delay { + return self.inner.get_pages(request).await; + } + + // Capture real Depot bytes first, then release them after another writer advances the head. + let response = self.inner.get_pages(request).await?; + self.captured.notify_one(); + self.release.notified().await; + Ok(response) + } + + async fn commit( + &self, + request: protocol::SqliteCommitRequest, + ) -> anyhow::Result { + self.inner.commit(request).await + } +} + #[test] fn startup_initial_pages_do_not_require_preload_hints_on_open() { let runtime = direct_runtime(); @@ -161,6 +299,7 @@ fn startup_initial_pages_do_not_require_preload_hints_on_open() { .block_on(fetch_initial_pages_for_registration( transport.clone(), "startup-preload-actor", + 0, &config, )) .expect("initial pages should load"); @@ -253,11 +392,12 @@ fn evicted_empty_page_one_can_be_synthesized_before_first_commit() { let config = VfsConfig::default(); let ctx = VfsContext::new( next_test_name("missing-db-actor"), + None, runtime.handle().clone(), Arc::new(MissingDbTransport), config.clone(), unsafe { std::mem::zeroed() }, - Vec::new(), + InitialPages::default(), None, ) .expect("vfs context should build"); @@ -336,6 +476,16 @@ impl DirectEngineHarness { config: VfsConfig, ) -> NativeDatabase { let transport = Arc::new(DirectDepotTransport::new(engine)); + self.open_db_with_transport(runtime, transport, actor_id, config) + } + + fn open_db_with_transport( + &self, + runtime: &tokio::runtime::Runtime, + transport: SqliteTransportHandle, + actor_id: &str, + config: VfsConfig, + ) -> NativeDatabase { let initial_main_page = runtime .block_on(fetch_initial_main_page_for_registration( transport.clone(), @@ -365,11 +515,12 @@ impl DirectEngineHarness { let engine = runtime.block_on(self.open_engine()); VfsContext::new( self.actor_id.clone(), + None, runtime.handle().clone(), Arc::new(DirectDepotTransport::new(engine)), VfsConfig::default(), unsafe { std::mem::zeroed() }, - Vec::new(), + InitialPages::default(), None, ) .expect("vfs context should build") @@ -407,6 +558,7 @@ fn open_worker_handle_with_vfs( .block_on(fetch_initial_pages_for_registration( transport.clone(), &harness.actor_id, + 0, &config, )) .expect("initial pages preload should succeed"); @@ -1027,6 +1179,162 @@ fn direct_engine_handles_large_rows_and_multi_page_growth() { ); } +#[test] +fn batch_atomic_commit_does_not_depend_on_aux_journal_open() { + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let db = harness.open_db(&runtime); + let ctx = direct_vfs_ctx(&db); + + ctx.fail_next_aux_open("InjectedAuxOpenError: journal open should not be needed"); + sqlite_exec( + db.as_ptr(), + "CREATE TABLE batch_atomic_no_journal (id INTEGER PRIMARY KEY, value TEXT NOT NULL);", + ) + .expect("batch atomic create should not need rollback journal open"); + sqlite_exec( + db.as_ptr(), + "INSERT INTO batch_atomic_no_journal (id, value) VALUES (1, 'ok');", + ) + .expect("batch atomic insert should not need rollback journal open"); + + assert_eq!( + sqlite_query_text(db.as_ptr(), "PRAGMA integrity_check;") + .expect("integrity_check should succeed"), + "ok" + ); +} + +#[test] +fn batch_atomic_oversized_commit_fails_closed_and_reopens_clean() { + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let actor_id = harness.actor_id.clone(); + let transport = Arc::new(CappedAtomicCommitTransport::new( + Arc::clone(&engine), + 256 * 1024, + )); + let db = harness.open_db_with_transport( + &runtime, + transport.clone(), + &actor_id, + VfsConfig::default(), + ); + + sqlite_exec( + db.as_ptr(), + "CREATE TABLE atomic_cap (id INTEGER PRIMARY KEY, payload BLOB NOT NULL);", + ) + .expect("create table should succeed"); + sqlite_exec( + db.as_ptr(), + "INSERT INTO atomic_cap (id, payload) VALUES (1, zeroblob(1024));", + ) + .expect("seed insert should succeed"); + assert_eq!( + sqlite_query_text(db.as_ptr(), "PRAGMA integrity_check;") + .expect("pre-cap integrity_check should succeed"), + "ok" + ); + + sqlite_exec(db.as_ptr(), "BEGIN IMMEDIATE;").expect("large transaction should begin"); + for id in 2..=130 { + sqlite_exec( + db.as_ptr(), + &format!("INSERT INTO atomic_cap (id, payload) VALUES ({id}, randomblob(4096));"), + ) + .expect("large transaction insert should succeed"); + } + let commit_err = sqlite_exec(db.as_ptr(), "COMMIT;").expect_err("oversized commit should fail"); + assert!( + commit_err.contains("disk I/O error") || commit_err.contains("transaction_too_large"), + "unexpected commit error: {commit_err}", + ); + assert!( + transport.rejected(), + "test did not exceed the simulated atomic commit cap; max_seen_dirty_bytes={} max_seen_dirty_pages={}", + transport.max_seen_dirty_bytes(), + transport.max_seen_dirty_pages(), + ); + drop(db); + + let reopened = harness.open_db_on_engine(&runtime, engine, &actor_id, VfsConfig::default()); + assert_eq!( + sqlite_query_i64(reopened.as_ptr(), "SELECT COUNT(*) FROM atomic_cap;") + .expect("count after capped commit should succeed"), + 1 + ); + assert_eq!( + sqlite_query_text(reopened.as_ptr(), "PRAGMA quick_check;") + .expect("quick_check after capped commit should succeed"), + "ok" + ); + assert_eq!( + sqlite_query_text(reopened.as_ptr(), "PRAGMA integrity_check;") + .expect("integrity_check after capped commit should succeed"), + "ok" + ); +} + +#[test] +fn volatile_rollback_journal_without_batch_atomic_can_make_failed_commit_durable() { + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let transport = Arc::new(DirectDepotTransport::new(Arc::clone(&engine))); + let hooks = transport.direct_hooks(); + let actor_id = harness.actor_id.clone(); + let mut config = VfsConfig::default(); + config.assert_batch_atomic = false; + config.advertise_batch_atomic = false; + let db = harness.open_db_with_transport(&runtime, transport, &actor_id, config); + + sqlite_exec( + db.as_ptr(), + "CREATE TABLE rollback_journal_loss (id INTEGER PRIMARY KEY, value TEXT NOT NULL);", + ) + .expect("create table should succeed"); + sqlite_exec( + db.as_ptr(), + "INSERT INTO rollback_journal_loss (id, value) VALUES (1, 'before');", + ) + .expect("seed insert should succeed"); + + hooks.fail_next_commit_after_apply("InjectedTransportError: main db sync response lost"); + sqlite_exec(db.as_ptr(), "BEGIN IMMEDIATE;").expect("transaction should begin"); + sqlite_exec( + db.as_ptr(), + "INSERT INTO rollback_journal_loss (id, value) VALUES (2, 'failed-commit');", + ) + .expect("transaction insert should succeed"); + let commit_err = + sqlite_exec(db.as_ptr(), "COMMIT;").expect_err("commit response loss should fail"); + assert!( + commit_err.contains("disk I/O error"), + "unexpected commit error: {commit_err}" + ); + + let leaked = ManuallyDrop::new(db); + let _fatal_after_failed_commit = direct_vfs_ctx(&leaked).clone_fatal_error(); + + let reopened = harness.open_db_on_engine(&runtime, engine, &actor_id, VfsConfig::default()); + assert_eq!( + sqlite_query_text(reopened.as_ptr(), "PRAGMA integrity_check;") + .expect("integrity_check should succeed after simulated crash"), + "ok" + ); + assert_eq!( + sqlite_query_i64( + reopened.as_ptr(), + "SELECT COUNT(*) FROM rollback_journal_loss;" + ) + .expect("count after simulated crash should succeed"), + 2, + "without batch atomic, a lost main-db sync response can make a failed commit durable because the rollback journal was process-local", + ); +} + #[test] fn direct_engine_persists_data_across_close_and_reopen() { let runtime = direct_runtime(); @@ -2688,11 +2996,12 @@ fn concurrent_reader_during_commit_atomic_observes_consistent_snapshot() { let hooks = transport.direct_hooks(); let ctx = VfsContext::new( harness.actor_id.clone(), + None, runtime.handle().clone(), transport, VfsConfig::default(), unsafe { std::mem::zeroed() }, - Vec::new(), + InitialPages::default(), None, ) .expect("vfs context should build"); @@ -3619,11 +3928,12 @@ fn resolve_pages_surfaces_read_path_error_response() { let hooks = transport.direct_hooks(); let ctx = VfsContext::new( harness.actor_id.clone(), + None, runtime.handle().clone(), transport, VfsConfig::default(), unsafe { std::mem::zeroed() }, - Vec::new(), + InitialPages::default(), None, ) .expect("vfs context should build"); @@ -3639,6 +3949,149 @@ fn resolve_pages_surfaces_read_path_error_response() { )); } +#[test] +fn dead_vfs_rejects_cache_only_reads_before_serving_page_cache() { + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let transport = Arc::new(DirectDepotTransport::new(engine)); + let hooks = transport.direct_hooks(); + let config = VfsConfig::default(); + let ctx = VfsContext::new( + harness.actor_id.clone(), + None, + runtime.handle().clone(), + transport, + config.clone(), + unsafe { std::mem::zeroed() }, + InitialPages::default(), + None, + ) + .expect("vfs context should build"); + { + let mut state = ctx.state.write(); + state.db_size_pages = 2; + state.cache_page(&config, PageCacheInsertKind::Target, 2, vec![0x42; 4096]); + } + + ctx.mark_fatal("generation fence lost".to_string()); + let err = ctx + .resolve_pages(&[2], false) + .expect_err("dead VFS should not serve cached pages"); + assert!(matches!( + err, + GetPagesError::Other(ref message) if message.contains("lost its fence") + )); + assert!( + hooks.get_pages_requests().is_empty(), + "dead cache-only read should fail before calling Depot" + ); +} + +#[test] +fn head_fence_mismatch_on_xread_poisoned_vfs_fails_later_operations_closed() { + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + runtime + .block_on(async { + engine + .actor_db(harness.actor_id.clone()) + .await + .commit( + vec![depot::types::DirtyPage { + pgno: 2, + bytes: vec![0x55; 4096], + }], + 2, + sqlite_now_ms().expect("now ms should build"), + ) + .await + }) + .expect("seed commit should advance Depot head"); + + let transport = Arc::new(DirectDepotTransport::new(engine)); + let hooks = transport.direct_hooks(); + let config = VfsConfig::default(); + let ctx = VfsContext::new( + harness.actor_id.clone(), + None, + runtime.handle().clone(), + transport, + config.clone(), + unsafe { std::mem::zeroed() }, + InitialPages::default(), + None, + ) + .expect("vfs context should build"); + { + let mut state = ctx.state.write(); + state.db_size_pages = 2; + state.head_txid = Some(0); + state.cache_page(&config, PageCacheInsertKind::Target, 1, empty_db_page()); + } + + let mut file = VfsFile { + base: unsafe { std::mem::zeroed() }, + ctx: &ctx, + aux: ptr::null_mut(), + }; + let mut buf = vec![0; 4096]; + let rc = unsafe { + io_read( + (&mut file as *mut VfsFile).cast::(), + buf.as_mut_ptr().cast(), + buf.len() as c_int, + 4096, + ) + }; + assert_eq!(rc, SQLITE_IOERR_READ); + assert!(ctx.is_dead()); + assert!( + ctx.clone_fatal_error() + .is_some_and(|message| message.contains("head fence mismatch")), + "head fence mismatch should be retained as the fatal reason" + ); + assert_eq!(hooks.get_pages_requests().len(), 1); + + let rc = unsafe { + io_read( + (&mut file as *mut VfsFile).cast::(), + buf.as_mut_ptr().cast(), + buf.len() as c_int, + 0, + ) + }; + assert_eq!(rc, SQLITE_IOERR_READ); + assert_eq!( + hooks.get_pages_requests().len(), + 1, + "dead cache-only read should not call Depot again" + ); + + let source = vec![0x7a; 4096]; + let rc = unsafe { + io_write( + (&mut file as *mut VfsFile).cast::(), + source.as_ptr().cast(), + source.len() as c_int, + 0, + ) + }; + assert_eq!(rc, SQLITE_IOERR_WRITE); + + { + let mut state = ctx.state.write(); + state.write_buffer.dirty.insert(1, empty_db_page()); + } + let rc = unsafe { io_sync((&mut file as *mut VfsFile).cast::(), 0) }; + assert_eq!(rc, SQLITE_IOERR_FSYNC); + assert!( + hooks.commit_requests().is_empty(), + "dead VFS should fail writes and sync before committing" + ); +} + #[test] fn resolve_pages_sends_known_head_txid_as_read_fence() { let runtime = direct_runtime(); @@ -3658,11 +4111,12 @@ fn resolve_pages_sends_known_head_txid_as_read_fence() { let hooks = transport.direct_hooks(); let ctx = VfsContext::new( harness.actor_id.clone(), + None, runtime.handle().clone(), transport, VfsConfig::default(), unsafe { std::mem::zeroed() }, - Vec::new(), + InitialPages::default(), None, ) .expect("vfs context should build"); @@ -3761,6 +4215,1944 @@ fn head_fence_ioerr_maps_to_fatal_worker_error_and_future_operations_fail_closed .expect("writer worker should close cleanly"); } +#[test] +fn stale_vfs_page_cache_writer_fails_head_fence_and_reopen_is_clean() { + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let config = VfsConfig { + page_cache_mode: SqliteVfsPageCacheMode::All, + ..VfsConfig::default() + }; + let db_a = + harness.open_db_on_engine(&runtime, engine.clone(), &harness.actor_id, config.clone()); + + sqlite_exec( + db_a.as_ptr(), + "CREATE TABLE cached_rows (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);", + ) + .expect("create should succeed"); + for id in 1..=64 { + sqlite_exec( + db_a.as_ptr(), + &format!( + "INSERT INTO cached_rows (id, payload) VALUES ({id}, printf('%04d:%s', {id}, zeroblob(512)));" + ), + ) + .expect("insert should succeed"); + } + + let cached_before = sqlite_query_text( + db_a.as_ptr(), + "SELECT payload FROM cached_rows WHERE id = 32;", + ) + .expect("stale handle should read target row"); + assert!(cached_before.starts_with("0032:")); + + let db_b = + harness.open_db_on_engine(&runtime, engine.clone(), &harness.actor_id, config.clone()); + sqlite_exec( + db_b.as_ptr(), + "UPDATE cached_rows SET payload = 'writer-b-new-value' WHERE id = 32;", + ) + .expect("fresh writer should commit"); + assert_eq!( + sqlite_query_text( + db_b.as_ptr(), + "SELECT payload FROM cached_rows WHERE id = 32;" + ) + .expect("fresh writer should read its update"), + "writer-b-new-value" + ); + + let stale_update = sqlite_exec( + db_a.as_ptr(), + "UPDATE cached_rows SET payload = 'stale-writer-value' WHERE id = 32;", + ); + assert!( + stale_update + .as_ref() + .expect_err("stale writer must fail the head fence") + .contains("disk I/O error"), + "unexpected stale writer error: {stale_update:?}" + ); + assert!( + direct_vfs_ctx(&db_a) + .clone_fatal_error() + .is_some_and(|message| message.contains("head fence mismatch")), + "stale VFS should retain the head fence mismatch" + ); + + drop(db_a); + drop(db_b); + + let db_c = harness.open_db_on_engine(&runtime, engine, &harness.actor_id, config); + assert_eq!( + sqlite_query_text(db_c.as_ptr(), "PRAGMA integrity_check;") + .expect("integrity check should run after stale writer failed"), + "ok" + ); + assert_eq!( + sqlite_query_text( + db_c.as_ptr(), + "SELECT payload FROM cached_rows WHERE id = 32;" + ) + .expect("fresh reopen should see writer B value"), + "writer-b-new-value" + ); +} + +#[test] +fn delayed_read_ahead_response_fails_head_fence_and_reopen_is_clean() { + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let config = VfsConfig { + page_cache_mode: SqliteVfsPageCacheMode::All, + ..VfsConfig::default() + }; + let db_setup = + harness.open_db_on_engine(&runtime, engine.clone(), &harness.actor_id, config.clone()); + + sqlite_exec( + db_setup.as_ptr(), + "CREATE TABLE delayed_rows (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);", + ) + .expect("create should succeed"); + for id in 1..=96 { + sqlite_exec( + db_setup.as_ptr(), + &format!( + "INSERT INTO delayed_rows (id, payload) VALUES ({id}, printf('%04d:%s', {id}, hex(zeroblob(2048))));" + ), + ) + .expect("insert should succeed"); + } + let page_count = sqlite_query_i64(db_setup.as_ptr(), "PRAGMA page_count;") + .expect("page count should be readable"); + assert!( + page_count >= 12, + "test setup should create enough pages for read-ahead, got {page_count}" + ); + drop(db_setup); + + let delaying_transport = Arc::new(DelayCapturedGetPagesTransport::new( + Arc::new(DirectDepotTransport::new(engine.clone())), + 0, + )); + let db_a = harness.open_db_with_transport( + &runtime, + delaying_transport.clone(), + &harness.actor_id, + config.clone(), + ); + + let ctx = direct_vfs_ctx(&db_a); + ctx.resolve_pages(&[2], false) + .expect("training read page 2 should succeed"); + ctx.resolve_pages(&[3], false) + .expect("training read page 3 should succeed"); + ctx.resolve_pages(&[4], false) + .expect("training read page 4 should succeed"); + + delaying_transport.enable(); + let vfs_a = db_a._vfs.clone(); + let delayed_read = thread::spawn(move || { + vfs_a + .ctx() + .resolve_pages(&[5], true) + .expect("delayed read-ahead fetch should eventually succeed") + }); + + runtime + .block_on(async { + tokio::time::timeout(Duration::from_secs(5), delaying_transport.wait_captured()).await + }) + .expect("delayed get_pages response should be captured before writer B commits"); + + let db_b = + harness.open_db_on_engine(&runtime, engine.clone(), &harness.actor_id, config.clone()); + sqlite_exec( + db_b.as_ptr(), + "UPDATE delayed_rows SET payload = 'writer-b-new-value' WHERE id = 48;", + ) + .expect("fresh writer should commit while delayed response is held"); + + delaying_transport.release(); + let delayed_pages = delayed_read + .join() + .expect("delayed read thread should not panic"); + assert!( + delayed_pages.contains_key(&5), + "delayed read should resolve the target page" + ); + + let stale_update = sqlite_exec( + db_a.as_ptr(), + "UPDATE delayed_rows SET payload = 'stale-writer-value' WHERE id = 48;", + ); + assert!( + stale_update + .as_ref() + .expect_err("stale writer must fail after delayed old response") + .contains("disk I/O error"), + "unexpected stale writer error: {stale_update:?}" + ); + assert!( + direct_vfs_ctx(&db_a) + .clone_fatal_error() + .is_some_and(|message| message.contains("head fence mismatch")), + "stale VFS should retain the head fence mismatch" + ); + + drop(db_a); + drop(db_b); + + let db_c = harness.open_db_on_engine(&runtime, engine, &harness.actor_id, config); + assert_eq!( + sqlite_query_text(db_c.as_ptr(), "PRAGMA integrity_check;") + .expect("integrity check should run after delayed stale writer failed"), + "ok" + ); + assert_eq!( + sqlite_query_text( + db_c.as_ptr(), + "SELECT payload FROM delayed_rows WHERE id = 48;" + ) + .expect("fresh reopen should see writer B value"), + "writer-b-new-value" + ); +} + +#[test] +fn delayed_startup_preload_response_fails_closed_and_reopen_is_clean() { + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let config = VfsConfig { + page_cache_mode: SqliteVfsPageCacheMode::All, + startup_preload_first_pages: true, + startup_preload_first_page_count: 8, + startup_preload_max_bytes: DEFAULT_PAGE_SIZE * 8, + ..VfsConfig::default() + }; + let db_setup = + harness.open_db_on_engine(&runtime, engine.clone(), &harness.actor_id, config.clone()); + + sqlite_exec( + db_setup.as_ptr(), + "CREATE TABLE preload_rows (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);", + ) + .expect("create should succeed"); + for id in 1..=96 { + sqlite_exec( + db_setup.as_ptr(), + &format!( + "INSERT INTO preload_rows (id, payload) VALUES ({id}, printf('%04d:%s', {id}, hex(zeroblob(2048))));" + ), + ) + .expect("insert should succeed"); + } + let page_count = sqlite_query_i64(db_setup.as_ptr(), "PRAGMA page_count;") + .expect("page count should be readable"); + assert!( + page_count >= 12, + "test setup should create enough pages for startup preload, got {page_count}" + ); + drop(db_setup); + + let delaying_transport = Arc::new(DelayCapturedGetPagesTransport::new( + Arc::new(DirectDepotTransport::new(engine.clone())), + 0, + )); + delaying_transport.enable(); + let actor_id = harness.actor_id.clone(); + let config_a = config.clone(); + let runtime_handle = runtime.handle().clone(); + let open_transport = delaying_transport.clone(); + let opening = thread::spawn(move || { + let initial_pages = runtime_handle + .block_on(fetch_initial_pages_for_registration( + open_transport.clone(), + &actor_id, + 0, + &config_a, + )) + .expect("startup preload should eventually succeed"); + let vfs = SqliteVfs::register_with_transport_and_initial_pages( + &next_test_name("sqlite-startup-preload-vfs"), + open_transport, + actor_id.clone(), + runtime_handle, + config_a, + initial_pages, + None, + ) + .expect("vfs should register after startup preload"); + open_database(vfs, &actor_id) + }); + + runtime + .block_on(async { + tokio::time::timeout(Duration::from_secs(5), delaying_transport.wait_captured()).await + }) + .expect("startup preload response should be captured before writer B commits"); + + let db_b = + harness.open_db_on_engine(&runtime, engine.clone(), &harness.actor_id, config.clone()); + sqlite_exec( + db_b.as_ptr(), + "UPDATE preload_rows SET payload = 'writer-b-new-value' WHERE id = 48;", + ) + .expect("fresh writer should commit while startup preload response is held"); + + delaying_transport.release(); + let opened_a = opening + .join() + .expect("startup preload open thread should not panic"); + if let Ok(db_a) = opened_a { + let stale_update = sqlite_exec( + db_a.as_ptr(), + "UPDATE preload_rows SET payload = 'stale-writer-value' WHERE id = 48;", + ); + assert!( + stale_update + .as_ref() + .expect_err("stale writer must fail after delayed startup preload") + .contains("disk I/O error"), + "unexpected stale writer error: {stale_update:?}" + ); + assert!( + direct_vfs_ctx(&db_a) + .clone_fatal_error() + .is_some_and(|message| message.contains("head fence mismatch")), + "stale VFS should retain the head fence mismatch" + ); + drop(db_a); + } else { + let Err(err) = opened_a else { + unreachable!("startup preload result was already checked"); + }; + assert!( + err.contains("disk I/O error"), + "unexpected startup preload open error: {err}" + ); + } + drop(db_b); + + let db_c = harness.open_db_on_engine(&runtime, engine, &harness.actor_id, config); + assert_eq!( + sqlite_query_text(db_c.as_ptr(), "PRAGMA integrity_check;") + .expect("integrity check should run after stale startup preload failed"), + "ok" + ); + assert_eq!( + sqlite_query_text( + db_c.as_ptr(), + "SELECT payload FROM preload_rows WHERE id = 48;" + ) + .expect("fresh reopen should see writer B value"), + "writer-b-new-value" + ); +} + +#[test] +fn warm_pidx_stale_read_then_rmw_commit_produces_malformed_db() { + // Reproduction attempt for the Depot warm-PIDX-cache hazard described in + // `.agent/notes/sqlite-corruption-deep-dive-findings.md` (Tier 1B / 2C): + // + // 1. Handle A populates the shared `Db` instance's warm PIDX cache while building real + // SQLite schema + rows. The warm cache records `(pgno -> owner_txid)` for sqlite_master + // and several root pages. + // 2. A second, independent `Db` instance over the same UniversalDB performs a real + // schema-mutating commit (`CREATE INDEX`). This advances the head and rewrites pgno 1 + // in FDB, but only updates the second `Db`'s cache; the shared `Db`'s warm PIDX keeps + // the pre-mutation `(pgno=1 -> old txid)` row. + // 3. A fresh VFS handle B opens through the shared `Db`. B starts with + // `state.head_txid = None`, so the read fence cannot trigger. The warm PIDX hands + // back pgno 1 bytes from the old owner DELTA blob paired with the current head txid. + // 4. B runs SQLite `CREATE INDEX`, forcing an RMW on pgno 1 (sqlite_master) computed + // from the stale bytes. B commits with `expected_head_txid = current head`, the + // fence passes, and the stale-derived pgno 1 lands durably. + // 5. All handles drop; the test reopens with a fresh `Db` (no warm cache) and runs + // `PRAGMA integrity_check` plus schema queries to look for corruption signatures. + use depot::conveyer::Db; + use depot::types::{CommitOptions, DirtyPage}; + use rivet_pools::__rivet_util::Id; + use rivet_pools::NodeId; + + struct PinnedDbTransport { + db: Arc, + actor_id: String, + } + + #[async_trait] + impl SqliteTransport for PinnedDbTransport { + async fn get_pages( + &self, + request: protocol::SqliteGetPagesRequest, + ) -> anyhow::Result { + // The pinned `Db` is the only routing target; reject actor-id drift loudly. + assert_eq!(request.actor_id, self.actor_id, "pinned transport actor id"); + match self + .db + .get_pages_with_options( + request.pgnos.clone(), + depot::types::GetPagesOptions { + expected_head_txid: request.expected_head_txid, + ..Default::default() + }, + ) + .await + { + Ok(result) => Ok(protocol::SqliteGetPagesResponse::SqliteGetPagesOk( + protocol::SqliteGetPagesOk { + pages: result + .pages + .into_iter() + .map(|page| protocol::SqliteFetchedPage { + pgno: page.pgno, + bytes: page.bytes, + }) + .collect(), + head_txid: Some(result.head_txid), + }, + )), + Err(err) => Ok(protocol::SqliteGetPagesResponse::SqliteErrorResponse( + sqlite_error_response(&err), + )), + } + } + + async fn commit( + &self, + request: protocol::SqliteCommitRequest, + ) -> anyhow::Result { + assert_eq!(request.actor_id, self.actor_id, "pinned transport actor id"); + let dirty_pages = request + .dirty_pages + .into_iter() + .map(|page| DirtyPage { + pgno: page.pgno, + bytes: page.bytes, + }) + .collect::>(); + match self + .db + .commit_with_options( + dirty_pages, + request.db_size_pages, + request.now_ms, + CommitOptions { + expected_head_txid: request.expected_head_txid, + }, + ) + .await + { + Ok(result) => Ok(protocol::SqliteCommitResponse::SqliteCommitOk( + protocol::SqliteCommitOk { + head_txid: Some(result.head_txid), + }, + )), + Err(err) => Ok(protocol::SqliteCommitResponse::SqliteErrorResponse( + sqlite_error_response(&err), + )), + } + } + } + + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let udb = engine.depot_database(); + + // Two independent `Db` instances sharing the same actor id and UniversalDB. + // `db_shared` is the cache-warming `Db` reused by handle A and (later) handle B. + // `db_writer` is the "second pegboard-envoy WS conn" that commits without touching + // `db_shared`'s warm PIDX cache. + let db_shared = Arc::new(Db::new( + udb.clone(), + Id::nil(), + harness.actor_id.clone(), + NodeId::new(), + )); + let db_writer = Arc::new(Db::new( + udb.clone(), + Id::nil(), + harness.actor_id.clone(), + NodeId::new(), + )); + + let config = VfsConfig { + page_cache_mode: SqliteVfsPageCacheMode::All, + ..VfsConfig::default() + }; + + // Step 1: handle A builds a real schema with enough rows to span btree internal pages. + // A's commits flow through `db_shared`, populating its warm PIDX cache. + let transport_a: Arc = Arc::new(PinnedDbTransport { + db: db_shared.clone(), + actor_id: harness.actor_id.clone(), + }); + let db_a = harness.open_db_with_transport( + &runtime, + transport_a.clone(), + &harness.actor_id, + config.clone(), + ); + sqlite_exec( + db_a.as_ptr(), + "CREATE TABLE t1 (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);", + ) + .expect("create table should succeed"); + sqlite_exec(db_a.as_ptr(), "CREATE INDEX idx_t1_payload ON t1(payload);") + .expect("create initial index should succeed"); + sqlite_exec(db_a.as_ptr(), "BEGIN;").expect("begin should succeed"); + for id in 1..=2000 { + sqlite_exec( + db_a.as_ptr(), + &format!( + "INSERT INTO t1 (id, payload) VALUES ({id}, printf('payload-%05d-%s', {id}, hex(zeroblob(64))));" + ), + ) + .expect("insert should succeed"); + } + sqlite_exec(db_a.as_ptr(), "COMMIT;").expect("commit should succeed"); + + let page_count_after_a = sqlite_query_i64(db_a.as_ptr(), "PRAGMA page_count;") + .expect("page_count should be readable"); + assert!( + page_count_after_a >= 32, + "setup should produce enough pages to span btree internals, got {page_count_after_a}" + ); + // Step 1b: reads that exercise sqlite_master + index root + table btree internals so + // the shared `Db` PIDX cache is warm for the pages production probes would touch. + let _row_count = + sqlite_query_i64(db_a.as_ptr(), "SELECT COUNT(*) FROM t1;").expect("count should succeed"); + let _sample = sqlite_query_i64(db_a.as_ptr(), "SELECT id FROM t1 WHERE id = 1;") + .expect("indexed lookup should succeed"); + + // Snapshot the post-A schema so we can detect mutations or rollback after the bug. + let schema_after_a = sqlite_query_text( + db_a.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ) + .expect("schema query should succeed"); + assert!( + schema_after_a.contains("idx_t1_payload"), + "baseline schema should include the seed index: {schema_after_a}" + ); + drop(db_a); + + // Step 1c: directly call `db_shared.get_pages(...)` for the low-numbered pages that + // future B reads will request. SQLite + the VFS page cache may have suppressed the + // per-page transport calls during A's operations, leaving the warm PIDX cache empty. + // We need entries in `db_shared.cache_snapshot.pidx` for the warm-cache hazard. + let warm_pgnos: Vec = (1..=10).collect(); + let warm_result = + runtime + .block_on(db_shared.get_pages_with_options( + warm_pgnos.clone(), + depot::types::GetPagesOptions::default(), + )) + .expect("warm read through db_shared should succeed"); + tracing::info!( + ?warm_pgnos, + warm_head_txid = warm_result.head_txid, + "warm pidx repro: warmed db_shared cache via direct get_pages" + ); + + // Step 2: an independent `Db` instance commits schema mutations on the same actor. + // This represents the other pegboard-envoy WS conn in production. The commit rewrites + // pgno 1 (sqlite_master) in FDB and advances the head, but `db_shared`'s warm PIDX + // cache is not touched. + let transport_writer: Arc = Arc::new(PinnedDbTransport { + db: db_writer.clone(), + actor_id: harness.actor_id.clone(), + }); + let db_writer_handle = harness.open_db_with_transport( + &runtime, + transport_writer.clone(), + &harness.actor_id, + config.clone(), + ); + sqlite_exec( + db_writer_handle.as_ptr(), + "CREATE INDEX idx_t1_payload2 ON t1(payload, id);", + ) + .expect("writer CREATE INDEX should commit"); + sqlite_exec(db_writer_handle.as_ptr(), "BEGIN;").expect("begin should succeed"); + for id in 1..=200 { + sqlite_exec( + db_writer_handle.as_ptr(), + &format!("UPDATE t1 SET payload = 'writer-rewrite-{id}' WHERE id = {id};"), + ) + .expect("writer update should succeed"); + } + sqlite_exec(db_writer_handle.as_ptr(), "COMMIT;").expect("writer commit should succeed"); + let writer_head = runtime + .block_on( + db_writer.get_pages_with_options(vec![1], depot::types::GetPagesOptions::default()), + ) + .expect("writer head fetch should succeed") + .head_txid; + drop(db_writer_handle); + + // Diagnostic: inspect the shared `Db` warm PIDX cache before B reads. If pgno 1 + // is present with the OLD owner txid (from A's commits) while `db_writer`'s cache + // shows pgno 1 with the NEW owner txid (from the writer commit), the warm-cache + // hazard is set up. + let shared_cache_pidx = runtime + .block_on(db_shared.branch_cache_snapshot_for_test()) + .map(|(_, _, _, pidx)| pidx) + .unwrap_or_default(); + let writer_cache_pidx = runtime + .block_on(db_writer.branch_cache_snapshot_for_test()) + .map(|(_, _, _, pidx)| pidx) + .unwrap_or_default(); + let shared_pg1 = shared_cache_pidx + .iter() + .find(|(pgno, _)| *pgno == 1) + .copied(); + let writer_pg1 = writer_cache_pidx + .iter() + .find(|(pgno, _)| *pgno == 1) + .copied(); + eprintln!( + "warm-pidx-repro: db_shared cache rows = {} (pgno 1 owner = {:?}), db_writer cache rows = {} (pgno 1 owner = {:?})", + shared_cache_pidx.len(), + shared_pg1, + writer_cache_pidx.len(), + writer_pg1, + ); + + // Step 3: open a fresh handle B through the warm `db_shared`. The startup preload + // requests pgno 1 with `expected_head_txid = None`, hitting the warm PIDX cache and + // returning old pgno 1 bytes paired with the current head. B then adopts the current + // head as its own. + let db_b = harness.open_db_with_transport( + &runtime, + transport_a.clone(), + &harness.actor_id, + config.clone(), + ); + let schema_seen_by_b = sqlite_query_text( + db_b.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ) + .expect("schema query through B should succeed"); + tracing::info!( + schema_after_a = %schema_after_a, + schema_seen_by_b = %schema_seen_by_b, + writer_head = ?writer_head, + "warm pidx repro: schema visibility through stale Db" + ); + + // Step 4: RMW on pgno 1 from the stale view. SQLite rewrites sqlite_master with a new + // schema row, allocates a new root, and commits. With the warm cache hazard the + // commit's expected_head_txid matches the current head, the fence passes, and the + // stale-derived pgno 1 lands durably. + let stale_create = sqlite_exec( + db_b.as_ptr(), + "CREATE INDEX idx_t1_payload_stale ON t1(payload, id, id);", + ); + let mut stale_commit_observed = stale_create.is_ok(); + if let Err(message) = &stale_create { + tracing::info!(?message, "stale RMW create index failed"); + } + // Force a freelist mutation via DELETE so the stale RMW may also rewrite the + // freelist trunk page from stale base bytes. + let stale_delete = sqlite_exec(db_b.as_ptr(), "DELETE FROM t1 WHERE id <= 50;"); + if stale_delete.is_ok() { + stale_commit_observed = true; + } + + drop(db_b); + drop(db_shared); + drop(db_writer); + runtime.block_on(engine.evict_actor_db(&harness.actor_id)); + + // Step 5: fresh reopen with no warm cache. Run integrity check + schema scan. + let db_c = harness.open_db_on_engine(&runtime, engine, &harness.actor_id, config); + let integrity = sqlite_query_text(db_c.as_ptr(), "PRAGMA integrity_check;"); + let quick_check = sqlite_query_text(db_c.as_ptr(), "PRAGMA quick_check;"); + let schema_after_reopen = sqlite_query_text( + db_c.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ); + let row_count = sqlite_query_i64(db_c.as_ptr(), "SELECT COUNT(*) FROM t1;"); + + // Exercise the schema and indexes more thoroughly so SQLite has to walk btree + // internal pages, indexes, and intermediate result sets. The literal runtime + // "database disk image is malformed" string typically surfaces during page + // reads through these paths rather than via PRAGMA integrity_check. + // + // Ordering matters: passive scans must run BEFORE any REINDEX. REINDEX rebuilds + // the index from a fresh table scan and writes a new root page, which would mask + // the malformation for any probe running afterwards. Probes that walk the stale + // index (page 165) are sequenced first. + let select_indexed_by_grouped = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM (SELECT payload, COUNT(*) FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00000' AND 'payload-99999' GROUP BY payload);", + ); + let select_indexed_by_count_star = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00000' AND 'payload-99999';", + ); + let select_distinct_via_index = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM (SELECT DISTINCT payload FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload >= 'payload-');", + ); + let update_through_index = sqlite_exec( + db_c.as_ptr(), + "UPDATE t1 SET payload = payload || '-tag' WHERE payload BETWEEN 'payload-00000' AND 'payload-99999';", + ); + let delete_through_index = sqlite_exec( + db_c.as_ptr(), + "DELETE FROM t1 WHERE payload IN (SELECT payload FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00500' AND 'payload-00600');", + ); + let analyze_stale = sqlite_exec(db_c.as_ptr(), "ANALYZE idx_t1_payload_stale;"); + let reindex_stale = sqlite_exec(db_c.as_ptr(), "REINDEX idx_t1_payload_stale;"); + + let runtime_probes: [(&str, Result); 7] = [ + ( + "select_indexed_by_grouped", + select_indexed_by_grouped + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "select_indexed_by_count_star", + select_indexed_by_count_star + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "select_distinct_via_index", + select_distinct_via_index + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "update_through_index", + update_through_index + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "delete_through_index", + delete_through_index + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "analyze", + analyze_stale + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "reindex_stale", + reindex_stale + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ]; + + let mut runtime_corrupt_observed: Option = None; + for (label, result) in &runtime_probes { + if let Err(message) = result { + if message.contains("database disk image is malformed") + && runtime_corrupt_observed.is_none() + { + runtime_corrupt_observed = Some(format!("{label}: {message}")); + } + } + } + + // Escalation: if none of the structured probes surfaced the literal substring, + // try a writable_schema toggle plus direct keyed lookups at probable leaf + // boundary payloads to force reads of cells 0 / 15 on page 165. + let mut escalation_probes: Vec<(String, Result)> = Vec::new(); + if runtime_corrupt_observed.is_none() { + let pragma_writable_schema_on = sqlite_exec(db_c.as_ptr(), "PRAGMA writable_schema = ON;"); + escalation_probes.push(( + "pragma_writable_schema_on".to_string(), + pragma_writable_schema_on + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + )); + for boundary_id in [125u32, 126, 127, 250, 251, 252, 375, 376, 377] { + let q = format!( + "SELECT id FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload = printf('payload-%05d-%s', {boundary_id}, hex(zeroblob(64)));" + ); + let result = sqlite_query_i64(db_c.as_ptr(), &q); + escalation_probes.push(( + format!("direct_page165_boundary_{boundary_id}"), + result + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + )); + } + for (label, result) in &escalation_probes { + if let Err(message) = result { + if message.contains("database disk image is malformed") + && runtime_corrupt_observed.is_none() + { + runtime_corrupt_observed = Some(format!("{label}: {message}")); + } + } + } + } + + tracing::warn!( + ?integrity, + ?quick_check, + ?schema_after_reopen, + ?row_count, + ?select_indexed_by_grouped, + ?select_indexed_by_count_star, + ?select_distinct_via_index, + ?update_through_index, + ?delete_through_index, + ?analyze_stale, + ?reindex_stale, + ?runtime_corrupt_observed, + stale_commit_observed, + "warm pidx repro: final reopen state" + ); + eprintln!( + "warm-pidx-repro: integrity={integrity:?} quick_check={quick_check:?} \ + schema_after_reopen={schema_after_reopen:?} row_count={row_count:?} \ + select_indexed_by_grouped={select_indexed_by_grouped:?} \ + select_indexed_by_count_star={select_indexed_by_count_star:?} \ + select_distinct_via_index={select_distinct_via_index:?} \ + update_through_index={update_through_index:?} \ + delete_through_index={delete_through_index:?} \ + analyze_stale={analyze_stale:?} reindex_stale={reindex_stale:?} \ + runtime_corrupt_observed={runtime_corrupt_observed:?} \ + stale_commit_observed={stale_commit_observed}" + ); + for (label, result) in &runtime_probes { + eprintln!("warm-pidx-repro: probe[{label}] = {result:?}"); + } + for (label, result) in &escalation_probes { + eprintln!("warm-pidx-repro: escalation[{label}] = {result:?}"); + } + + // Primary assertion: detect any malformed-DB signature. + // + // Any non-"ok" result from `PRAGMA integrity_check` or `PRAGMA quick_check` is a + // malformed-DB signal; SQLite returns "ok" exactly when both pass. SQLite reports + // btree structure violations (e.g. "Tree N page X cell Y: 2nd reference to page Z"), + // orphan rows, dangling pointers, header mismatches, etc. as multi-line text. + let malformed_keywords = [ + "malformed", + "corrupt", + "wrong page type", + "reference to page", + "out of order", + "page is never used", + "missing from index", + "row count", + "unordered", + ]; + let is_malformed_text = |text: &str| -> bool { + text != "ok" + && (malformed_keywords.iter().any(|k| text.contains(k)) + || text.contains("***") + || text.lines().count() > 1) + }; + let mut malformed = false; + for probe in [&integrity, &quick_check] { + match probe { + Ok(text) => { + if is_malformed_text(text) { + malformed = true; + } + } + Err(message) => { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + } + } + if let Err(message) = &row_count { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + if let Err(message) = &schema_after_reopen { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + if runtime_corrupt_observed.is_some() { + malformed = true; + } + + if malformed { + panic!( + "REPRO: warm-pidx-cache hazard produced a malformed DB after reopen. \ + integrity={integrity:?} quick_check={quick_check:?} \ + schema_after_reopen={schema_after_reopen:?} row_count={row_count:?} \ + runtime_corrupt_observed={runtime_corrupt_observed:?}. \ + Likely landing site: depot-client/src/vfs.rs (VFS adopts response head_txid \ + unconditionally around line 1396-1399) + depot/src/conveyer/read.rs:413 \ + (warm PIDX returns stale bytes paired with current head)." + ); + } + + // No repro on this run. The test stands as a documented attempt and regression guard; + // see the doc comment at the top for production landing sites. Variations to try next: + // - VACUUM through `db_writer` to renumber pages, then warm-stale-read a btree + // internal page rather than sqlite_master. + // - Larger DB (>= 20k rows) so the freelist trunk page is referenced by the warm + // cache and the stale RMW lands on the freelist instead of sqlite_master. + // - Force PRAGMA auto_vacuum = FULL during open so page renumbering happens + // transparently after the writer's DELETE. + // - Replicate the production batch-atomic probe sequence (CREATE TABLE + // __rivet_batch_probe / INSERT / DELETE / DROP) exactly to mirror the 30-second + // post-Started window. + // Note: once handle B's open-time read populates state.head_txid from the stale warm + // cache response, all subsequent get_pages still pass the fence (the response + // advertised the current head). The dangerous `(stale bytes, current head)` pair is + // what the test deliberately preserves; if the underlying repro hits it should show up + // in integrity_check or schema queries after the fresh reopen above. + tracing::info!( + "warm pidx repro: no malformed DB observed on this run; see eprintln for details" + ); +} + +#[test] +fn warm_pidx_stale_read_then_rmw_commit_natural_repro() { + // Sibling of `warm_pidx_stale_read_then_rmw_commit_produces_malformed_db` that + // removes the artificial Step 1c warm-up call (`db_shared.get_pages_with_options`). + // Production code never calls `get_pages_with_options` directly; the warm PIDX cache + // is supposed to be populated naturally by commits flowing through `db_shared` + // (see `apply.rs` `pidx.insert(pgno, result.txid)` for each dirty pgno on commit). + // This test checks whether the warm-cache hazard reproduces under those natural + // conditions, i.e. with cache entries only from commit-side population. + use depot::conveyer::Db; + use depot::types::{CommitOptions, DirtyPage}; + use rivet_pools::__rivet_util::Id; + use rivet_pools::NodeId; + + struct PinnedDbTransport { + db: Arc, + actor_id: String, + } + + #[async_trait] + impl SqliteTransport for PinnedDbTransport { + async fn get_pages( + &self, + request: protocol::SqliteGetPagesRequest, + ) -> anyhow::Result { + assert_eq!(request.actor_id, self.actor_id, "pinned transport actor id"); + match self + .db + .get_pages_with_options( + request.pgnos.clone(), + depot::types::GetPagesOptions { + expected_head_txid: request.expected_head_txid, + ..Default::default() + }, + ) + .await + { + Ok(result) => Ok(protocol::SqliteGetPagesResponse::SqliteGetPagesOk( + protocol::SqliteGetPagesOk { + pages: result + .pages + .into_iter() + .map(|page| protocol::SqliteFetchedPage { + pgno: page.pgno, + bytes: page.bytes, + }) + .collect(), + head_txid: Some(result.head_txid), + }, + )), + Err(err) => Ok(protocol::SqliteGetPagesResponse::SqliteErrorResponse( + sqlite_error_response(&err), + )), + } + } + + async fn commit( + &self, + request: protocol::SqliteCommitRequest, + ) -> anyhow::Result { + assert_eq!(request.actor_id, self.actor_id, "pinned transport actor id"); + let dirty_pages = request + .dirty_pages + .into_iter() + .map(|page| DirtyPage { + pgno: page.pgno, + bytes: page.bytes, + }) + .collect::>(); + match self + .db + .commit_with_options( + dirty_pages, + request.db_size_pages, + request.now_ms, + CommitOptions { + expected_head_txid: request.expected_head_txid, + }, + ) + .await + { + Ok(result) => Ok(protocol::SqliteCommitResponse::SqliteCommitOk( + protocol::SqliteCommitOk { + head_txid: Some(result.head_txid), + }, + )), + Err(err) => Ok(protocol::SqliteCommitResponse::SqliteErrorResponse( + sqlite_error_response(&err), + )), + } + } + } + + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let udb = engine.depot_database(); + + let db_shared = Arc::new(Db::new( + udb.clone(), + Id::nil(), + harness.actor_id.clone(), + NodeId::new(), + )); + let db_writer = Arc::new(Db::new( + udb.clone(), + Id::nil(), + harness.actor_id.clone(), + NodeId::new(), + )); + + let config = VfsConfig { + page_cache_mode: SqliteVfsPageCacheMode::All, + ..VfsConfig::default() + }; + + // Step 1: handle A builds a real schema with enough rows to span btree internal pages. + // A's commits flow through `db_shared`, populating its warm PIDX cache naturally via + // the commit-side `pidx.insert(pgno, result.txid)` path. + let transport_a: Arc = Arc::new(PinnedDbTransport { + db: db_shared.clone(), + actor_id: harness.actor_id.clone(), + }); + let db_a = harness.open_db_with_transport( + &runtime, + transport_a.clone(), + &harness.actor_id, + config.clone(), + ); + sqlite_exec( + db_a.as_ptr(), + "CREATE TABLE t1 (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);", + ) + .expect("create table should succeed"); + sqlite_exec(db_a.as_ptr(), "CREATE INDEX idx_t1_payload ON t1(payload);") + .expect("create initial index should succeed"); + sqlite_exec(db_a.as_ptr(), "BEGIN;").expect("begin should succeed"); + for id in 1..=2000 { + sqlite_exec( + db_a.as_ptr(), + &format!( + "INSERT INTO t1 (id, payload) VALUES ({id}, printf('payload-%05d-%s', {id}, hex(zeroblob(64))));" + ), + ) + .expect("insert should succeed"); + } + sqlite_exec(db_a.as_ptr(), "COMMIT;").expect("commit should succeed"); + + let page_count_after_a = sqlite_query_i64(db_a.as_ptr(), "PRAGMA page_count;") + .expect("page_count should be readable"); + assert!( + page_count_after_a >= 32, + "setup should produce enough pages to span btree internals, got {page_count_after_a}" + ); + let _row_count = + sqlite_query_i64(db_a.as_ptr(), "SELECT COUNT(*) FROM t1;").expect("count should succeed"); + let _sample = sqlite_query_i64(db_a.as_ptr(), "SELECT id FROM t1 WHERE id = 1;") + .expect("indexed lookup should succeed"); + + let schema_after_a = sqlite_query_text( + db_a.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ) + .expect("schema query should succeed"); + assert!( + schema_after_a.contains("idx_t1_payload"), + "baseline schema should include the seed index: {schema_after_a}" + ); + drop(db_a); + + // NOTE: Step 1c removed. No explicit `db_shared.get_pages_with_options(...)` call. + // We rely entirely on commit-side `pidx.insert(...)` from handle A's commits to + // populate `db_shared.cache_snapshot.pidx`. + + // Step 2: an independent `Db` instance commits schema mutations on the same actor. + let transport_writer: Arc = Arc::new(PinnedDbTransport { + db: db_writer.clone(), + actor_id: harness.actor_id.clone(), + }); + let db_writer_handle = harness.open_db_with_transport( + &runtime, + transport_writer.clone(), + &harness.actor_id, + config.clone(), + ); + sqlite_exec( + db_writer_handle.as_ptr(), + "CREATE INDEX idx_t1_payload2 ON t1(payload, id);", + ) + .expect("writer CREATE INDEX should commit"); + sqlite_exec(db_writer_handle.as_ptr(), "BEGIN;").expect("begin should succeed"); + for id in 1..=200 { + sqlite_exec( + db_writer_handle.as_ptr(), + &format!("UPDATE t1 SET payload = 'writer-rewrite-{id}' WHERE id = {id};"), + ) + .expect("writer update should succeed"); + } + sqlite_exec(db_writer_handle.as_ptr(), "COMMIT;").expect("writer commit should succeed"); + let writer_head = runtime + .block_on( + db_writer.get_pages_with_options(vec![1], depot::types::GetPagesOptions::default()), + ) + .expect("writer head fetch should succeed") + .head_txid; + drop(db_writer_handle); + + // Diagnostic: inspect the shared `Db` warm PIDX cache. Without the explicit warm-up + // call, this shows whether commits alone populate the cache for the test's workload. + let shared_cache_pidx = runtime + .block_on(db_shared.branch_cache_snapshot_for_test()) + .map(|(_, _, _, pidx)| pidx) + .unwrap_or_default(); + let writer_cache_pidx = runtime + .block_on(db_writer.branch_cache_snapshot_for_test()) + .map(|(_, _, _, pidx)| pidx) + .unwrap_or_default(); + let shared_pg1 = shared_cache_pidx + .iter() + .find(|(pgno, _)| *pgno == 1) + .copied(); + let writer_pg1 = writer_cache_pidx + .iter() + .find(|(pgno, _)| *pgno == 1) + .copied(); + let shared_pgnos: Vec = shared_cache_pidx.iter().map(|(p, _)| *p).collect(); + eprintln!( + "warm-pidx-natural-repro: db_shared cache rows = {} (pgno 1 owner = {:?}), db_writer cache rows = {} (pgno 1 owner = {:?}), db_shared pgnos = {:?}", + shared_cache_pidx.len(), + shared_pg1, + writer_cache_pidx.len(), + writer_pg1, + shared_pgnos, + ); + + // Step 3: open a fresh handle B through `db_shared`. + let db_b = harness.open_db_with_transport( + &runtime, + transport_a.clone(), + &harness.actor_id, + config.clone(), + ); + let schema_seen_by_b = sqlite_query_text( + db_b.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ) + .expect("schema query through B should succeed"); + tracing::info!( + schema_after_a = %schema_after_a, + schema_seen_by_b = %schema_seen_by_b, + writer_head = ?writer_head, + "warm pidx natural repro: schema visibility through stale Db" + ); + + // Step 4: RMW on pgno 1 from the (possibly stale) view. + let stale_create = sqlite_exec( + db_b.as_ptr(), + "CREATE INDEX idx_t1_payload_stale ON t1(payload, id, id);", + ); + let mut stale_commit_observed = stale_create.is_ok(); + if let Err(message) = &stale_create { + tracing::info!(?message, "stale RMW create index failed"); + } + let stale_delete = sqlite_exec(db_b.as_ptr(), "DELETE FROM t1 WHERE id <= 50;"); + if stale_delete.is_ok() { + stale_commit_observed = true; + } + + drop(db_b); + drop(db_shared); + drop(db_writer); + runtime.block_on(engine.evict_actor_db(&harness.actor_id)); + + // Step 5: fresh reopen with no warm cache. Run integrity check + schema scan + the + // post-reopen probes that surface `database disk image is malformed` at runtime. + let db_c = harness.open_db_on_engine(&runtime, engine, &harness.actor_id, config); + let integrity = sqlite_query_text(db_c.as_ptr(), "PRAGMA integrity_check;"); + let quick_check = sqlite_query_text(db_c.as_ptr(), "PRAGMA quick_check;"); + let schema_after_reopen = sqlite_query_text( + db_c.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ); + let row_count = sqlite_query_i64(db_c.as_ptr(), "SELECT COUNT(*) FROM t1;"); + + // Ordering matters: passive scans before REINDEX (which would mask malformation). + let select_indexed_by_grouped = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM (SELECT payload, COUNT(*) FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00000' AND 'payload-99999' GROUP BY payload);", + ); + let select_indexed_by_count_star = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00000' AND 'payload-99999';", + ); + let select_distinct_via_index = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM (SELECT DISTINCT payload FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload >= 'payload-');", + ); + let update_through_index = sqlite_exec( + db_c.as_ptr(), + "UPDATE t1 SET payload = payload || '-tag' WHERE payload BETWEEN 'payload-00000' AND 'payload-99999';", + ); + let delete_through_index = sqlite_exec( + db_c.as_ptr(), + "DELETE FROM t1 WHERE payload IN (SELECT payload FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00500' AND 'payload-00600');", + ); + let analyze_stale = sqlite_exec(db_c.as_ptr(), "ANALYZE idx_t1_payload_stale;"); + let reindex_stale = sqlite_exec(db_c.as_ptr(), "REINDEX idx_t1_payload_stale;"); + + let runtime_probes: [(&str, Result); 7] = [ + ( + "select_indexed_by_grouped", + select_indexed_by_grouped + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "select_indexed_by_count_star", + select_indexed_by_count_star + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "select_distinct_via_index", + select_distinct_via_index + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "update_through_index", + update_through_index + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "delete_through_index", + delete_through_index + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "analyze", + analyze_stale + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "reindex_stale", + reindex_stale + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ]; + + let mut runtime_corrupt_observed: Option = None; + for (label, result) in &runtime_probes { + if let Err(message) = result { + if message.contains("database disk image is malformed") + && runtime_corrupt_observed.is_none() + { + runtime_corrupt_observed = Some(format!("{label}: {message}")); + } + } + } + + let mut escalation_probes: Vec<(String, Result)> = Vec::new(); + if runtime_corrupt_observed.is_none() { + let pragma_writable_schema_on = sqlite_exec(db_c.as_ptr(), "PRAGMA writable_schema = ON;"); + escalation_probes.push(( + "pragma_writable_schema_on".to_string(), + pragma_writable_schema_on + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + )); + for boundary_id in [125u32, 126, 127, 250, 251, 252, 375, 376, 377] { + let q = format!( + "SELECT id FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload = printf('payload-%05d-%s', {boundary_id}, hex(zeroblob(64)));" + ); + let result = sqlite_query_i64(db_c.as_ptr(), &q); + escalation_probes.push(( + format!("direct_page165_boundary_{boundary_id}"), + result + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + )); + } + for (label, result) in &escalation_probes { + if let Err(message) = result { + if message.contains("database disk image is malformed") + && runtime_corrupt_observed.is_none() + { + runtime_corrupt_observed = Some(format!("{label}: {message}")); + } + } + } + } + + tracing::warn!( + ?integrity, + ?quick_check, + ?schema_after_reopen, + ?row_count, + ?select_indexed_by_grouped, + ?select_indexed_by_count_star, + ?select_distinct_via_index, + ?update_through_index, + ?delete_through_index, + ?analyze_stale, + ?reindex_stale, + ?runtime_corrupt_observed, + stale_commit_observed, + "warm pidx natural repro: final reopen state" + ); + eprintln!( + "warm-pidx-natural-repro: integrity={integrity:?} quick_check={quick_check:?} \ + schema_after_reopen={schema_after_reopen:?} row_count={row_count:?} \ + select_indexed_by_grouped={select_indexed_by_grouped:?} \ + select_indexed_by_count_star={select_indexed_by_count_star:?} \ + select_distinct_via_index={select_distinct_via_index:?} \ + update_through_index={update_through_index:?} \ + delete_through_index={delete_through_index:?} \ + analyze_stale={analyze_stale:?} reindex_stale={reindex_stale:?} \ + runtime_corrupt_observed={runtime_corrupt_observed:?} \ + stale_commit_observed={stale_commit_observed}" + ); + for (label, result) in &runtime_probes { + eprintln!("warm-pidx-natural-repro: probe[{label}] = {result:?}"); + } + for (label, result) in &escalation_probes { + eprintln!("warm-pidx-natural-repro: escalation[{label}] = {result:?}"); + } + + let malformed_keywords = [ + "malformed", + "corrupt", + "wrong page type", + "reference to page", + "out of order", + "page is never used", + "missing from index", + "row count", + "unordered", + ]; + let is_malformed_text = |text: &str| -> bool { + text != "ok" + && (malformed_keywords.iter().any(|k| text.contains(k)) + || text.contains("***") + || text.lines().count() > 1) + }; + let mut malformed = false; + for probe in [&integrity, &quick_check] { + match probe { + Ok(text) => { + if is_malformed_text(text) { + malformed = true; + } + } + Err(message) => { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + } + } + if let Err(message) = &row_count { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + if let Err(message) = &schema_after_reopen { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + if runtime_corrupt_observed.is_some() { + malformed = true; + } + + if malformed { + panic!( + "REPRO (natural): warm-pidx-cache hazard produced a malformed DB after reopen \ + WITHOUT explicit cache warm-up. integrity={integrity:?} quick_check={quick_check:?} \ + schema_after_reopen={schema_after_reopen:?} row_count={row_count:?} \ + runtime_corrupt_observed={runtime_corrupt_observed:?}." + ); + } + + tracing::info!( + "warm pidx natural repro: no malformed DB observed on this run; see eprintln for details" + ); +} + +#[test] +fn warm_pidx_stale_read_then_rmw_commit_via_natural_reopen() { + // Sibling of `warm_pidx_stale_read_then_rmw_commit_natural_repro` that populates + // `db_shared.cache_snapshot.pidx` entirely through natural VFS open/SELECT/commit + // paths. The previous natural test showed `db_shared cache rows = 0` because + // commits route through `PinnedDbTransport::commit` -> `db_shared.commit_with_options`, + // which populates the writer-side cache through `apply.rs`, but the `db_shared` cache + // remained empty because none of A's commits routed back through `db_shared`'s own + // get_pages path during steady-state SQLite operations (VFS-level caches absorbed + // them). + // + // This variant adds a "handle A_warm" step: an extra VFS handle opened through + // `db_shared` after A1's commits land. Its startup preload calls + // `db_shared.get_pages_with_options(vec![1..=N], expected_head_txid=None)` through + // the natural `fetch_initial_pages_for_registration` path, which populates + // `db_shared.cache_snapshot.pidx`. A few representative SELECTs follow to give the + // VFS more cache misses to route through depot. Then A_warm drops, the writer + // advances FDB head, and handle C opens through the still-warm-but-now-stale + // `db_shared`. + // + // No explicit `db_shared.get_pages_with_options(...)` is called outside the + // transport. The cache is populated entirely by the production-equivalent startup + // preload path. + use depot::conveyer::Db; + use depot::types::{CommitOptions, DirtyPage}; + use rivet_pools::__rivet_util::Id; + use rivet_pools::NodeId; + + struct PinnedDbTransport { + db: Arc, + actor_id: String, + } + + #[async_trait] + impl SqliteTransport for PinnedDbTransport { + async fn get_pages( + &self, + request: protocol::SqliteGetPagesRequest, + ) -> anyhow::Result { + assert_eq!(request.actor_id, self.actor_id, "pinned transport actor id"); + match self + .db + .get_pages_with_options( + request.pgnos.clone(), + depot::types::GetPagesOptions { + expected_head_txid: request.expected_head_txid, + ..Default::default() + }, + ) + .await + { + Ok(result) => Ok(protocol::SqliteGetPagesResponse::SqliteGetPagesOk( + protocol::SqliteGetPagesOk { + pages: result + .pages + .into_iter() + .map(|page| protocol::SqliteFetchedPage { + pgno: page.pgno, + bytes: page.bytes, + }) + .collect(), + head_txid: Some(result.head_txid), + }, + )), + Err(err) => Ok(protocol::SqliteGetPagesResponse::SqliteErrorResponse( + sqlite_error_response(&err), + )), + } + } + + async fn commit( + &self, + request: protocol::SqliteCommitRequest, + ) -> anyhow::Result { + assert_eq!(request.actor_id, self.actor_id, "pinned transport actor id"); + let dirty_pages = request + .dirty_pages + .into_iter() + .map(|page| DirtyPage { + pgno: page.pgno, + bytes: page.bytes, + }) + .collect::>(); + match self + .db + .commit_with_options( + dirty_pages, + request.db_size_pages, + request.now_ms, + CommitOptions { + expected_head_txid: request.expected_head_txid, + }, + ) + .await + { + Ok(result) => Ok(protocol::SqliteCommitResponse::SqliteCommitOk( + protocol::SqliteCommitOk { + head_txid: Some(result.head_txid), + }, + )), + Err(err) => Ok(protocol::SqliteCommitResponse::SqliteErrorResponse( + sqlite_error_response(&err), + )), + } + } + } + + let runtime = direct_runtime(); + let harness = DirectEngineHarness::new(); + let engine = runtime.block_on(harness.open_engine()); + let udb = engine.depot_database(); + + let db_shared = Arc::new(Db::new( + udb.clone(), + Id::nil(), + harness.actor_id.clone(), + NodeId::new(), + )); + let db_writer = Arc::new(Db::new( + udb.clone(), + Id::nil(), + harness.actor_id.clone(), + NodeId::new(), + )); + + // Pump the preload up so the startup preload requests multiple pages, giving the + // natural warming step a wider footprint to populate `db_shared.cache_snapshot.pidx`. + let config = VfsConfig { + page_cache_mode: SqliteVfsPageCacheMode::All, + startup_preload_first_pages: true, + startup_preload_first_page_count: 32, + startup_preload_max_bytes: 32 * 4096, + ..VfsConfig::default() + }; + + let transport_shared: Arc = Arc::new(PinnedDbTransport { + db: db_shared.clone(), + actor_id: harness.actor_id.clone(), + }); + + // Step 1: handle A1 builds a real schema with enough rows to span btree internals. + let db_a = harness.open_db_with_transport( + &runtime, + transport_shared.clone(), + &harness.actor_id, + config.clone(), + ); + sqlite_exec( + db_a.as_ptr(), + "CREATE TABLE t1 (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);", + ) + .expect("create table should succeed"); + sqlite_exec(db_a.as_ptr(), "CREATE INDEX idx_t1_payload ON t1(payload);") + .expect("create initial index should succeed"); + sqlite_exec(db_a.as_ptr(), "BEGIN;").expect("begin should succeed"); + for id in 1..=2000 { + sqlite_exec( + db_a.as_ptr(), + &format!( + "INSERT INTO t1 (id, payload) VALUES ({id}, printf('payload-%05d-%s', {id}, hex(zeroblob(64))));" + ), + ) + .expect("insert should succeed"); + } + sqlite_exec(db_a.as_ptr(), "COMMIT;").expect("commit should succeed"); + + let page_count_after_a = sqlite_query_i64(db_a.as_ptr(), "PRAGMA page_count;") + .expect("page_count should be readable"); + assert!( + page_count_after_a >= 32, + "setup should produce enough pages to span btree internals, got {page_count_after_a}" + ); + let schema_after_a = sqlite_query_text( + db_a.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ) + .expect("schema query should succeed"); + assert!( + schema_after_a.contains("idx_t1_payload"), + "baseline schema should include the seed index: {schema_after_a}" + ); + drop(db_a); + + // Step 1.5: open A_warm through `db_shared` and let the startup preload populate + // `db_shared.cache_snapshot.pidx` naturally via `fetch_initial_pages_for_registration`. + // Also run representative SELECTs to widen the VFS cache miss footprint, sending more + // `get_pages` requests through `db_shared`. + let db_a_warm = harness.open_db_with_transport( + &runtime, + transport_shared.clone(), + &harness.actor_id, + config.clone(), + ); + let _ = sqlite_query_i64(db_a_warm.as_ptr(), "SELECT COUNT(*) FROM t1;") + .expect("count through A_warm should succeed"); + let _ = sqlite_query_i64( + db_a_warm.as_ptr(), + "SELECT id FROM t1 WHERE payload LIKE 'payload-00500%' LIMIT 1;", + ) + .expect("indexed lookup through A_warm should succeed"); + let _ = sqlite_query_i64(db_a_warm.as_ptr(), "SELECT id FROM t1 WHERE id = 1;") + .expect("pk lookup through A_warm should succeed"); + let _ = sqlite_query_text( + db_a_warm.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ) + .expect("schema scan through A_warm should succeed"); + drop(db_a_warm); + + // Snapshot db_shared.cache_snapshot.pidx after the natural warming step so we know + // whether the startup preload alone is enough to populate it. + let shared_cache_after_warm = runtime + .block_on(db_shared.branch_cache_snapshot_for_test()) + .map(|(_, _, _, pidx)| pidx) + .unwrap_or_default(); + let shared_pg1_after_warm = shared_cache_after_warm + .iter() + .find(|(pgno, _)| *pgno == 1) + .copied(); + let shared_pgnos_after_warm: Vec = + shared_cache_after_warm.iter().map(|(p, _)| *p).collect(); + eprintln!( + "warm-pidx-natural-reopen: after A_warm db_shared cache rows = {} (pgno 1 owner = {:?}), pgnos = {:?}", + shared_cache_after_warm.len(), + shared_pg1_after_warm, + shared_pgnos_after_warm, + ); + + // Step 2: independent `Db` instance commits schema mutations on the same actor. + let transport_writer: Arc = Arc::new(PinnedDbTransport { + db: db_writer.clone(), + actor_id: harness.actor_id.clone(), + }); + let db_writer_handle = harness.open_db_with_transport( + &runtime, + transport_writer.clone(), + &harness.actor_id, + config.clone(), + ); + sqlite_exec( + db_writer_handle.as_ptr(), + "CREATE INDEX idx_t1_payload2 ON t1(payload, id);", + ) + .expect("writer CREATE INDEX should commit"); + sqlite_exec(db_writer_handle.as_ptr(), "BEGIN;").expect("begin should succeed"); + for id in 1..=200 { + sqlite_exec( + db_writer_handle.as_ptr(), + &format!("UPDATE t1 SET payload = 'writer-rewrite-{id}' WHERE id = {id};"), + ) + .expect("writer update should succeed"); + } + sqlite_exec(db_writer_handle.as_ptr(), "COMMIT;").expect("writer commit should succeed"); + let writer_head = runtime + .block_on( + db_writer.get_pages_with_options(vec![1], depot::types::GetPagesOptions::default()), + ) + .expect("writer head fetch should succeed") + .head_txid; + drop(db_writer_handle); + + // Diagnostic: inspect the shared `Db` warm PIDX cache. Step 2 must NOT have touched + // it. If `shared_cache_after_warm` matches what we see now and writer pgno 1 owner is + // fresher, the hazard is set up. + let shared_cache_pidx = runtime + .block_on(db_shared.branch_cache_snapshot_for_test()) + .map(|(_, _, _, pidx)| pidx) + .unwrap_or_default(); + let writer_cache_pidx = runtime + .block_on(db_writer.branch_cache_snapshot_for_test()) + .map(|(_, _, _, pidx)| pidx) + .unwrap_or_default(); + let shared_pg1 = shared_cache_pidx + .iter() + .find(|(pgno, _)| *pgno == 1) + .copied(); + let writer_pg1 = writer_cache_pidx + .iter() + .find(|(pgno, _)| *pgno == 1) + .copied(); + let shared_pgnos: Vec = shared_cache_pidx.iter().map(|(p, _)| *p).collect(); + eprintln!( + "warm-pidx-natural-reopen: db_shared cache rows = {} (pgno 1 owner = {:?}), db_writer cache rows = {} (pgno 1 owner = {:?}), db_shared pgnos = {:?}", + shared_cache_pidx.len(), + shared_pg1, + writer_cache_pidx.len(), + writer_pg1, + shared_pgnos, + ); + + // Step 3: open a fresh handle B (the "C" of the spec, but kept as `db_b` for symmetry + // with the sibling test) through the still-warm `db_shared`. Startup preload sees + // `db_shared.cache_snapshot.pidx` with stale pgno 1 entries. + let db_b = harness.open_db_with_transport( + &runtime, + transport_shared.clone(), + &harness.actor_id, + config.clone(), + ); + let schema_seen_by_b = sqlite_query_text( + db_b.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ) + .expect("schema query through B should succeed"); + tracing::info!( + schema_after_a = %schema_after_a, + schema_seen_by_b = %schema_seen_by_b, + writer_head = ?writer_head, + "warm pidx natural reopen repro: schema visibility through stale Db" + ); + + // Step 4: RMW on pgno 1 from the (possibly stale) view. + let stale_create = sqlite_exec( + db_b.as_ptr(), + "CREATE INDEX idx_t1_payload_stale ON t1(payload, id, id);", + ); + let mut stale_commit_observed = stale_create.is_ok(); + if let Err(message) = &stale_create { + tracing::info!(?message, "stale RMW create index failed"); + } + let stale_delete = sqlite_exec(db_b.as_ptr(), "DELETE FROM t1 WHERE id <= 50;"); + if stale_delete.is_ok() { + stale_commit_observed = true; + } + + drop(db_b); + drop(db_shared); + drop(db_writer); + runtime.block_on(engine.evict_actor_db(&harness.actor_id)); + + // Step 5: fresh reopen with no warm cache. Run integrity check + schema scan + the + // post-reopen probes that surface `database disk image is malformed` at runtime. + let db_c = harness.open_db_on_engine(&runtime, engine, &harness.actor_id, config); + let integrity = sqlite_query_text(db_c.as_ptr(), "PRAGMA integrity_check;"); + let quick_check = sqlite_query_text(db_c.as_ptr(), "PRAGMA quick_check;"); + let schema_after_reopen = sqlite_query_text( + db_c.as_ptr(), + "SELECT group_concat(name, ',') FROM sqlite_master ORDER BY name;", + ); + let row_count = sqlite_query_i64(db_c.as_ptr(), "SELECT COUNT(*) FROM t1;"); + + let select_indexed_by_grouped = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM (SELECT payload, COUNT(*) FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00000' AND 'payload-99999' GROUP BY payload);", + ); + let select_indexed_by_count_star = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00000' AND 'payload-99999';", + ); + let select_distinct_via_index = sqlite_query_i64( + db_c.as_ptr(), + "SELECT COUNT(*) FROM (SELECT DISTINCT payload FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload >= 'payload-');", + ); + let update_through_index = sqlite_exec( + db_c.as_ptr(), + "UPDATE t1 SET payload = payload || '-tag' WHERE payload BETWEEN 'payload-00000' AND 'payload-99999';", + ); + let delete_through_index = sqlite_exec( + db_c.as_ptr(), + "DELETE FROM t1 WHERE payload IN (SELECT payload FROM t1 INDEXED BY idx_t1_payload_stale WHERE payload BETWEEN 'payload-00500' AND 'payload-00600');", + ); + let analyze_stale = sqlite_exec(db_c.as_ptr(), "ANALYZE idx_t1_payload_stale;"); + let reindex_stale = sqlite_exec(db_c.as_ptr(), "REINDEX idx_t1_payload_stale;"); + + let runtime_probes: [(&str, Result); 7] = [ + ( + "select_indexed_by_grouped", + select_indexed_by_grouped + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "select_indexed_by_count_star", + select_indexed_by_count_star + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "select_distinct_via_index", + select_distinct_via_index + .as_ref() + .map(|v| v.to_string()) + .map_err(|e| e.clone()), + ), + ( + "update_through_index", + update_through_index + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "delete_through_index", + delete_through_index + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "analyze", + analyze_stale + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ( + "reindex_stale", + reindex_stale + .as_ref() + .map(|_| String::new()) + .map_err(|e| e.clone()), + ), + ]; + + let mut runtime_corrupt_observed: Option = None; + for (label, result) in &runtime_probes { + if let Err(message) = result { + if message.contains("database disk image is malformed") + && runtime_corrupt_observed.is_none() + { + runtime_corrupt_observed = Some(format!("{label}: {message}")); + } + } + } + + tracing::warn!( + ?integrity, + ?quick_check, + ?schema_after_reopen, + ?row_count, + ?select_indexed_by_grouped, + ?select_indexed_by_count_star, + ?select_distinct_via_index, + ?update_through_index, + ?delete_through_index, + ?analyze_stale, + ?reindex_stale, + ?runtime_corrupt_observed, + stale_commit_observed, + "warm pidx natural reopen repro: final reopen state" + ); + eprintln!( + "warm-pidx-natural-reopen: integrity={integrity:?} quick_check={quick_check:?} \ + schema_after_reopen={schema_after_reopen:?} row_count={row_count:?} \ + select_indexed_by_grouped={select_indexed_by_grouped:?} \ + select_indexed_by_count_star={select_indexed_by_count_star:?} \ + select_distinct_via_index={select_distinct_via_index:?} \ + update_through_index={update_through_index:?} \ + delete_through_index={delete_through_index:?} \ + analyze_stale={analyze_stale:?} reindex_stale={reindex_stale:?} \ + runtime_corrupt_observed={runtime_corrupt_observed:?} \ + stale_commit_observed={stale_commit_observed}" + ); + for (label, result) in &runtime_probes { + eprintln!("warm-pidx-natural-reopen: probe[{label}] = {result:?}"); + } + + let malformed_keywords = [ + "malformed", + "corrupt", + "wrong page type", + "reference to page", + "out of order", + "page is never used", + "missing from index", + "row count", + "unordered", + ]; + let is_malformed_text = |text: &str| -> bool { + text != "ok" + && (malformed_keywords.iter().any(|k| text.contains(k)) + || text.contains("***") + || text.lines().count() > 1) + }; + let mut malformed = false; + for probe in [&integrity, &quick_check] { + match probe { + Ok(text) => { + if is_malformed_text(text) { + malformed = true; + } + } + Err(message) => { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + } + } + if let Err(message) = &row_count { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + if let Err(message) = &schema_after_reopen { + if malformed_keywords.iter().any(|k| message.contains(k)) { + malformed = true; + } + } + if runtime_corrupt_observed.is_some() { + malformed = true; + } + + if malformed { + panic!( + "REPRO (natural reopen): warm-pidx-cache hazard produced a malformed DB after reopen \ + with cache populated entirely by VFS startup preload. integrity={integrity:?} \ + quick_check={quick_check:?} schema_after_reopen={schema_after_reopen:?} \ + row_count={row_count:?} runtime_corrupt_observed={runtime_corrupt_observed:?}." + ); + } + + tracing::info!( + "warm pidx natural reopen repro: no malformed DB observed on this run; see eprintln for details" + ); +} + #[test] fn commit_buffered_pages_uses_fast_path() { let runtime = direct_runtime(); diff --git a/engine/packages/error/src/error.rs b/engine/packages/error/src/error.rs index 9e7390be6c..939e05a307 100644 --- a/engine/packages/error/src/error.rs +++ b/engine/packages/error/src/error.rs @@ -181,8 +181,7 @@ impl Serialize for RivetError { { use serde::ser::SerializeStruct; - let field_count = - 3 + usize::from(self.meta.is_some()) + usize::from(self.actor.is_some()); + let field_count = 3 + usize::from(self.meta.is_some()) + usize::from(self.actor.is_some()); let mut state = serializer.serialize_struct("RivetError", field_count)?; state.serialize_field("group", self.group())?; diff --git a/engine/packages/metrics/src/buckets.rs b/engine/packages/metrics/src/buckets.rs index 2d589aea2b..28d2918c34 100644 --- a/engine/packages/metrics/src/buckets.rs +++ b/engine/packages/metrics/src/buckets.rs @@ -9,6 +9,37 @@ pub const MICRO_BUCKETS: &[f64] = &[ 5.0, 10.0, 25.0, 50.0, ]; +pub const LIFETIME_BUCKETS: &[f64] = &[ + 0.1, 0.5, 1.0, 5.0, 15.0, 60.0, 300.0, 900.0, 1800.0, 3600.0, 7200.0, 14400.0, 28800.0, + 86400.0, 259200.0, 604800.0, +]; + +pub const BYTES_BUCKETS: &[f64] = &[ + 128.0, + 256.0, + 512.0, + 1024.0, + 2048.0, + 4096.0, + 8192.0, + 16384.0, + 32768.0, + 65536.0, + 131072.0, + 262144.0, + 524288.0, + 1048576.0, + 2097152.0, + 4194304.0, + 8388608.0, + 16777216.0, +]; + +pub const MESSAGE_COUNT_BUCKETS: &[f64] = &[ + 0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 4096.0, 16384.0, + 65536.0, +]; + // Calculated based on the LogHistogram configuration in `packages/common/runtime/src/lib.rs` pub const TASK_POLL_BUCKETS: &[f64] = &[ 0.00002, diff --git a/engine/packages/metrics/src/lib.rs b/engine/packages/metrics/src/lib.rs index 4d389daccb..da84a86b49 100644 --- a/engine/packages/metrics/src/lib.rs +++ b/engine/packages/metrics/src/lib.rs @@ -1,6 +1,9 @@ mod buckets; mod registry; -pub use buckets::{BUCKETS, MICRO_BUCKETS, TASK_POLL_BUCKETS}; +pub use buckets::{ + BUCKETS, BYTES_BUCKETS, LIFETIME_BUCKETS, MESSAGE_COUNT_BUCKETS, MICRO_BUCKETS, + TASK_POLL_BUCKETS, +}; pub use prometheus; pub use registry::REGISTRY; diff --git a/engine/packages/pegboard-envoy/src/actor_lifecycle.rs b/engine/packages/pegboard-envoy/src/actor_lifecycle.rs index c3a787273a..4479e77183 100644 --- a/engine/packages/pegboard-envoy/src/actor_lifecycle.rs +++ b/engine/packages/pegboard-envoy/src/actor_lifecycle.rs @@ -1,7 +1,12 @@ +use std::time::Instant; + use anyhow::Result; use rivet_envoy_protocol as protocol; -use crate::conn::Conn; +use crate::{ + conn::{ActorStopMeta, Conn}, + metrics, +}; pub async fn stop_actor(conn: &Conn, checkpoint: &protocol::ActorCheckpoint) -> Result<()> { // Depot owns SQLite correctness in FDB. The connection only holds a perf cache, so @@ -17,3 +22,55 @@ pub async fn shutdown_conn_actors(conn: &Conn) { conn.actor_dbs.clear_sync(); conn.remote_sqlite_executors.clear_sync(); } + +/// Convert the BARE `StopActorReason` enum into a bounded metric label. +pub fn stop_reason_label(reason: &protocol::StopActorReason) -> &'static str { + match reason { + protocol::StopActorReason::SleepIntent => "sleep_intent", + protocol::StopActorReason::StopIntent => "stop_intent", + protocol::StopActorReason::Destroy => "destroy", + protocol::StopActorReason::GoingAway => "going_away", + protocol::StopActorReason::Lost => "lost", + } +} + +/// Record that the engine dispatched a start command for the actor. Tracks the start +/// timestamp so we can emit `actor_lifetime_seconds` at stop time. +pub async fn record_actor_start(conn: &Conn, actor_id: &str, create_ts_ms: i64) { + let _ = conn + .actor_started_at + .insert_async(actor_id.to_string(), create_ts_ms) + .await; +} + +/// Record that the engine dispatched a stop command for the actor. Captures the stop +/// reason + dispatch instant for `actor_stop_total` and `actor_stop_to_close_seconds`, +/// and increments `pegboard_actor_lost_total` for the `engine_command` origin when the +/// reason is `Lost`. +pub async fn record_actor_stop_dispatch( + conn: &Conn, + actor_id: &str, + reason: &protocol::StopActorReason, +) { + let reason_label = stop_reason_label(reason); + let _ = conn + .actor_stop_meta + .insert_async( + actor_id.to_string(), + ActorStopMeta { + reason: reason_label, + dispatched_at: Instant::now(), + }, + ) + .await; + + if matches!(reason, protocol::StopActorReason::Lost) { + metrics::ACTOR_LOST_TOTAL + .with_label_values(&[ + conn.namespace_id.to_string().as_str(), + &conn.pool_name, + "engine_command", + ]) + .inc(); + } +} diff --git a/engine/packages/pegboard-envoy/src/conn.rs b/engine/packages/pegboard-envoy/src/conn.rs index 0278026988..278bd30059 100644 --- a/engine/packages/pegboard-envoy/src/conn.rs +++ b/engine/packages/pegboard-envoy/src/conn.rs @@ -26,6 +26,12 @@ use crate::{actor_lifecycle, errors, hibernating_requests, metrics, utils::UrlDa pub type RemoteSqliteExecutors = HashMap<(String, u64), Arc>>; +#[derive(Clone)] +pub struct ActorStopMeta { + pub reason: &'static str, + pub dispatched_at: Instant, +} + pub struct Conn { pub namespace_id: Id, pub namespace_name: String, @@ -46,6 +52,10 @@ pub struct Conn { pub last_rtt: AtomicU32, /// Timestamp (epoch ms) of the last pong received from the envoy. pub last_ping_ts: AtomicI64, + /// Per-actor start timestamp (epoch ms) used for lifetime histograms. + pub actor_started_at: HashMap, + /// Per-actor stop dispatch metadata used for stop-cause and stop-to-close metrics. + pub actor_stop_meta: HashMap, } #[tracing::instrument(skip_all)] @@ -327,6 +337,8 @@ pub async fn init_conn( is_serverless, last_rtt: AtomicU32::new(0), last_ping_ts: AtomicI64::new(util::timestamp::now()), + actor_started_at: HashMap::new(), + actor_stop_meta: HashMap::new(), }); // Send missed commands after the init packet. @@ -334,8 +346,24 @@ pub async fn init_conn( let replay_result: Result<()> = async { for cmd_wrapper in &mut missed_commands { hibernating_requests::hydrate_command_wrapper(ctx, cmd_wrapper).await?; - if let protocol::Command::CommandStopActor(_) = cmd_wrapper.inner { - actor_lifecycle::stop_actor(&conn, &cmd_wrapper.checkpoint).await?; + match &cmd_wrapper.inner { + protocol::Command::CommandStartActor(start) => { + actor_lifecycle::record_actor_start( + &conn, + &cmd_wrapper.checkpoint.actor_id, + start.config.create_ts, + ) + .await; + } + protocol::Command::CommandStopActor(stop) => { + actor_lifecycle::record_actor_stop_dispatch( + &conn, + &cmd_wrapper.checkpoint.actor_id, + &stop.reason, + ) + .await; + actor_lifecycle::stop_actor(&conn, &cmd_wrapper.checkpoint).await?; + } } } Ok(()) diff --git a/engine/packages/pegboard-envoy/src/lib.rs b/engine/packages/pegboard-envoy/src/lib.rs index b25d3c131f..0609b74bce 100644 --- a/engine/packages/pegboard-envoy/src/lib.rs +++ b/engine/packages/pegboard-envoy/src/lib.rs @@ -34,8 +34,11 @@ pub struct PegboardEnvoyWs { ctx: StandaloneCtx, } +static METRICS_PREPOPULATE: std::sync::Once = std::sync::Once::new(); + impl PegboardEnvoyWs { pub fn new(ctx: StandaloneCtx) -> Self { + METRICS_PREPOPULATE.call_once(metrics::prepopulate); let service = Self { ctx: ctx.clone() }; service @@ -212,7 +215,14 @@ impl CustomServeTrait for PegboardEnvoyWs { // flight kv requests from being completed immediately. This guarantees the invariant that an // actor's KV is only being accessed from one place at a time. if res.is_err() { - tracing::warn!(?res, "ping task failed, aborting ws_to_tunnel"); + let now = util::timestamp::now(); + let last_ping_ts = conn.last_ping_ts.load(std::sync::atomic::Ordering::SeqCst); + tracing::warn!( + ?res, + envoy_key = %conn.envoy_key, + time_since_last_pong_ms = now - last_ping_ts, + "ping task failed, aborting ws_to_tunnel" + ); hard_abort_ws_to_tunnel.abort(); } @@ -269,7 +279,62 @@ impl CustomServeTrait for PegboardEnvoyWs { ]) .dec(); + // Classify the close for `ws_close_initiator_total` and `connection_close_total`. + let ns_label = conn.namespace_id.to_string(); + let pool_label = conn.pool_name.as_str(); + let (initiator, cause, close_code, reason_group) = classify_ws_close(&lifecycle_res); + metrics::WS_CLOSE_INITIATOR_TOTAL + .with_label_values(&[ns_label.as_str(), pool_label, initiator, cause.as_str()]) + .inc(); + metrics::CONNECTION_CLOSE_TOTAL + .with_label_values(&[ns_label.as_str(), pool_label, close_code, reason_group.as_str()]) + .inc(); + // This will determine the close frame sent back to the envoy websocket lifecycle_res.map(|_| None) } } + +/// Determine `(initiator, cause, close_code, reason_group)` metric labels from the lifecycle +/// result returned by the envoy websocket tasks. +fn classify_ws_close( + res: &Result, +) -> (&'static str, String, &'static str, String) { + match res { + Ok(LifecycleResult::Closed) => ( + "client_close", + "ok".to_string(), + "1000", + "ok".to_string(), + ), + Ok(LifecycleResult::Aborted) => ( + "engine_shutdown", + "ok".to_string(), + "1001", + "ok".to_string(), + ), + Ok(LifecycleResult::Evicted) => ( + "engine_eviction", + "ws.eviction".to_string(), + "1008", + "ws".to_string(), + ), + Err(err) => { + let structured = rivet_error::RivetError::extract(err); + let group = structured.group().to_string(); + let code = structured.code().to_string(); + let cause = format!("{group}.{code}"); + let (initiator, close_code) = if group == "ws" { + match code.as_str() { + "timed_out" => ("engine_ping_timeout", "1011"), + "going_away" => ("engine_shutdown", "1001"), + "eviction" => ("engine_eviction", "1008"), + _ => ("network", "1006"), + } + } else { + ("network", "1011") + }; + (initiator, cause, close_code, group) + } + } +} diff --git a/engine/packages/pegboard-envoy/src/metrics.rs b/engine/packages/pegboard-envoy/src/metrics.rs index f292e69746..6db4e1f29e 100644 --- a/engine/packages/pegboard-envoy/src/metrics.rs +++ b/engine/packages/pegboard-envoy/src/metrics.rs @@ -1,4 +1,6 @@ -use rivet_metrics::{BUCKETS, REGISTRY, prometheus::*}; +use rivet_metrics::{ + BUCKETS, BYTES_BUCKETS, LIFETIME_BUCKETS, MESSAGE_COUNT_BUCKETS, REGISTRY, prometheus::*, +}; lazy_static::lazy_static! { pub static ref CONNECTION_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( @@ -88,4 +90,197 @@ lazy_static::lazy_static! { BUCKETS.to_vec(), *REGISTRY ).unwrap(); + + // MARK: Stop / eviction causality + pub static ref ACTOR_STOP_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_envoy_actor_stop_total", + "Count of actor stops observed by pegboard-envoy.", + &["namespace_id", "pool_name", "reason", "code"], + *REGISTRY + ).unwrap(); + + pub static ref ACTOR_LIFETIME_SECONDS: HistogramVec = register_histogram_vec_with_registry!( + "pegboard_envoy_actor_lifetime_seconds", + "Lifetime of actors from start to stop in seconds.", + &["namespace_id", "pool_name", "reason"], + LIFETIME_BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + + pub static ref WS_CLOSE_INITIATOR_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_envoy_ws_close_initiator_total", + "Count of websocket closes by the initiating party.", + &["namespace_id", "pool_name", "initiator", "cause"], + *REGISTRY + ).unwrap(); + + pub static ref ACTOR_LOST_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_actor_lost_total", + "Count of actors marked lost by origin.", + &["namespace_id", "pool_name", "origin"], + *REGISTRY + ).unwrap(); + + pub static ref CONNECTION_CLOSE_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_envoy_connection_close_total", + "Count of envoy connection closes by ws close code and reason group.", + &["namespace_id", "pool_name", "close_code", "reason_group"], + *REGISTRY + ).unwrap(); + + pub static ref EVICTION_WITH_REASON_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_envoy_eviction_with_reason_total", + "Count of envoy connections evicted by reason.", + &["namespace_id", "pool_name", "protocol_version", "reason"], + *REGISTRY + ).unwrap(); + + // MARK: WS traffic shape + pub static ref WS_MESSAGES_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_envoy_ws_messages_total", + "Count of websocket messages by direction and kind.", + &["namespace_id", "pool_name", "direction", "message_kind"], + *REGISTRY + ).unwrap(); + + pub static ref WS_BYTES_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_envoy_ws_bytes_total", + "Encoded websocket message byte volume by direction and kind.", + &["namespace_id", "pool_name", "direction", "message_kind"], + *REGISTRY + ).unwrap(); + + pub static ref TUNNEL_MESSAGE_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_envoy_tunnel_message_total", + "Count of tunnel messages dispatched by direction and tunnel kind.", + &["namespace_id", "pool_name", "direction", "tunnel_kind"], + *REGISTRY + ).unwrap(); + + pub static ref WS_FRAME_SIZE_BYTES: HistogramVec = register_histogram_vec_with_registry!( + "pegboard_envoy_ws_frame_size_bytes", + "Size of websocket frames in bytes.", + &["namespace_id", "pool_name", "direction"], + BYTES_BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + + pub static ref ACK_LAG_MESSAGES: HistogramVec = register_histogram_vec_with_registry!( + "pegboard_envoy_ack_lag_messages", + "Lag in messages between sent and acked sequence numbers.", + &["namespace_id", "pool_name", "direction"], + MESSAGE_COUNT_BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + + // MARK: Nice-to-haves + pub static ref ACTOR_STOP_TO_CLOSE_SECONDS: HistogramVec = register_histogram_vec_with_registry!( + "pegboard_envoy_actor_stop_to_close_seconds", + "Time from sending Stop to receiving Stopping/close in seconds.", + &["namespace_id", "pool_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + + pub static ref PONG_MISSED_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_envoy_pong_missed_total", + "Count of pongs that arrived after the slow threshold but before timeout.", + &["namespace_id", "pool_name"], + *REGISTRY + ).unwrap(); +} + +/// Bounded enum-style label values used across the metrics in this module. Listed here so +/// `prepopulate` can zero-initialize the canonical label combinations and expose them on a +/// cold `/metrics` scrape. +const STOP_REASONS: &[&str] = &[ + "sleep_intent", + "stop_intent", + "destroy", + "going_away", + "lost", +]; + +const STOP_CODES: &[&str] = &["ok", "error"]; + +const WS_CLOSE_INITIATORS: &[&str] = &[ + "engine_ping_timeout", + "engine_eviction", + "engine_shutdown", + "client_close", + "network", +]; + +const ACTOR_LOST_ORIGINS: &[&str] = &[ + "engine_workflow_liveness", + "client_self_evict", + "engine_command", +]; + +const CONNECTION_CLOSE_CODES: &[&str] = &["1000", "1001", "1006", "1008", "1011", "other"]; + +const EVICTION_REASONS: &[&str] = &[ + "duplicate_key", + "protocol_mismatch", + "version_drain", + "shutdown", +]; + +const DIRECTIONS: &[&str] = &["inbound", "outbound"]; + +const TUNNEL_KINDS: &[&str] = &[ + "request_start", + "request_chunk", + "request_abort", + "response_start", + "response_chunk", + "response_abort", + "ws_open", + "ws_message", + "ws_message_ack", + "ws_close", +]; + +/// Zero-initialize the canonical label combinations for the metrics defined in this module so +/// they are present on the first `/metrics` scrape. Called once via a `Once` from the +/// `PegboardEnvoyWs::new` constructor. +pub fn prepopulate() { + // Existing metrics that already use label vectors are fine to leave to their first + // observation; zero-initialize the new ones with empty-string values for unbounded labels + // and one-of-each for bounded enums. + for reason in STOP_REASONS { + for code in STOP_CODES { + ACTOR_STOP_TOTAL.with_label_values(&["", "", reason, code]); + } + ACTOR_LIFETIME_SECONDS.with_label_values(&["", "", reason]); + } + + for initiator in WS_CLOSE_INITIATORS { + WS_CLOSE_INITIATOR_TOTAL.with_label_values(&["", "", initiator, ""]); + } + + for origin in ACTOR_LOST_ORIGINS { + ACTOR_LOST_TOTAL.with_label_values(&["", "", origin]); + } + + for code in CONNECTION_CLOSE_CODES { + CONNECTION_CLOSE_TOTAL.with_label_values(&["", "", code, ""]); + } + + for reason in EVICTION_REASONS { + EVICTION_WITH_REASON_TOTAL.with_label_values(&["", "", "", reason]); + } + + for direction in DIRECTIONS { + WS_MESSAGES_TOTAL.with_label_values(&["", "", direction, ""]); + WS_BYTES_TOTAL.with_label_values(&["", "", direction, ""]); + WS_FRAME_SIZE_BYTES.with_label_values(&["", "", direction]); + ACK_LAG_MESSAGES.with_label_values(&["", "", direction]); + for tunnel_kind in TUNNEL_KINDS { + TUNNEL_MESSAGE_TOTAL.with_label_values(&["", "", direction, tunnel_kind]); + } + } + + ACTOR_STOP_TO_CLOSE_SECONDS.with_label_values(&["", ""]); + PONG_MISSED_TOTAL.with_label_values(&["", ""]); } diff --git a/engine/packages/pegboard-envoy/src/ping_task.rs b/engine/packages/pegboard-envoy/src/ping_task.rs index 281962aa36..5e07b71b3d 100644 --- a/engine/packages/pegboard-envoy/src/ping_task.rs +++ b/engine/packages/pegboard-envoy/src/ping_task.rs @@ -32,19 +32,35 @@ pub async fn task( // Check if the last ping is past the timeout threshold let last_ping_ts = conn.last_ping_ts.load(Ordering::SeqCst); let now = util::timestamp::now(); - if now - last_ping_ts > ping_timeout_ms { + let gap_ms = now - last_ping_ts; + if gap_ms > ping_timeout_ms { + tracing::warn!( + envoy_key = %conn.envoy_key, + last_ping_ts, + now, + gap_ms, + threshold_ms = ping_timeout_ms, + "envoy ping timed out, closing connection" + ); return Err(WsError::TimedOut.build()); } // Update ping + let last_rtt = conn.last_rtt.load(Ordering::Relaxed); ctx.op(pegboard::ops::envoy::update_ping::Input { namespace_id: conn.namespace_id, envoy_key: conn.envoy_key.clone(), update_lb: !conn.is_serverless, - rtt: conn.last_rtt.load(Ordering::Relaxed), + rtt: last_rtt, }) .await?; + tracing::debug!( + gap_since_last_pong_ms = gap_ms, + last_rtt_ms = last_rtt, + "sending ping" + ); + // Send ping to envoy let ping_msg = versioned::ToEnvoy::wrap_latest(protocol::ToEnvoy::ToEnvoyPing( protocol::ToEnvoyPing { diff --git a/engine/packages/pegboard-envoy/src/tunnel_to_ws_task.rs b/engine/packages/pegboard-envoy/src/tunnel_to_ws_task.rs index 16673dc05b..310009f2c0 100644 --- a/engine/packages/pegboard-envoy/src/tunnel_to_ws_task.rs +++ b/engine/packages/pegboard-envoy/src/tunnel_to_ws_task.rs @@ -64,6 +64,14 @@ async fn recv_msg( conn.protocol_version.to_string().as_str(), ]) .inc(); + metrics::EVICTION_WITH_REASON_TOTAL + .with_label_values(&[ + conn.namespace_id.to_string().as_str(), + &conn.pool_name, + conn.protocol_version.to_string().as_str(), + "duplicate_key", + ]) + .inc(); return Ok(Err(LifecycleResult::Evicted)); } @@ -95,6 +103,9 @@ async fn handle_message( } }; + let ns_label = conn.namespace_id.to_string(); + let pool_label = conn.pool_name.as_str(); + // Convert to ToEnvoy types let to_client_msg = match msg { protocol::ToEnvoyConn::ToEnvoyConnPing(ping) => { @@ -127,8 +138,24 @@ async fn handle_message( // TODO: Parallelize for command_wrapper in &mut command_wrappers { hibernating_requests::hydrate_command_wrapper(ctx, command_wrapper).await?; - if let protocol::Command::CommandStopActor(_) = &command_wrapper.inner { - actor_lifecycle::stop_actor(conn, &command_wrapper.checkpoint).await?; + match &command_wrapper.inner { + protocol::Command::CommandStartActor(start) => { + actor_lifecycle::record_actor_start( + conn, + &command_wrapper.checkpoint.actor_id, + start.config.create_ts, + ) + .await; + } + protocol::Command::CommandStopActor(stop) => { + actor_lifecycle::record_actor_stop_dispatch( + conn, + &command_wrapper.checkpoint.actor_id, + &stop.reason, + ) + .await; + actor_lifecycle::stop_actor(conn, &command_wrapper.checkpoint).await?; + } } } @@ -142,13 +169,33 @@ async fn handle_message( .authorized_tunnel_routes .insert_async((x.message_id.gateway_id, x.message_id.request_id), ()) .await; + metrics::TUNNEL_MESSAGE_TOTAL + .with_label_values(&[ + ns_label.as_str(), + pool_label, + "outbound", + to_envoy_tunnel_kind_label(&x.message_kind), + ]) + .inc(); protocol::ToEnvoy::ToEnvoyTunnelMessage(x) } }; + let message_kind_label = to_envoy_message_kind_label(&to_client_msg); + // Forward raw message to WebSocket let serialized_msg = versioned::ToEnvoy::wrap_latest(to_client_msg).serialize(conn.protocol_version)?; + let serialized_len = serialized_msg.len(); + metrics::WS_MESSAGES_TOTAL + .with_label_values(&[ns_label.as_str(), pool_label, "outbound", message_kind_label]) + .inc(); + metrics::WS_BYTES_TOTAL + .with_label_values(&[ns_label.as_str(), pool_label, "outbound", message_kind_label]) + .inc_by(serialized_len as u64); + metrics::WS_FRAME_SIZE_BYTES + .with_label_values(&[ns_label.as_str(), pool_label, "outbound"]) + .observe(serialized_len as f64); let ws_msg = Message::Binary(serialized_msg.into()); conn.ws_handle .send(ws_msg) @@ -157,3 +204,31 @@ async fn handle_message( Ok(false) } + +/// Map a `ToEnvoy` variant to a bounded `message_kind` metric label. +fn to_envoy_message_kind_label(msg: &protocol::ToEnvoy) -> &'static str { + match msg { + protocol::ToEnvoy::ToEnvoyInit(_) => "init", + protocol::ToEnvoy::ToEnvoyCommands(_) => "commands", + protocol::ToEnvoy::ToEnvoyAckEvents(_) => "ack_events", + protocol::ToEnvoy::ToEnvoyKvResponse(_) => "kv_response", + protocol::ToEnvoy::ToEnvoyTunnelMessage(_) => "tunnel_message", + protocol::ToEnvoy::ToEnvoyPing(_) => "ping", + protocol::ToEnvoy::ToEnvoySqliteGetPagesResponse(_) => "sqlite_get_pages_response", + protocol::ToEnvoy::ToEnvoySqliteCommitResponse(_) => "sqlite_commit_response", + protocol::ToEnvoy::ToEnvoySqliteExecResponse(_) => "sqlite_exec_response", + protocol::ToEnvoy::ToEnvoySqliteExecuteResponse(_) => "sqlite_execute_response", + } +} + +/// Map a `ToEnvoyTunnelMessageKind` variant to a bounded `tunnel_kind` metric label. +fn to_envoy_tunnel_kind_label(kind: &protocol::ToEnvoyTunnelMessageKind) -> &'static str { + match kind { + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestStart(_) => "request_start", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestChunk(_) => "request_chunk", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestAbort => "request_abort", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketOpen(_) => "ws_open", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketMessage(_) => "ws_message", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketClose(_) => "ws_close", + } +} diff --git a/engine/packages/pegboard-envoy/src/ws_to_tunnel_task.rs b/engine/packages/pegboard-envoy/src/ws_to_tunnel_task.rs index b9a33dfd80..39c2422647 100644 --- a/engine/packages/pegboard-envoy/src/ws_to_tunnel_task.rs +++ b/engine/packages/pegboard-envoy/src/ws_to_tunnel_task.rs @@ -37,7 +37,7 @@ use crate::{ LifecycleResult, actor_event_demuxer::ActorEventDemuxer, conn::{Conn, RemoteSqliteExecutors}, - errors, sqlite_runtime, + errors, metrics, sqlite_runtime, }; const MAX_REMOTE_SQL_BIND_BYTES: usize = 128 * 1024; @@ -131,6 +131,13 @@ async fn handle_message( event_demuxer: &mut ActorEventDemuxer, msg: Bytes, ) -> Result<()> { + let ns_label = conn.namespace_id.to_string(); + let pool_label = conn.pool_name.as_str(); + let msg_len = msg.len(); + metrics::WS_FRAME_SIZE_BYTES + .with_label_values(&[ns_label.as_str(), pool_label, "inbound"]) + .observe(msg_len as f64); + // Parse message let msg = match versioned::ToRivet::deserialize(&msg, conn.protocol_version) { Ok(x) => x, @@ -140,6 +147,14 @@ async fn handle_message( } }; + let message_kind_label = to_rivet_message_kind_label(&msg); + metrics::WS_MESSAGES_TOTAL + .with_label_values(&[ns_label.as_str(), pool_label, "inbound", message_kind_label]) + .inc(); + metrics::WS_BYTES_TOTAL + .with_label_values(&[ns_label.as_str(), pool_label, "inbound", message_kind_label]) + .inc_by(msg_len as u64); + tracing::debug!(?msg, "received message from envoy"); match msg { @@ -154,6 +169,27 @@ async fn handle_message( u32::MAX }; + // Count pongs that arrived after a "slow" threshold (1.5x normal ping interval) but + // before the ping timeout fired in `ping_task::task`. + let update_ping_interval = ctx.config().pegboard().envoy_update_ping_interval(); + let slow_threshold_ms = update_ping_interval.saturating_mul(3) / 2; + if u64::from(rtt) > slow_threshold_ms { + metrics::PONG_MISSED_TOTAL + .with_label_values(&[ns_label.as_str(), pool_label]) + .inc(); + tracing::warn!( + rtt_ms = rtt, + slow_threshold_ms, + "slow pong" + ); + } + + // Independent high-rtt warning at a hard 500ms threshold so operators + // see latency spikes even when slow_threshold_ms is configured higher. + if rtt > 500 && u64::from(rtt) <= slow_threshold_ms { + tracing::warn!(rtt_ms = rtt, "high rtt pong"); + } + conn.last_rtt.store(rtt, Ordering::SeqCst); conn.last_ping_ts .store(util::timestamp::now(), Ordering::SeqCst); @@ -419,6 +455,14 @@ async fn handle_message( send_sqlite_execute_response(&conn, req.request_id, response).await?; } protocol::ToRivet::ToRivetTunnelMessage(tunnel_msg) => { + metrics::TUNNEL_MESSAGE_TOTAL + .with_label_values(&[ + ns_label.as_str(), + pool_label, + "inbound", + to_rivet_tunnel_kind_label(&tunnel_msg.message_kind), + ]) + .inc(); handle_tunnel_message(ctx, &conn.authorized_tunnel_routes, tunnel_msg) .await .context("failed to handle tunnel message")?; @@ -429,13 +473,37 @@ async fn handle_message( // Forward to demuxer which forwards to actor wf protocol::ToRivet::ToRivetEvents(events) => { for event in events { - event_demuxer.ingest(Id::parse(&event.checkpoint.actor_id)?, event); + let actor_id = Id::parse(&event.checkpoint.actor_id)?; + // Inspect actor state transitions to drive lifecycle metrics before we hand the + // event off to the demuxer (which takes ownership). + if let protocol::Event::EventActorStateUpdate(update) = &event.inner { + if let protocol::ActorState::ActorStateStopped(stopped) = &update.state { + observe_actor_stopped( + &conn, + ns_label.as_str(), + pool_label, + &event.checkpoint.actor_id, + stopped, + ) + .await; + } + } + event_demuxer.ingest(actor_id, event); } } protocol::ToRivet::ToRivetAckCommands(ack) => { + // TODO(metrics): `pegboard_envoy_ack_lag_messages` requires the latest sent command + // sequence per actor, which is tracked in the pegboard workflow rather than in + // pegboard-envoy. Wire from the workflow when available. ack_commands(&ctx, conn.namespace_id, &conn.envoy_key, ack).await?; } protocol::ToRivet::ToRivetStopping => { + // The envoy is voluntarily going away. Treat all actors hosted on it as lost from + // the client's side. + metrics::ACTOR_LOST_TOTAL + .with_label_values(&[ns_label.as_str(), pool_label, "client_self_evict"]) + .inc(); + // For serverful, remove from lb if !conn.is_serverless { ctx.op(pegboard::ops::envoy::expire::Input { @@ -1158,6 +1226,74 @@ fn sqlite_error_response(err: &anyhow::Error) -> protocol::SqliteErrorResponse { } } +/// Map a `ToRivet` variant to a bounded `message_kind` metric label. +fn to_rivet_message_kind_label(msg: &protocol::ToRivet) -> &'static str { + match msg { + protocol::ToRivet::ToRivetMetadata(_) => "metadata", + protocol::ToRivet::ToRivetEvents(_) => "events", + protocol::ToRivet::ToRivetAckCommands(_) => "ack_commands", + protocol::ToRivet::ToRivetStopping => "stopping", + protocol::ToRivet::ToRivetPong(_) => "pong", + protocol::ToRivet::ToRivetKvRequest(_) => "kv_request", + protocol::ToRivet::ToRivetTunnelMessage(_) => "tunnel_message", + protocol::ToRivet::ToRivetSqliteGetPagesRequest(_) => "sqlite_get_pages_request", + protocol::ToRivet::ToRivetSqliteCommitRequest(_) => "sqlite_commit_request", + protocol::ToRivet::ToRivetSqliteExecRequest(_) => "sqlite_exec_request", + protocol::ToRivet::ToRivetSqliteExecuteRequest(_) => "sqlite_execute_request", + } +} + +/// Map a `ToRivetTunnelMessageKind` variant to a bounded `tunnel_kind` metric label. +fn to_rivet_tunnel_kind_label(kind: &protocol::ToRivetTunnelMessageKind) -> &'static str { + match kind { + protocol::ToRivetTunnelMessageKind::ToRivetResponseStart(_) => "response_start", + protocol::ToRivetTunnelMessageKind::ToRivetResponseChunk(_) => "response_chunk", + protocol::ToRivetTunnelMessageKind::ToRivetResponseAbort => "response_abort", + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketOpen(_) => "ws_open", + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketMessage(_) => "ws_message", + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketMessageAck(_) => "ws_message_ack", + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketClose(_) => "ws_close", + } +} + +/// Emit lifecycle metrics for an observed `ActorStateStopped` transition, then drop the +/// tracking entries for that actor. +async fn observe_actor_stopped( + conn: &Conn, + ns_label: &str, + pool_label: &str, + actor_id: &str, + stopped: &protocol::ActorStateStopped, +) { + let code_label = match stopped.code { + protocol::StopCode::Ok => "ok", + protocol::StopCode::Error => "error", + }; + let stop_meta = conn.actor_stop_meta.remove_async(actor_id).await; + let reason_label = stop_meta + .as_ref() + .map(|(_, meta)| meta.reason) + .unwrap_or("stop_intent"); + + metrics::ACTOR_STOP_TOTAL + .with_label_values(&[ns_label, pool_label, reason_label, code_label]) + .inc(); + + if let Some((_, started_at_ms)) = conn.actor_started_at.remove_async(actor_id).await { + let now_ms = util::timestamp::now(); + let lifetime_seconds = (now_ms.saturating_sub(started_at_ms) as f64) / 1000.0; + metrics::ACTOR_LIFETIME_SECONDS + .with_label_values(&[ns_label, pool_label, reason_label]) + .observe(lifetime_seconds.max(0.0)); + } + + if let Some((_, meta)) = stop_meta { + metrics::ACTOR_STOP_TO_CLOSE_SECONDS + .with_label_values(&[ns_label, pool_label]) + .observe(meta.dispatched_at.elapsed().as_secs_f64()); + } +} + /// Returns the length of the inner data payload for a tunnel message kind. fn tunnel_message_inner_data_len(kind: &protocol::ToRivetTunnelMessageKind) -> usize { use protocol::ToRivetTunnelMessageKind; diff --git a/engine/sdks/rust/envoy-client/Cargo.toml b/engine/sdks/rust/envoy-client/Cargo.toml index 6cc6958283..95ccd2447f 100644 --- a/engine/sdks/rust/envoy-client/Cargo.toml +++ b/engine/sdks/rust/envoy-client/Cargo.toml @@ -18,6 +18,7 @@ hex.workspace = true js-sys = { version = "0.3", optional = true } rand.workspace = true rivet-envoy-protocol.workspace = true +rivet-metrics.workspace = true rivet-util-serde.workspace = true rustls = { workspace = true, optional = true } scc.workspace = true diff --git a/engine/sdks/rust/envoy-client/src/actor.rs b/engine/sdks/rust/envoy-client/src/actor.rs index e15f789c68..3a65462f62 100644 --- a/engine/sdks/rust/envoy-client/src/actor.rs +++ b/engine/sdks/rust/envoy-client/src/actor.rs @@ -17,7 +17,7 @@ use crate::context::SharedContext; use crate::handle::EnvoyHandle; use crate::stringify::stringify_to_rivet_tunnel_message_kind; use crate::utils::{ - BufferMap, id_to_str, spawn_detached, wrapping_add_u16, wrapping_lte_u16, wrapping_sub_u16, + BufferMap, display_id, spawn_detached, wrapping_add_u16, wrapping_lte_u16, wrapping_sub_u16, }; pub enum ToActor { @@ -88,6 +88,9 @@ struct ActorContext { ws_entries: BufferMap, hibernating_requests: Vec, active_http_request_count: Arc, + /// Captured at the top of `actor_inner` so `actor_lifetime_seconds` can be + /// observed at stop time. + started_at: crate::time::Instant, } struct ActiveHttpRequestGuard { @@ -179,6 +182,7 @@ async fn actor_inner( ws_entries: BufferMap::new(), hibernating_requests, active_http_request_count, + started_at: crate::time::Instant::now(), }; let mut http_request_tasks = JoinSet::new(); let mut pending_stop: Option = None; @@ -302,11 +306,14 @@ async fn actor_inner( ); continue; } + + ctx.error = Some("actor lost due to timeout".to_string()); + match begin_stop( &mut ctx, &handle, &mut http_request_tasks, - protocol::StopActorReason::Lost, + protocol::StopActorReason::SleepIntent, ) .await { @@ -371,17 +378,28 @@ async fn actor_inner( } abort_and_join_http_request_tasks(&mut ctx, &mut http_request_tasks).await; - tracing::debug!("envoy actor stopped"); + tracing::info!("actor stopped"); } fn send_event(ctx: &mut ActorContext, inner: protocol::Event) { let checkpoint = increment_checkpoint(ctx); - let _ = ctx - .shared - .envoy_tx - .send(crate::envoy::ToEnvoyMessage::SendEvents { + let _ = crate::envoy::send_to_envoy_tx( + &ctx.shared, + crate::envoy::ToEnvoyMessage::SendEvents { events: vec![protocol::EventWrapper { checkpoint, inner }], - }); + }, + ); +} + +/// Bounded label values for `actor_stop_total` / `actor_lifetime_seconds`. +fn stop_actor_reason_label(reason: &protocol::StopActorReason) -> &'static str { + match reason { + protocol::StopActorReason::SleepIntent => "sleep_intent", + protocol::StopActorReason::StopIntent => "stop_intent", + protocol::StopActorReason::Destroy => "destroy", + protocol::StopActorReason::GoingAway => "going_away", + protocol::StopActorReason::Lost => "lost", + } } async fn begin_stop( @@ -390,12 +408,23 @@ async fn begin_stop( _http_request_tasks: &mut JoinSet<()>, reason: protocol::StopActorReason, ) -> StopProgress { - let mut stop_code = if ctx.error.is_some() { - protocol::StopCode::Error + // A Lost stop must surface as Stopped(Error). The runner side detected its + // own WS to pegboard-envoy was unhealthy and gave up on the actor; that is + // not a graceful exit. If we emitted Stopped(Ok) here, pegboard's + // `handle_stopped` would see Stopped(Ok) from `Transition::Running` (no + // prior `ActorIntent` was sent) and take `Decision::Destroy`, wiping the + // actor and its KV after every transient WS flap that exceeds + // `envoy_lost_threshold`. + let (mut stop_code, mut stop_message) = if let Some(err) = ctx.error.clone() { + (protocol::StopCode::Error, Some(err)) + } else if matches!(reason, protocol::StopActorReason::Lost) { + ( + protocol::StopCode::Error, + Some("envoy connection lost".to_string()), + ) } else { - protocol::StopCode::Ok + (protocol::StopCode::Ok, None) }; - let mut stop_message = ctx.error.clone(); let (stop_tx, mut stop_rx) = oneshot::channel(); let stop_result = ctx @@ -647,13 +676,23 @@ fn spawn_ws_outgoing_task( idx += 1; match msg { crate::config::WsOutgoing::Message { data, binary } => { - ws_send( + let data_len = data.len(); + let message_index = idx; + tracing::trace!( + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "sending websocket message to engine" + ); + let failed = ws_send( &shared, protocol::ToRivet::ToRivetTunnelMessage(protocol::ToRivetTunnelMessage { message_id: protocol::MessageId { gateway_id, request_id, - message_index: idx, + message_index, }, message_kind: protocol::ToRivetTunnelMessageKind::ToRivetWebSocketMessage( @@ -662,6 +701,25 @@ fn spawn_ws_outgoing_task( }), ) .await; + if failed { + tracing::warn!( + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "failed sending websocket message to engine" + ); + } else { + tracing::trace!( + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "sent websocket message to engine" + ); + } } crate::config::WsOutgoing::Flush { tx } => { let _ = tx.send(()); @@ -892,6 +950,19 @@ async fn handle_ws_message( message_id: protocol::MessageId, msg: protocol::ToEnvoyWebSocketMessage, ) { + let data_len = msg.data.len(); + let binary = msg.binary; + let gateway_id = message_id.gateway_id; + let request_id = message_id.request_id; + let message_index = message_id.message_index; + tracing::trace!( + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "received websocket message from engine" + ); let ws = ctx .ws_entries .get_mut(&[&message_id.gateway_id, &message_id.request_id]); @@ -904,7 +975,7 @@ async fn handle_ws_message( if wrapping_lte_u16(received_index, previous_index) { tracing::info!( - request_id = id_to_str(&message_id.request_id), + request_id = %display_id(&message_id.request_id), previous_index, received_index, "received duplicate hibernating websocket message" @@ -915,7 +986,7 @@ async fn handle_ws_message( let expected_index = wrapping_add_u16(previous_index, 1); if received_index != expected_index { tracing::warn!( - request_id = id_to_str(&message_id.request_id), + request_id = %display_id(&message_id.request_id), previous_index, expected_index, received_index, @@ -954,10 +1025,33 @@ async fn handle_ws_message( message_index: message_id.message_index, sender, }; - (handler.on_message)(ws_msg).await; - } - } else { - tracing::warn!("received message for unknown ws"); + tracing::trace!( + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "dispatching websocket message to actor handler" + ); + (handler.on_message)(ws_msg).await; + tracing::trace!( + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "dispatched websocket message to actor handler" + ); + } + } else { + tracing::warn!( + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "received message for unknown ws" + ); } } @@ -1089,13 +1183,13 @@ async fn handle_hws_restore( } } tracing::info!( - request_id = id_to_str(&hib_req.request_id), + request_id = %display_id(&hib_req.request_id), "connection successfully restored" ); } Err(error) => { tracing::error!( - request_id = id_to_str(&hib_req.request_id), + request_id = %display_id(&hib_req.request_id), ?error, "error creating websocket during restore" ); @@ -1120,7 +1214,7 @@ async fn handle_hws_restore( } } else { tracing::warn!( - request_id = id_to_str(&hib_req.request_id), + request_id = %display_id(&hib_req.request_id), "closing websocket that is not persisted" ); @@ -1148,7 +1242,7 @@ async fn handle_hws_restore( if !is_connected { tracing::warn!( - request_id = id_to_str(&meta.request_id), + request_id = %display_id(&meta.request_id), "removing stale persisted websocket" ); @@ -1200,7 +1294,7 @@ async fn handle_hws_ack( envoy_message_index: u16, ) { tracing::debug!( - request_id = id_to_str(&request_id), + request_id = %display_id(&request_id), index = envoy_message_index, "ack ws msg" ); @@ -1241,8 +1335,8 @@ async fn send_actor_message( idx } else { tracing::warn!( - gateway_id = id_to_str(&gateway_id), - request_id = id_to_str(&request_id), + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), "missing pending request for send message" ); return; @@ -1263,15 +1357,15 @@ async fn send_actor_message( if failed { if tracing::enabled!(tracing::Level::DEBUG) { tracing::debug!( - request_id = id_to_str(&request_id), + request_id = %display_id(&request_id), message = stringify_to_rivet_tunnel_message_kind(&message_kind), "buffering tunnel message, socket not connected to engine" ); } - let _ = ctx - .shared - .envoy_tx - .send(crate::envoy::ToEnvoyMessage::BufferTunnelMsg { msg: buffer_msg }); + let _ = crate::envoy::send_to_envoy_tx( + &ctx.shared, + crate::envoy::ToEnvoyMessage::BufferTunnelMsg { msg: buffer_msg }, + ); } } @@ -1662,6 +1756,8 @@ mod tests { protocol_metadata: Arc::new(tokio::sync::Mutex::new(None)), shutting_down: std::sync::atomic::AtomicBool::new(false), last_ping_ts: std::sync::atomic::AtomicI64::new(crate::time::now_millis()), + last_pong_sent_ts: std::sync::atomic::AtomicI64::new(crate::time::now_millis()), + ws_tx_depth: std::sync::atomic::AtomicI64::new(0), stopped_tx: tokio::sync::watch::channel(true).0, }); (shared, envoy_rx) diff --git a/engine/sdks/rust/envoy-client/src/connection/mod.rs b/engine/sdks/rust/envoy-client/src/connection/mod.rs index c2c066fb0b..3e02688ea9 100644 --- a/engine/sdks/rust/envoy-client/src/connection/mod.rs +++ b/engine/sdks/rust/envoy-client/src/connection/mod.rs @@ -15,12 +15,14 @@ use crate::context::WsTxMessage; all(feature = "wasm-transport", target_arch = "wasm32") ))] use crate::envoy::ToEnvoyMessage; +use crate::metrics::METRICS; #[cfg(any( feature = "native-transport", all(feature = "wasm-transport", target_arch = "wasm32") ))] use crate::stringify::stringify_to_envoy; use crate::stringify::stringify_to_rivet; +use crate::utils::display_id; #[cfg(all(feature = "native-transport", feature = "wasm-transport"))] compile_error!( @@ -95,32 +97,189 @@ async fn forward_to_envoy(shared: &SharedContext, message: protocol::ToEnvoy) { .await; } other => { - let _ = shared - .envoy_tx - .send(ToEnvoyMessage::ConnMessage { message: other }); + let _ = crate::envoy::send_to_envoy_tx( + shared, + ToEnvoyMessage::ConnMessage { message: other }, + ); } } } +fn observe_ping_unhealthy_on_close(shared: &SharedContext) { + let last_ping_ts = shared.last_ping_ts.load(Ordering::Acquire); + if last_ping_ts == 0 { + return; + } + + let unhealthy_ms = crate::time::now_millis() + .saturating_sub(last_ping_ts) + .saturating_sub(crate::handle::EnvoyHandle::PING_HEALTHY_THRESHOLD_MS); + if unhealthy_ms > 0 { + METRICS + .ping_unhealthy_seconds_total + .inc_by(unhealthy_ms as f64 / 1_000.0); + } +} + /// Send a message over the WebSocket. Returns true if the message could not be sent. pub async fn ws_send(shared: &SharedContext, message: protocol::ToRivet) -> bool { if tracing::enabled!(tracing::Level::DEBUG) { tracing::debug!(data = stringify_to_rivet(&message), "sending message"); } + let wait_start = crate::time::Instant::now(); + let is_pong = matches!(message, protocol::ToRivet::ToRivetPong(_)); + let (message_kind, gateway_id, request_id, message_index, inner_data_len) = + to_rivet_message_meta(&message); + let guard = shared.ws_tx.lock().await; + let wait_elapsed = wait_start.elapsed(); + METRICS + .ws_tx_lock_wait_duration_seconds + .with_label_values(&[message_kind]) + .observe(wait_elapsed.as_secs_f64()); + + let hold_start = crate::time::Instant::now(); let Some(tx) = guard.as_ref() else { - tracing::error!("websocket not available for sending"); + METRICS + .ws_tx_lock_hold_duration_seconds + .with_label_values(&[message_kind]) + .observe(hold_start.elapsed().as_secs_f64()); + if let (Some(gateway_id), Some(request_id), Some(message_index)) = + (gateway_id.as_ref(), request_id.as_ref(), message_index) + { + tracing::error!( + message_kind, + gateway_id = %display_id(gateway_id), + request_id = %display_id(request_id), + message_index, + inner_data_len, + "websocket not available for sending" + ); + } else { + tracing::error!( + message_kind, + inner_data_len, + "websocket not available for sending" + ); + } return true; }; let encoded = crate::protocol::versioned::ToRivet::wrap_latest(message) .serialize(protocol::PROTOCOL_VERSION) .expect("failed to encode message"); - let _ = tx.send(WsTxMessage::Send(encoded)); + let payload_len = encoded.len(); + let _ = tx.send(WsTxMessage::Send { + data: encoded, + enqueue_ts: crate::time::now_millis(), + is_pong, + message_kind, + gateway_id, + request_id, + message_index, + inner_data_len, + }); + shared.ws_tx_depth.fetch_add(1, Ordering::Release); + drop(guard); + METRICS + .ws_tx_lock_hold_duration_seconds + .with_label_values(&[message_kind]) + .observe(hold_start.elapsed().as_secs_f64()); + if let (Some(gateway_id), Some(request_id), Some(message_index)) = + (gateway_id.as_ref(), request_id.as_ref(), message_index) + { + tracing::trace!( + envoy_key = %shared.envoy_key, + message_kind, + gateway_id = %display_id(gateway_id), + request_id = %display_id(request_id), + message_index, + inner_data_len, + payload_len, + "queued websocket message to engine" + ); + } else { + tracing::trace!( + envoy_key = %shared.envoy_key, + message_kind, + inner_data_len, + payload_len, + "queued websocket message to engine" + ); + } false } +fn to_rivet_message_meta( + message: &protocol::ToRivet, +) -> ( + &'static str, + Option, + Option, + Option, + usize, +) { + match message { + protocol::ToRivet::ToRivetMetadata(_) => ("ToRivetMetadata", None, None, None, 0), + protocol::ToRivet::ToRivetEvents(_) => ("ToRivetEvents", None, None, None, 0), + protocol::ToRivet::ToRivetAckCommands(_) => ("ToRivetAckCommands", None, None, None, 0), + protocol::ToRivet::ToRivetStopping => ("ToRivetStopping", None, None, None, 0), + protocol::ToRivet::ToRivetPong(_) => ("ToRivetPong", None, None, None, 0), + protocol::ToRivet::ToRivetKvRequest(_) => ("ToRivetKvRequest", None, None, None, 0), + protocol::ToRivet::ToRivetTunnelMessage(msg) => ( + to_rivet_tunnel_message_kind_name(&msg.message_kind), + Some(msg.message_id.gateway_id), + Some(msg.message_id.request_id), + Some(msg.message_id.message_index), + to_rivet_tunnel_message_inner_data_len(&msg.message_kind), + ), + protocol::ToRivet::ToRivetSqliteGetPagesRequest(_) => { + ("ToRivetSqliteGetPagesRequest", None, None, None, 0) + } + protocol::ToRivet::ToRivetSqliteCommitRequest(_) => { + ("ToRivetSqliteCommitRequest", None, None, None, 0) + } + protocol::ToRivet::ToRivetSqliteExecRequest(_) => { + ("ToRivetSqliteExecRequest", None, None, None, 0) + } + protocol::ToRivet::ToRivetSqliteExecuteRequest(_) => { + ("ToRivetSqliteExecuteRequest", None, None, None, 0) + } + } +} + +fn to_rivet_tunnel_message_kind_name( + kind: &protocol::ToRivetTunnelMessageKind, +) -> &'static str { + match kind { + protocol::ToRivetTunnelMessageKind::ToRivetResponseStart(_) => "ToRivetResponseStart", + protocol::ToRivetTunnelMessageKind::ToRivetResponseChunk(_) => "ToRivetResponseChunk", + protocol::ToRivetTunnelMessageKind::ToRivetResponseAbort => "ToRivetResponseAbort", + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketOpen(_) => "ToRivetWebSocketOpen", + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketMessage(_) => { + "ToRivetWebSocketMessage" + } + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketMessageAck(_) => { + "ToRivetWebSocketMessageAck" + } + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketClose(_) => "ToRivetWebSocketClose", + } +} + +fn to_rivet_tunnel_message_inner_data_len(kind: &protocol::ToRivetTunnelMessageKind) -> usize { + match kind { + protocol::ToRivetTunnelMessageKind::ToRivetResponseStart(msg) => { + msg.body.as_ref().map_or(0, Vec::len) + } + protocol::ToRivetTunnelMessageKind::ToRivetResponseChunk(msg) => msg.body.len(), + protocol::ToRivetTunnelMessageKind::ToRivetWebSocketMessage(msg) => msg.data.len(), + protocol::ToRivetTunnelMessageKind::ToRivetResponseAbort + | protocol::ToRivetTunnelMessageKind::ToRivetWebSocketOpen(_) + | protocol::ToRivetTunnelMessageKind::ToRivetWebSocketMessageAck(_) + | protocol::ToRivetTunnelMessageKind::ToRivetWebSocketClose(_) => 0, + } +} #[cfg(any( feature = "native-transport", all(feature = "wasm-transport", target_arch = "wasm32") diff --git a/engine/sdks/rust/envoy-client/src/connection/native.rs b/engine/sdks/rust/envoy-client/src/connection/native.rs index 59d76e587d..f3023640c0 100644 --- a/engine/sdks/rust/envoy-client/src/connection/native.rs +++ b/engine/sdks/rust/envoy-client/src/connection/native.rs @@ -11,7 +11,8 @@ use vbare::OwnedVersionedData; use crate::context::{SharedContext, WsTxMessage}; use crate::envoy::ToEnvoyMessage; use crate::handle::EnvoyHandle; -use crate::utils::{BackoffOptions, calculate_backoff, parse_ws_close_reason}; +use crate::metrics::METRICS; +use crate::utils::{BackoffOptions, calculate_backoff, display_id, parse_ws_close_reason}; const STABLE_CONNECTION_MS: u64 = 60_000; @@ -36,21 +37,33 @@ async fn connection_loop(shared: Arc) { if let Some(reason) = &close_reason { if reason.group == "ws" && reason.error == "eviction" { tracing::debug!("connection evicted"); - let _ = shared - .envoy_tx - .send(ToEnvoyMessage::ConnClose { evict: true }); + let _ = crate::envoy::send_to_envoy_tx( + &shared, + ToEnvoyMessage::ConnClose { + evict: true, + was_error: false, + }, + ); return; } } - let _ = shared - .envoy_tx - .send(ToEnvoyMessage::ConnClose { evict: false }); + let _ = crate::envoy::send_to_envoy_tx( + &shared, + ToEnvoyMessage::ConnClose { + evict: false, + was_error: false, + }, + ); } Err(error) => { tracing::error!(?error, "connection failed"); - let _ = shared - .envoy_tx - .send(ToEnvoyMessage::ConnClose { evict: false }); + let _ = crate::envoy::send_to_envoy_tx( + &shared, + ToEnvoyMessage::ConnClose { + evict: false, + was_error: true, + }, + ); } } @@ -123,6 +136,9 @@ async fn single_connection( .callbacks .on_connect(EnvoyHandle::from_shared(shared.clone())); + let session_start = std::time::Instant::now(); + let mut disconnect_reason: &'static str = "stream_end"; + // Spawn write task let shared2 = shared.clone(); let write_span = tracing::debug_span!("envoy_ws_write", envoy_key = %shared2.envoy_key); @@ -130,16 +146,92 @@ async fn single_connection( async move { super::send_initial_metadata(&shared2).await; + // Threshold above which we log per-message timing diagnostics. Picked to surface + // backpressure that could plausibly contribute to engine ping timeouts (15s default) + // without spamming on normal operation. + const SLOW_WRITE_THRESHOLD_MS: i64 = 1_000; + while let Some(msg) = ws_rx.recv().await { match msg { - WsTxMessage::Send(data) => { + WsTxMessage::Send { + data, + enqueue_ts, + is_pong, + message_kind, + gateway_id, + request_id, + message_index, + inner_data_len, + } => { + let depth_after_recv = + shared2.ws_tx_depth.fetch_sub(1, Ordering::AcqRel) - 1; + let payload_len = data.len(); + let dequeue_ts = crate::time::now_millis(); + let queue_wait_ms = dequeue_ts - enqueue_ts; + let write_start = std::time::Instant::now(); let result = write .send(tungstenite::Message::Binary(data.into())) .await; + let write_elapsed_ms = write_start.elapsed().as_millis() as i64; if let Err(e) = result { tracing::error!(?e, "failed to send ws message"); break; } + let now = crate::time::now_millis(); + if is_pong { + shared2.last_pong_sent_ts.store(now, Ordering::Release); + } + let total_latency_ms = now - enqueue_ts; + if let (Some(gateway_id), Some(request_id), Some(message_index)) = + (gateway_id.as_ref(), request_id.as_ref(), message_index) + { + tracing::trace!( + envoy_key = %shared2.envoy_key, + message_kind, + gateway_id = %display_id(gateway_id), + request_id = %display_id(request_id), + message_index, + inner_data_len, + payload_len, + queue_wait_ms, + write_elapsed_ms, + total_latency_ms, + ws_tx_depth = depth_after_recv, + "wrote websocket message to engine" + ); + } else { + tracing::trace!( + envoy_key = %shared2.envoy_key, + message_kind, + inner_data_len, + payload_len, + queue_wait_ms, + write_elapsed_ms, + total_latency_ms, + ws_tx_depth = depth_after_recv, + "wrote websocket message to engine" + ); + } + if is_pong && total_latency_ms >= SLOW_WRITE_THRESHOLD_MS { + tracing::warn!( + envoy_key = %shared2.envoy_key, + queue_wait_ms, + write_elapsed_ms, + total_latency_ms, + ws_tx_depth = depth_after_recv, + "pong write exceeded slow threshold" + ); + } else if write_elapsed_ms >= SLOW_WRITE_THRESHOLD_MS { + tracing::warn!( + envoy_key = %shared2.envoy_key, + is_pong, + queue_wait_ms, + write_elapsed_ms, + total_latency_ms, + ws_tx_depth = depth_after_recv, + "ws outbound write exceeded slow threshold" + ); + } } WsTxMessage::Close => { let _ = write @@ -175,12 +267,20 @@ async fn single_connection( super::forward_to_envoy(shared, decoded).await; } Ok(tungstenite::Message::Close(frame)) => { + disconnect_reason = "close"; if let Some(frame) = frame { let reason_str = frame.reason.to_string(); let code: u16 = frame.code.into(); - tracing::info!( + let now = crate::time::now_millis(); + let since_last_pong_sent_ms = + now - shared.last_pong_sent_ts.load(Ordering::Acquire); + let ws_tx_depth = shared.ws_tx_depth.load(Ordering::Acquire); + tracing::warn!( + envoy_key = %shared.envoy_key, code, reason = %reason_str, + since_last_pong_sent_ms, + ws_tx_depth, "websocket closed" ); result = parse_ws_close_reason(&reason_str); @@ -188,13 +288,40 @@ async fn single_connection( break; } Err(e) => { - tracing::error!(?e, "websocket error"); + disconnect_reason = "error"; + let last_ping_ts = shared.last_ping_ts.load(std::sync::atomic::Ordering::Acquire); + let time_since_last_ping_ms = if last_ping_ts == 0 { + None + } else { + Some(crate::time::now_millis() - last_ping_ts) + }; + tracing::error!( + ?e, + ?time_since_last_ping_ms, + "websocket error" + ); break; } _ => {} } } + let session_duration = session_start.elapsed(); + METRICS + .ws_session_duration_seconds + .observe(session_duration.as_secs_f64()); + METRICS + .ws_reconnect_total + .with_label_values(&[disconnect_reason]) + .inc(); + super::observe_ping_unhealthy_on_close(shared); + tracing::info!( + envoy_key = %shared.envoy_key, + reason = disconnect_reason, + session_duration_ms = session_duration.as_millis() as u64, + "websocket session ended" + ); + // Clean up { let mut guard = shared.ws_tx.lock().await; diff --git a/engine/sdks/rust/envoy-client/src/connection/wasm.rs b/engine/sdks/rust/envoy-client/src/connection/wasm.rs index 03ac95b111..09446b5d12 100644 --- a/engine/sdks/rust/envoy-client/src/connection/wasm.rs +++ b/engine/sdks/rust/envoy-client/src/connection/wasm.rs @@ -15,7 +15,7 @@ mod imp { use crate::context::{SharedContext, WsTxMessage}; use crate::envoy::ToEnvoyMessage; - use crate::utils::{BackoffOptions, calculate_backoff, parse_ws_close_reason}; + use crate::utils::{BackoffOptions, calculate_backoff, display_id, parse_ws_close_reason}; const STABLE_CONNECTION_MS: u64 = 60_000; const NORMAL_CLOSE_CODE: u16 = 1000; @@ -49,21 +49,33 @@ mod imp { if let Some(reason) = &close_reason { if reason.group == "ws" && reason.error == "eviction" { tracing::debug!("connection evicted"); - let _ = shared - .envoy_tx - .send(ToEnvoyMessage::ConnClose { evict: true }); + let _ = crate::envoy::send_to_envoy_tx( + &shared, + ToEnvoyMessage::ConnClose { + evict: true, + was_error: false, + }, + ); return; } } - let _ = shared - .envoy_tx - .send(ToEnvoyMessage::ConnClose { evict: false }); + let _ = crate::envoy::send_to_envoy_tx( + &shared, + ToEnvoyMessage::ConnClose { + evict: false, + was_error: false, + }, + ); } Err(error) => { tracing::error!(?error, "connection failed"); - let _ = shared - .envoy_tx - .send(ToEnvoyMessage::ConnClose { evict: false }); + let _ = crate::envoy::send_to_envoy_tx( + &shared, + ToEnvoyMessage::ConnClose { + evict: false, + was_error: true, + }, + ); } } @@ -177,13 +189,63 @@ mod imp { while let Some(msg) = ws_rx.recv().await { match msg { - WsTxMessage::Send(data) => { + WsTxMessage::Send { + data, + enqueue_ts, + is_pong, + message_kind, + gateway_id, + request_id, + message_index, + inner_data_len, + } => { + let depth_after_recv = shared.ws_tx_depth.fetch_sub(1, Ordering::AcqRel) - 1; + let payload_len = data.len(); + let dequeue_ts = crate::time::now_millis(); + let queue_wait_ms = dequeue_ts - enqueue_ts; + let write_start = crate::time::now_millis(); let data = Uint8Array::from(data.as_slice()); if let Err(error) = ws.send_with_array_buffer(&data.buffer()) { tracing::error!(error = %js_error(error), "failed to send ws message"); let _ = event_tx.send(ConnectionEvent::WriteFailed); break; } + let now = crate::time::now_millis(); + if is_pong { + shared + .last_pong_sent_ts + .store(now, Ordering::Release); + } + if let (Some(gateway_id), Some(request_id), Some(message_index)) = + (gateway_id.as_ref(), request_id.as_ref(), message_index) + { + tracing::trace!( + envoy_key = %shared.envoy_key, + message_kind, + gateway_id = %display_id(gateway_id), + request_id = %display_id(request_id), + message_index, + inner_data_len, + payload_len, + queue_wait_ms, + write_elapsed_ms = now - write_start, + total_latency_ms = now - enqueue_ts, + ws_tx_depth = depth_after_recv, + "wrote websocket message to engine" + ); + } else { + tracing::trace!( + envoy_key = %shared.envoy_key, + message_kind, + inner_data_len, + payload_len, + queue_wait_ms, + write_elapsed_ms = now - write_start, + total_latency_ms = now - enqueue_ts, + ws_tx_depth = depth_after_recv, + "wrote websocket message to engine" + ); + } } WsTxMessage::Close => { let _ = @@ -231,6 +293,8 @@ mod imp { } } + super::super::observe_ping_unhealthy_on_close(shared); + { let mut guard = shared.ws_tx.lock().await; *guard = None; @@ -318,9 +382,13 @@ mod imp { use crate::envoy::ToEnvoyMessage; pub fn start_connection(shared: Arc) { - let _ = shared - .envoy_tx - .send(ToEnvoyMessage::ConnClose { evict: false }); + let _ = crate::envoy::send_to_envoy_tx( + &shared, + ToEnvoyMessage::ConnClose { + evict: false, + was_error: true, + }, + ); tracing::error!("wasm envoy transport requires the wasm32 target"); } } diff --git a/engine/sdks/rust/envoy-client/src/context.rs b/engine/sdks/rust/envoy-client/src/context.rs index d286e64a45..2b0387d36e 100644 --- a/engine/sdks/rust/envoy-client/src/context.rs +++ b/engine/sdks/rust/envoy-client/src/context.rs @@ -37,6 +37,12 @@ pub struct SharedContext { /// Initialized to the construction time so a freshly created envoy reports healthy until /// its first ping arrives or the threshold elapses without one. pub last_ping_ts: AtomicI64, + /// Epoch ms timestamp of when the most recent pong was actually written onto the WS by the + /// write task. Used to detect ws_tx backpressure on close. + pub last_pong_sent_ts: AtomicI64, + /// Current depth of the ws_tx mpsc channel. Bumped on every `ws_send` enqueue and decremented + /// when the write task dequeues. Used to expose backpressure at the moment of WS close. + pub ws_tx_depth: AtomicI64, // Latched signal fired by `envoy_loop` after its cleanup block completes. // Waiters observing `true` are guaranteed that the loop has exited and // every pending KV/SQLite request has been resolved (with `EnvoyShutdownError` @@ -46,6 +52,19 @@ pub struct SharedContext { #[derive(Debug)] pub enum WsTxMessage { - Send(Vec), + Send { + data: Vec, + /// Epoch ms when this message was enqueued. Used by the write task to compute internal + /// queue + write latency for diagnostic logs. + enqueue_ts: i64, + /// True if this message is a `ToRivetPong`. Pong-specific latency drives the engine ping + /// timeout detection, so we log its end-to-end timing separately. + is_pong: bool, + message_kind: &'static str, + gateway_id: Option, + request_id: Option, + message_index: Option, + inner_data_len: usize, + }, Close, } diff --git a/engine/sdks/rust/envoy-client/src/envoy.rs b/engine/sdks/rust/envoy-client/src/envoy.rs index 0e3cf80494..52fbdba529 100644 --- a/engine/sdks/rust/envoy-client/src/envoy.rs +++ b/engine/sdks/rust/envoy-client/src/envoy.rs @@ -24,6 +24,7 @@ use crate::kv::{ KV_CLEANUP_INTERVAL_MS, KvRequestEntry, cleanup_old_kv_requests, handle_kv_request, handle_kv_response, process_unsent_kv_requests, }; +use crate::metrics::METRICS; use crate::sqlite::{ RemoteSqliteRequest, RemoteSqliteRequestEntry, RemoteSqliteResponse, SqliteRequest, SqliteRequestEntry, SqliteResponse, cleanup_old_remote_sqlite_requests, @@ -90,6 +91,7 @@ pub enum ToEnvoyMessage { }, ConnClose { evict: bool, + was_error: bool, }, SendEvents { events: Vec, @@ -308,6 +310,8 @@ fn start_envoy_sync_inner(config: EnvoyConfig) -> EnvoyHandle { protocol_metadata: Arc::new(tokio::sync::Mutex::new(None)), shutting_down: std::sync::atomic::AtomicBool::new(false), last_ping_ts: std::sync::atomic::AtomicI64::new(crate::time::now_millis()), + last_pong_sent_ts: std::sync::atomic::AtomicI64::new(crate::time::now_millis()), + ws_tx_depth: std::sync::atomic::AtomicI64::new(0), stopped_tx, }); @@ -350,20 +354,73 @@ async fn envoy_loop( let mut kv_cleanup_tick = boxed_sleep(std::time::Duration::from_millis(KV_CLEANUP_INTERVAL_MS)); let mut lost_timeout: Option = None; + // Captured at the moment we arm the lost-threshold timer so we can observe + // `reconnect_within_grace_seconds` if a reconnect/init beats the timer. + let mut lost_timer_armed_at: Option = None; loop { + let iter_start = crate::time::Instant::now(); + #[allow(unused_assignments)] + let mut branch: &'static str = "unknown"; tokio::select! { msg = rx.recv() => { - let Some(msg) = msg else { break }; + branch = "envoy_msg"; + let Some(msg) = msg else { + observe_envoy_loop_iteration(branch, iter_start); + break; + }; + METRICS.envoy_tx_depth.dec(); match msg { ToEnvoyMessage::ConnMessage { message } => { + let was_armed = lost_timeout.is_some(); + let prev_armed_at = lost_timer_armed_at; lost_timeout = handle_conn_message(&mut ctx, &start_tx, lost_timeout, message).await; + // Detect lost-timer cancellation by a successful init/reconnect. + if was_armed && lost_timeout.is_none() { + METRICS + .lost_timer_outcome_total + .with_label_values(&["cancelled_by_reconnect"]) + .inc(); + if let Some(at) = prev_armed_at { + METRICS + .reconnect_within_grace_seconds + .observe(at.elapsed().as_secs_f64()); + } + lost_timer_armed_at = None; + } } - ToEnvoyMessage::ConnClose { evict } => { + ToEnvoyMessage::ConnClose { evict, was_error } => { fail_sent_remote_sqlite_requests_with_indeterminate_result(&mut ctx); + let was_armed = lost_timeout.is_some(); lost_timeout = handle_conn_close(&ctx, lost_timeout); - if evict { break; } + // Only count metrics on the first arm (handle_conn_close + // is a no-op when a timer is already armed). + if !was_armed && lost_timeout.is_some() { + let reason = if was_error { + "conn_close_error" + } else { + "conn_close_clean" + }; + METRICS + .lost_timer_armed_total + .with_label_values(&[reason]) + .inc(); + let source = if has_protocol_metadata(&ctx).await { + "metadata" + } else { + "fallback" + }; + METRICS + .lost_threshold_source_total + .with_label_values(&[source]) + .inc(); + lost_timer_armed_at = Some(crate::time::Instant::now()); + } + if evict { + observe_envoy_loop_iteration(branch, iter_start); + break; + } } ToEnvoyMessage::SendEvents { events } => { handle_send_events(&mut ctx, events).await; @@ -379,6 +436,7 @@ async fn envoy_loop( } ToEnvoyMessage::BufferTunnelMsg { msg } => { ctx.buffered_messages.push(msg); + METRICS.outbound_queue_depth.inc(); } ToEnvoyMessage::ActorIntent { actor_id, generation, intent, error } => { if let Some(entry) = ctx.get_actor(&actor_id, generation) { @@ -426,15 +484,18 @@ async fn envoy_loop( handle_shutdown(&mut ctx).await; } ToEnvoyMessage::Stop => { + observe_envoy_loop_iteration(branch, iter_start); break; } } } _ = ack_tick.as_mut() => { + branch = "ack_tick"; send_command_ack(&mut ctx).await; ack_tick = boxed_sleep(std::time::Duration::from_millis(ACK_COMMANDS_INTERVAL_MS)); } _ = kv_cleanup_tick.as_mut() => { + branch = "cleanup_tick"; cleanup_old_kv_requests(&mut ctx); cleanup_old_sqlite_requests(&mut ctx); cleanup_old_remote_sqlite_requests(&mut ctx); @@ -446,22 +507,56 @@ async fn envoy_loop( None => std::future::pending::<()>().await, } } => { + branch = "lost_timeout"; + METRICS + .lost_timer_outcome_total + .with_label_values(&["fired"]) + .inc(); // Lost timeout fired for (_id, request) in ctx.kv_requests.drain() { + METRICS.kv_requests_inflight.dec(); let _ = request.response_tx.send(Err(anyhow::anyhow!(EnvoyShutdownError))); } fail_sqlite_requests_with_shutdown(&mut ctx); fail_remote_sqlite_requests_with_shutdown(&mut ctx); if !ctx.actors.is_empty() { - tracing::warn!("stopping all actors due to envoy lost threshold"); + let threshold_source = if ctx.shared.protocol_metadata.try_lock() + .ok() + .and_then(|g| g.as_ref().map(|_| ())) + .is_some() + { + "metadata" + } else { + "fallback" + }; + let time_since_close_ms = lost_timer_armed_at + .map(|t| t.elapsed().as_millis() as u64) + .unwrap_or(0); + let actor_count: u64 = ctx + .actors + .values() + .map(|gens| gens.len() as u64) + .sum(); + tracing::warn!( + actor_count, + time_since_close_ms, + threshold_source, + "stopping all actors due to envoy lost threshold" + ); + let mut evicted = 0u64; for (_actor_id, gens) in &ctx.actors { for (_g, entry) in gens { if !entry.handle.is_closed() { let _ = entry.handle.send(ToActor::Lost); + evicted += 1; } } } + METRICS + .actor_evicted_total + .with_label_values(&["lost_threshold"]) + .inc_by(evicted); ctx.actors.clear(); ctx.shared .actors @@ -471,8 +566,10 @@ async fn envoy_loop( } lost_timeout = None; + lost_timer_armed_at = None; } } + observe_envoy_loop_iteration(branch, iter_start); } // Cleanup @@ -484,6 +581,7 @@ async fn envoy_loop( } for (_id, request) in ctx.kv_requests.drain() { + METRICS.kv_requests_inflight.dec(); let _ = request .response_tx .send(Err(anyhow::anyhow!("envoy shutting down"))); @@ -508,6 +606,30 @@ async fn envoy_loop( let _ = ctx.shared.stopped_tx.send(true); } +fn observe_envoy_loop_iteration(branch: &'static str, start: crate::time::Instant) { + let elapsed = start.elapsed(); + METRICS + .envoy_loop_iteration_duration_seconds + .with_label_values(&[branch]) + .observe(elapsed.as_secs_f64()); +} + +/// Send a message into the envoy_loop's mpsc and bump the depth gauge. +/// Producers should prefer this over calling `shared.envoy_tx.send` directly +/// so the `envoy_tx_depth` gauge stays in sync. +pub fn send_to_envoy_tx( + shared: &crate::context::SharedContext, + msg: ToEnvoyMessage, +) -> Result<(), tokio::sync::mpsc::error::SendError> { + match shared.envoy_tx.send(msg) { + Ok(()) => { + METRICS.envoy_tx_depth.inc(); + Ok(()) + } + Err(e) => Err(e), + } +} + async fn handle_conn_message( ctx: &mut EnvoyContext, start_tx: &tokio::sync::watch::Sender<()>, @@ -563,6 +685,13 @@ async fn handle_conn_message( lost_timeout } +/// True if the engine has delivered a `ToEnvoyInit` containing a protocol +/// metadata block (so the lost-threshold value would come from metadata rather +/// than the 10s local fallback). +async fn has_protocol_metadata(ctx: &EnvoyContext) -> bool { + ctx.shared.protocol_metadata.lock().await.is_some() +} + fn handle_conn_close(ctx: &EnvoyContext, lost_timeout: Option) -> Option { if lost_timeout.is_some() { return lost_timeout; @@ -576,7 +705,19 @@ fn handle_conn_close(ctx: &EnvoyContext, lost_timeout: Option) -> O .unwrap_or(10_000) }; - tracing::debug!(ms = lost_threshold, "starting envoy lost timeout"); + let source = if ctx + .shared + .protocol_metadata + .try_lock() + .ok() + .and_then(|guard| guard.as_ref().map(|m| m.envoy_lost_threshold)) + .is_some() + { + "metadata" + } else { + "fallback" + }; + tracing::info!(ms = lost_threshold, source, "starting envoy lost timeout"); Some(boxed_sleep(std::time::Duration::from_millis( lost_threshold, @@ -604,7 +745,7 @@ async fn handle_shutdown(ctx: &mut EnvoyContext) { .map(|entry| entry.handle.clone()) .collect(); - let envoy_tx = ctx.shared.envoy_tx.clone(); + let shared = ctx.shared.clone(); let shutdown_span = tracing::debug_span!( parent: tracing::Span::current(), "envoy_graceful_shutdown", @@ -614,7 +755,7 @@ async fn handle_shutdown(ctx: &mut EnvoyContext) { async move { futures_util::future::join_all(actor_handles.iter().map(|h| h.closed())).await; tracing::debug!("all actors stopped during graceful shutdown"); - let _ = envoy_tx.send(ToEnvoyMessage::Stop); + let _ = send_to_envoy_tx(&shared, ToEnvoyMessage::Stop); } .instrument(shutdown_span), ); diff --git a/engine/sdks/rust/envoy-client/src/events.rs b/engine/sdks/rust/envoy-client/src/events.rs index b3fcbdf7a0..9a021e00ed 100644 --- a/engine/sdks/rust/envoy-client/src/events.rs +++ b/engine/sdks/rust/envoy-client/src/events.rs @@ -173,6 +173,8 @@ mod tests { protocol_metadata: Arc::new(tokio::sync::Mutex::new(None)), shutting_down: std::sync::atomic::AtomicBool::new(false), last_ping_ts: std::sync::atomic::AtomicI64::new(crate::time::now_millis()), + last_pong_sent_ts: std::sync::atomic::AtomicI64::new(crate::time::now_millis()), + ws_tx_depth: std::sync::atomic::AtomicI64::new(0), stopped_tx: tokio::sync::watch::channel(true).0, }); let handle = EnvoyHandle { diff --git a/engine/sdks/rust/envoy-client/src/handle.rs b/engine/sdks/rust/envoy-client/src/handle.rs index 1d1b1ee9cc..65996d41a7 100644 --- a/engine/sdks/rust/envoy-client/src/handle.rs +++ b/engine/sdks/rust/envoy-client/src/handle.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::sync::{Arc, Weak}; use std::sync::atomic::Ordering; use crate::async_counter::AsyncCounter; @@ -7,6 +7,7 @@ use tokio::sync::oneshot; use crate::context::SharedContext; use crate::envoy::{ActorInfo, ToEnvoyMessage}; +use crate::metrics::METRICS; use crate::sqlite::{RemoteSqliteRequest, RemoteSqliteResponse, SqliteRequest, SqliteResponse}; use crate::tunnel::HibernatingWebSocketMetadata; @@ -17,6 +18,11 @@ pub struct EnvoyHandle { pub(crate) started_rx: tokio::sync::watch::Receiver<()>, } +#[derive(Clone)] +pub struct EnvoyStatusHandle { + shared: Weak, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ServerlessActorStart { pub actor_id: String, @@ -36,9 +42,9 @@ impl EnvoyHandle { self.shared.shutting_down.store(true, Ordering::Release); if immediate { - let _ = self.shared.envoy_tx.send(ToEnvoyMessage::Stop); + let _ = crate::envoy::send_to_envoy_tx(&self.shared, ToEnvoyMessage::Stop); } else { - let _ = self.shared.envoy_tx.send(ToEnvoyMessage::Shutdown); + let _ = crate::envoy::send_to_envoy_tx(&self.shared, ToEnvoyMessage::Shutdown); } } @@ -78,12 +84,27 @@ impl EnvoyHandle { /// Threshold for `is_ping_healthy`. pub const PING_HEALTHY_THRESHOLD_MS: i64 = 20_000; - /// True when the engine sent a ping within `PING_HEALTHY_THRESHOLD_MS`. Returns false once - /// the engine link has been silently dead long enough that an upstream health check should - /// treat this envoy as unhealthy and recycle it. - pub fn is_ping_healthy(&self) -> bool { + /// Epoch ms timestamp of the most recent engine ping. + pub fn last_ping_at_ms(&self) -> Option { let last = self.shared.last_ping_ts.load(Ordering::Acquire); - crate::time::now_millis() - last < Self::PING_HEALTHY_THRESHOLD_MS + if last == 0 { + None + } else { + Some(last) + } + } + + /// Milliseconds since the most recent engine ping. + pub fn last_ping_age_ms(&self) -> Option { + self.last_ping_at_ms() + .map(|last| crate::time::now_millis().saturating_sub(last)) + } + + /// True when the most recent engine ping timestamp is within `PING_HEALTHY_THRESHOLD_MS`. + /// Fresh envoys start healthy until the threshold elapses without a ping. + pub fn is_ping_healthy(&self) -> bool { + self.last_ping_age_ms() + .is_some_and(|age_ms| age_ms < Self::PING_HEALTHY_THRESHOLD_MS) } pub fn get_envoy_key(&self) -> &str { @@ -103,20 +124,13 @@ impl EnvoyHandle { } pub fn active_actor_count(&self) -> usize { - let guard = self - .shared - .actors - .lock() - .expect("shared actor registry poisoned"); - guard - .values() - .map(|generations| { - generations - .values() - .filter(|actor| !actor.handle.is_closed()) - .count() - }) - .sum() + active_actor_count(&self.shared) + } + + pub fn status_handle(&self) -> EnvoyStatusHandle { + EnvoyStatusHandle { + shared: Arc::downgrade(&self.shared), + } } pub fn pool_name(&self) -> &str { @@ -133,42 +147,52 @@ impl EnvoyHandle { } pub fn sleep_actor(&self, actor_id: String, generation: Option) { - let _ = self.shared.envoy_tx.send(ToEnvoyMessage::ActorIntent { - actor_id, - generation, - intent: protocol::ActorIntent::ActorIntentSleep, - error: None, - }); + let _ = crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::ActorIntent { + actor_id, + generation, + intent: protocol::ActorIntent::ActorIntentSleep, + error: None, + }, + ); } pub fn stop_actor(&self, actor_id: String, generation: Option, error: Option) { - let _ = self.shared.envoy_tx.send(ToEnvoyMessage::ActorIntent { - actor_id, - generation, - intent: protocol::ActorIntent::ActorIntentStop, - error, - }); + let _ = crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::ActorIntent { + actor_id, + generation, + intent: protocol::ActorIntent::ActorIntentStop, + error, + }, + ); } pub fn destroy_actor(&self, actor_id: String, generation: Option) { - let _ = self.shared.envoy_tx.send(ToEnvoyMessage::ActorIntent { - actor_id, - generation, - intent: protocol::ActorIntent::ActorIntentStop, - error: None, - }); + let _ = crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::ActorIntent { + actor_id, + generation, + intent: protocol::ActorIntent::ActorIntentStop, + error: None, + }, + ); } pub async fn get_actor(&self, actor_id: &str, generation: Option) -> Option { let (tx, rx) = tokio::sync::oneshot::channel(); - self.shared - .envoy_tx - .send(ToEnvoyMessage::GetActor { + crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::GetActor { actor_id: actor_id.to_string(), generation, response_tx: tx, - }) - .ok()?; + }, + ) + .ok()?; rx.await.ok().flatten() } @@ -282,12 +306,15 @@ impl EnvoyHandle { generation: Option, ack_tx: Option>, ) { - let _ = self.shared.envoy_tx.send(ToEnvoyMessage::SetAlarm { - actor_id, - generation, - alarm_ts, - ack_tx, - }); + let _ = crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::SetAlarm { + actor_id, + generation, + alarm_ts, + ack_tx, + }, + ); } pub async fn kv_get( @@ -542,11 +569,14 @@ impl EnvoyHandle { request_id: protocol::RequestId, client_message_index: u16, ) { - let _ = self.shared.envoy_tx.send(ToEnvoyMessage::HwsAck { - gateway_id, - request_id, - envoy_message_index: client_message_index, - }); + let _ = crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::HwsAck { + gateway_id, + request_id, + envoy_message_index: client_message_index, + }, + ); } /// Inject a serverless start payload into the envoy. @@ -567,9 +597,7 @@ impl EnvoyHandle { data = crate::stringify::stringify_to_envoy(&message), "received serverless start" ); - self.shared - .envoy_tx - .send(ToEnvoyMessage::ConnMessage { message }) + crate::envoy::send_to_envoy_tx(&self.shared, ToEnvoyMessage::ConnMessage { message }) .map_err(|_| anyhow::anyhow!("envoy channel closed"))?; Ok(()) @@ -584,6 +612,51 @@ impl EnvoyHandle { } } +impl EnvoyStatusHandle { + pub fn last_ping_at_ms(&self) -> Option { + self.shared.upgrade().and_then(|shared| { + let last = shared.last_ping_ts.load(Ordering::Acquire); + if last == 0 { + None + } else { + Some(last) + } + }) + } + + pub fn last_ping_age_ms(&self) -> Option { + self.last_ping_at_ms() + .map(|last| crate::time::now_millis().saturating_sub(last)) + } + + pub fn is_ping_healthy(&self) -> bool { + self.last_ping_age_ms() + .is_some_and(|age_ms| age_ms < EnvoyHandle::PING_HEALTHY_THRESHOLD_MS) + } + + pub fn active_actor_count(&self) -> Option { + self.shared + .upgrade() + .map(|shared| active_actor_count(&shared)) + } +} + +fn active_actor_count(shared: &SharedContext) -> usize { + let guard = shared + .actors + .lock() + .expect("shared actor registry poisoned"); + guard + .values() + .map(|generations| { + generations + .values() + .filter(|actor| !actor.handle.is_closed()) + .count() + }) + .sum() +} + fn decode_serverless_actor_start_payload( payload: &[u8], ) -> anyhow::Result<(protocol::ToEnvoy, ServerlessActorStart)> { @@ -641,45 +714,90 @@ impl EnvoyHandle { data: protocol::KvRequestData, ) -> anyhow::Result { let (tx, rx) = tokio::sync::oneshot::channel(); - self.shared - .envoy_tx - .send(ToEnvoyMessage::KvRequest { + crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::KvRequest { actor_id, data, response_tx: tx, - }) - .map_err(|_| anyhow::anyhow!("envoy channel closed"))?; + }, + ) + .map_err(|_| anyhow::anyhow!("envoy channel closed"))?; rx.await .map_err(|_| anyhow::anyhow!("kv response channel closed"))? } async fn send_sqlite_request(&self, request: SqliteRequest) -> anyhow::Result { + let kind = request.kind(); + let total_start = crate::time::Instant::now(); + let submit_start = crate::time::Instant::now(); let (tx, rx) = tokio::sync::oneshot::channel(); - self.shared - .envoy_tx - .send(ToEnvoyMessage::SqliteRequest { + crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::SqliteRequest { request, response_tx: tx, - }) - .map_err(|_| anyhow::anyhow!("envoy channel closed"))?; - rx.await - .map_err(|_| anyhow::anyhow!("sqlite response channel closed"))? + }, + ) + .map_err(|_| anyhow::anyhow!("envoy channel closed"))?; + let submit_elapsed = submit_start.elapsed(); + METRICS + .sqlite_request_submit_duration_seconds + .with_label_values(&[kind]) + .observe(submit_elapsed.as_secs_f64()); + + let wait_start = crate::time::Instant::now(); + let result = rx + .await + .map_err(|_| anyhow::anyhow!("sqlite response channel closed"))?; + let wait_elapsed = wait_start.elapsed(); + METRICS + .sqlite_request_wait_duration_seconds + .with_label_values(&[kind]) + .observe(wait_elapsed.as_secs_f64()); + METRICS + .sqlite_request_total_duration_seconds + .with_label_values(&[kind]) + .observe(total_start.elapsed().as_secs_f64()); + result } async fn send_remote_sqlite_request( &self, request: RemoteSqliteRequest, ) -> anyhow::Result { + let kind = request.kind(); + let total_start = crate::time::Instant::now(); + let submit_start = crate::time::Instant::now(); let (tx, rx) = tokio::sync::oneshot::channel(); - self.shared - .envoy_tx - .send(ToEnvoyMessage::RemoteSqliteRequest { + crate::envoy::send_to_envoy_tx( + &self.shared, + ToEnvoyMessage::RemoteSqliteRequest { request, response_tx: tx, - }) - .map_err(|_| anyhow::anyhow!("envoy channel closed"))?; - rx.await - .map_err(|_| anyhow::anyhow!("remote sqlite response channel closed"))? + }, + ) + .map_err(|_| anyhow::anyhow!("envoy channel closed"))?; + let submit_elapsed = submit_start.elapsed(); + METRICS + .sqlite_request_submit_duration_seconds + .with_label_values(&[kind]) + .observe(submit_elapsed.as_secs_f64()); + + let wait_start = crate::time::Instant::now(); + let result = rx + .await + .map_err(|_| anyhow::anyhow!("remote sqlite response channel closed"))?; + let wait_elapsed = wait_start.elapsed(); + METRICS + .sqlite_request_wait_duration_seconds + .with_label_values(&[kind]) + .observe(wait_elapsed.as_secs_f64()); + METRICS + .sqlite_request_total_duration_seconds + .with_label_values(&[kind]) + .observe(total_start.elapsed().as_secs_f64()); + result } } diff --git a/engine/sdks/rust/envoy-client/src/kv.rs b/engine/sdks/rust/envoy-client/src/kv.rs index 1088aa902a..24d109d2f8 100644 --- a/engine/sdks/rust/envoy-client/src/kv.rs +++ b/engine/sdks/rust/envoy-client/src/kv.rs @@ -3,6 +3,7 @@ use tokio::sync::oneshot; use crate::connection::ws_send; use crate::envoy::EnvoyContext; +use crate::metrics::METRICS; pub struct KvRequestEntry { pub actor_id: String, @@ -33,6 +34,7 @@ pub async fn handle_kv_request( }; ctx.kv_requests.insert(request_id, entry); + METRICS.kv_requests_inflight.inc(); let ws_available = { let guard = ctx.shared.ws_tx.lock().await; @@ -48,6 +50,7 @@ pub async fn handle_kv_response(ctx: &mut EnvoyContext, response: protocol::ToEn let request = ctx.kv_requests.remove(&response.request_id); if let Some(request) = request { + METRICS.kv_requests_inflight.dec(); match response.data { protocol::KvResponseData::KvErrorResponse(ref e) => { let _ = request @@ -124,6 +127,13 @@ pub fn cleanup_old_kv_requests(ctx: &mut EnvoyContext) { for request_id in to_delete { if let Some(request) = ctx.kv_requests.remove(&request_id) { + METRICS.kv_requests_inflight.dec(); + tracing::warn!( + request_id, + was_sent = request.sent, + age_ms = now.duration_since(request.timestamp).as_millis() as u64, + "kv request expired by cleanup" + ); let _ = request .response_tx .send(Err(anyhow::anyhow!("KV request timed out"))); diff --git a/engine/sdks/rust/envoy-client/src/lib.rs b/engine/sdks/rust/envoy-client/src/lib.rs index 5a225112d6..583ac3a5f2 100644 --- a/engine/sdks/rust/envoy-client/src/lib.rs +++ b/engine/sdks/rust/envoy-client/src/lib.rs @@ -9,6 +9,7 @@ pub mod events; pub mod handle; pub mod kv; pub mod latency_channel; +pub mod metrics; pub mod sqlite; pub mod stringify; pub(crate) mod time { diff --git a/engine/sdks/rust/envoy-client/src/metrics.rs b/engine/sdks/rust/envoy-client/src/metrics.rs new file mode 100644 index 0000000000..46cdd6b197 --- /dev/null +++ b/engine/sdks/rust/envoy-client/src/metrics.rs @@ -0,0 +1,346 @@ +//! Process-wide envoy-client metrics. +//! +//! These metrics live in the actor pod and are intended to make the +//! single-WebSocket transport between the actor and the engine debuggable +//! during incidents (e.g. pod-local handshake latency spikes where sent +//! requests get abandoned on disconnect). +//! +//! All labels here are bounded by code-defined enums. No `actor_id`, +//! `actor_key`, `runner_id`, `request_id`, or other unbounded values may be +//! added here. + +use std::sync::LazyLock; + +use rivet_metrics::prometheus::{ + Counter, Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGauge, Opts, + Registry, +}; + +const SQLITE_REQUEST_EXPIRED_LABELS: &[&str] = &["kind", "was_sent"]; +const ENVOY_LOOP_ITER_LABELS: &[&str] = &["branch"]; +const WS_TX_LOCK_LABELS: &[&str] = &["message_kind"]; +const SQLITE_SEND_LABELS: &[&str] = &["kind"]; +const WS_RECONNECT_LABELS: &[&str] = &["reason"]; +const LOST_TIMER_ARMED_LABELS: &[&str] = &["reason"]; +const LOST_TIMER_OUTCOME_LABELS: &[&str] = &["outcome"]; +const LOST_THRESHOLD_SOURCE_LABELS: &[&str] = &["source"]; +const ACTOR_EVICTED_LABELS: &[&str] = &["reason"]; +const ACTOR_STOP_LABELS: &[&str] = &["reason"]; +const ACTOR_LIFETIME_LABELS: &[&str] = &["reason"]; + +pub struct EnvoyClientMetrics { + pub sqlite_request_expired_total: IntCounterVec, + pub sqlite_requests_inflight: IntGauge, + pub remote_sqlite_requests_inflight: IntGauge, + pub kv_requests_inflight: IntGauge, + pub envoy_loop_iteration_duration_seconds: HistogramVec, + pub ws_tx_lock_wait_duration_seconds: HistogramVec, + pub ws_tx_lock_hold_duration_seconds: HistogramVec, + pub sqlite_request_total_duration_seconds: HistogramVec, + pub sqlite_request_submit_duration_seconds: HistogramVec, + pub sqlite_request_wait_duration_seconds: HistogramVec, + pub ws_reconnect_total: IntCounterVec, + pub ws_session_duration_seconds: Histogram, + pub envoy_tx_depth: IntGauge, + pub lost_timer_armed_total: IntCounterVec, + pub lost_timer_outcome_total: IntCounterVec, + pub reconnect_within_grace_seconds: Histogram, + pub lost_threshold_source_total: IntCounterVec, + pub actor_evicted_total: IntCounterVec, + pub outbound_queue_depth: IntGauge, + pub ping_unhealthy_seconds_total: Counter, + pub ping_unhealthy_recovered_total: IntCounter, + pub actor_stop_total: IntCounterVec, + pub actor_lifetime_seconds: HistogramVec, +} + +impl EnvoyClientMetrics { + fn new() -> Self { + let sqlite_request_expired_total = IntCounterVec::new( + Opts::new( + "rivetkit_envoy_client_sqlite_request_expired_total", + "total VFS sqlite requests expired by cleanup (smoking-gun signal: was_sent=true means a request was abandoned without resolution)", + ), + SQLITE_REQUEST_EXPIRED_LABELS, + ) + .expect("create envoy_client_sqlite_request_expired_total counter"); + + let sqlite_requests_inflight = IntGauge::new( + "rivetkit_envoy_client_sqlite_requests_inflight", + "current in-flight VFS sqlite requests tracked in envoy ctx", + ) + .expect("create envoy_client_sqlite_requests_inflight gauge"); + + let remote_sqlite_requests_inflight = IntGauge::new( + "rivetkit_envoy_client_remote_sqlite_requests_inflight", + "current in-flight remote sqlite (exec/execute) requests tracked in envoy ctx", + ) + .expect("create envoy_client_remote_sqlite_requests_inflight gauge"); + + let kv_requests_inflight = IntGauge::new( + "rivetkit_envoy_client_kv_requests_inflight", + "current in-flight KV requests tracked in envoy ctx", + ) + .expect("create envoy_client_kv_requests_inflight gauge"); + + let envoy_loop_iteration_duration_seconds = HistogramVec::new( + HistogramOpts::new( + "rivetkit_envoy_client_envoy_loop_iteration_duration_seconds", + "duration of one envoy_loop select branch in seconds; long tails indicate the single fan-in loop stalled", + ) + .buckets(rivet_metrics::MICRO_BUCKETS.to_vec()), + ENVOY_LOOP_ITER_LABELS, + ) + .expect("create envoy_client_envoy_loop_iteration_duration_seconds histogram"); + + let ws_tx_lock_wait_duration_seconds = HistogramVec::new( + HistogramOpts::new( + "rivetkit_envoy_client_ws_tx_lock_wait_duration_seconds", + "time spent waiting to acquire the ws_tx mutex in seconds", + ) + .buckets(rivet_metrics::MICRO_BUCKETS.to_vec()), + WS_TX_LOCK_LABELS, + ) + .expect("create envoy_client_ws_tx_lock_wait_duration_seconds histogram"); + + let ws_tx_lock_hold_duration_seconds = HistogramVec::new( + HistogramOpts::new( + "rivetkit_envoy_client_ws_tx_lock_hold_duration_seconds", + "time the ws_tx mutex guard was held in seconds (covers encode + send)", + ) + .buckets(rivet_metrics::MICRO_BUCKETS.to_vec()), + WS_TX_LOCK_LABELS, + ) + .expect("create envoy_client_ws_tx_lock_hold_duration_seconds histogram"); + + let sqlite_request_total_duration_seconds = HistogramVec::new( + HistogramOpts::new( + "rivetkit_envoy_client_sqlite_request_total_duration_seconds", + "end-to-end duration from send_sqlite_request call to oneshot resolution in seconds", + ) + .buckets(rivet_metrics::BUCKETS.to_vec()), + SQLITE_SEND_LABELS, + ) + .expect("create envoy_client_sqlite_request_total_duration_seconds histogram"); + + let sqlite_request_submit_duration_seconds = HistogramVec::new( + HistogramOpts::new( + "rivetkit_envoy_client_sqlite_request_submit_duration_seconds", + "time from send_sqlite_request entry to envoy_tx.send return in seconds", + ) + .buckets(rivet_metrics::MICRO_BUCKETS.to_vec()), + SQLITE_SEND_LABELS, + ) + .expect("create envoy_client_sqlite_request_submit_duration_seconds histogram"); + + let sqlite_request_wait_duration_seconds = HistogramVec::new( + HistogramOpts::new( + "rivetkit_envoy_client_sqlite_request_wait_duration_seconds", + "time from envoy_tx.send return to oneshot resolution in seconds", + ) + .buckets(rivet_metrics::BUCKETS.to_vec()), + SQLITE_SEND_LABELS, + ) + .expect("create envoy_client_sqlite_request_wait_duration_seconds histogram"); + + let ws_reconnect_total = IntCounterVec::new( + Opts::new( + "rivetkit_envoy_client_ws_reconnect_total", + "total websocket reconnect events by reason", + ), + WS_RECONNECT_LABELS, + ) + .expect("create envoy_client_ws_reconnect_total counter"); + + let ws_session_duration_seconds = Histogram::with_opts( + HistogramOpts::new( + "rivetkit_envoy_client_ws_session_duration_seconds", + "duration of a single websocket session from connect to disconnect in seconds", + ) + .buckets(rivet_metrics::BUCKETS.to_vec()), + ) + .expect("create envoy_client_ws_session_duration_seconds histogram"); + + let envoy_tx_depth = IntGauge::new( + "rivetkit_envoy_client_envoy_tx_depth", + "current depth of the unbounded envoy_tx mpsc between WS read task and envoy_loop", + ) + .expect("create envoy_client_envoy_tx_depth gauge"); + + let lost_timer_armed_total = IntCounterVec::new( + Opts::new( + "rivetkit_envoy_client_lost_timer_armed_total", + "total lost-threshold timers armed in handle_conn_close, labeled by close reason", + ), + LOST_TIMER_ARMED_LABELS, + ) + .expect("create envoy_client_lost_timer_armed_total counter"); + + let lost_timer_outcome_total = IntCounterVec::new( + Opts::new( + "rivetkit_envoy_client_lost_timer_outcome_total", + "total lost-threshold timer outcomes (fired vs cancelled by reconnect/init)", + ), + LOST_TIMER_OUTCOME_LABELS, + ) + .expect("create envoy_client_lost_timer_outcome_total counter"); + + let reconnect_within_grace_seconds = Histogram::with_opts( + HistogramOpts::new( + "rivetkit_envoy_client_reconnect_within_grace_seconds", + "seconds from WS close to successful reconnect/init when the reconnect beats the lost-threshold timer", + ) + .buckets(rivet_metrics::BUCKETS.to_vec()), + ) + .expect("create envoy_client_reconnect_within_grace_seconds histogram"); + + let lost_threshold_source_total = IntCounterVec::new( + Opts::new( + "rivetkit_envoy_client_lost_threshold_source_total", + "source of the lost-threshold value used per timer arm (protocol metadata vs local fallback)", + ), + LOST_THRESHOLD_SOURCE_LABELS, + ) + .expect("create envoy_client_lost_threshold_source_total counter"); + + let actor_evicted_total = IntCounterVec::new( + Opts::new( + "rivetkit_envoy_client_actor_evicted_total", + "total actors evicted by the envoy, labeled by eviction reason", + ), + ACTOR_EVICTED_LABELS, + ) + .expect("create envoy_client_actor_evicted_total counter"); + + let outbound_queue_depth = IntGauge::new( + "rivetkit_envoy_client_outbound_queue_depth", + "current depth of the ToRivet outbound tunnel-message buffer awaiting a reconnect", + ) + .expect("create envoy_client_outbound_queue_depth gauge"); + + let ping_unhealthy_seconds_total = Counter::new( + "rivetkit_envoy_client_ping_unhealthy_seconds_total", + "cumulative seconds spent with is_ping_healthy()==false while the WS is still open", + ) + .expect("create envoy_client_ping_unhealthy_seconds_total counter"); + + let ping_unhealthy_recovered_total = IntCounter::new( + "rivetkit_envoy_client_ping_unhealthy_recovered_total", + "total transitions from ping-unhealthy back to ping-healthy without a WS close", + ) + .expect("create envoy_client_ping_unhealthy_recovered_total counter"); + + let actor_stop_total = IntCounterVec::new( + Opts::new( + "rivetkit_envoy_client_actor_stop_total", + "total actor stops handled by the envoy, labeled by StopActorReason", + ), + ACTOR_STOP_LABELS, + ) + .expect("create envoy_client_actor_stop_total counter"); + + let actor_lifetime_seconds = HistogramVec::new( + HistogramOpts::new( + "rivetkit_envoy_client_actor_lifetime_seconds", + "actor lifetime from create_actor to stop in seconds, labeled by StopActorReason", + ) + .buckets(rivet_metrics::LIFETIME_BUCKETS.to_vec()), + ACTOR_LIFETIME_LABELS, + ) + .expect("create envoy_client_actor_lifetime_seconds histogram"); + + register(&rivet_metrics::REGISTRY, sqlite_request_expired_total.clone()); + register(&rivet_metrics::REGISTRY, sqlite_requests_inflight.clone()); + register( + &rivet_metrics::REGISTRY, + remote_sqlite_requests_inflight.clone(), + ); + register(&rivet_metrics::REGISTRY, kv_requests_inflight.clone()); + register( + &rivet_metrics::REGISTRY, + envoy_loop_iteration_duration_seconds.clone(), + ); + register( + &rivet_metrics::REGISTRY, + ws_tx_lock_wait_duration_seconds.clone(), + ); + register( + &rivet_metrics::REGISTRY, + ws_tx_lock_hold_duration_seconds.clone(), + ); + register( + &rivet_metrics::REGISTRY, + sqlite_request_total_duration_seconds.clone(), + ); + register( + &rivet_metrics::REGISTRY, + sqlite_request_submit_duration_seconds.clone(), + ); + register( + &rivet_metrics::REGISTRY, + sqlite_request_wait_duration_seconds.clone(), + ); + register(&rivet_metrics::REGISTRY, ws_reconnect_total.clone()); + register( + &rivet_metrics::REGISTRY, + ws_session_duration_seconds.clone(), + ); + register(&rivet_metrics::REGISTRY, envoy_tx_depth.clone()); + register(&rivet_metrics::REGISTRY, lost_timer_armed_total.clone()); + register(&rivet_metrics::REGISTRY, lost_timer_outcome_total.clone()); + register( + &rivet_metrics::REGISTRY, + reconnect_within_grace_seconds.clone(), + ); + register(&rivet_metrics::REGISTRY, lost_threshold_source_total.clone()); + register(&rivet_metrics::REGISTRY, actor_evicted_total.clone()); + register(&rivet_metrics::REGISTRY, outbound_queue_depth.clone()); + register(&rivet_metrics::REGISTRY, ping_unhealthy_seconds_total.clone()); + register( + &rivet_metrics::REGISTRY, + ping_unhealthy_recovered_total.clone(), + ); + register(&rivet_metrics::REGISTRY, actor_stop_total.clone()); + register(&rivet_metrics::REGISTRY, actor_lifetime_seconds.clone()); + + Self { + sqlite_request_expired_total, + sqlite_requests_inflight, + remote_sqlite_requests_inflight, + kv_requests_inflight, + envoy_loop_iteration_duration_seconds, + ws_tx_lock_wait_duration_seconds, + ws_tx_lock_hold_duration_seconds, + sqlite_request_total_duration_seconds, + sqlite_request_submit_duration_seconds, + sqlite_request_wait_duration_seconds, + ws_reconnect_total, + ws_session_duration_seconds, + envoy_tx_depth, + lost_timer_armed_total, + lost_timer_outcome_total, + reconnect_within_grace_seconds, + lost_threshold_source_total, + actor_evicted_total, + outbound_queue_depth, + ping_unhealthy_seconds_total, + ping_unhealthy_recovered_total, + actor_stop_total, + actor_lifetime_seconds, + } + } +} + +pub static METRICS: LazyLock = LazyLock::new(EnvoyClientMetrics::new); + +fn register(registry: &Registry, metric: M) +where + M: rivet_metrics::prometheus::core::Collector + Clone + Send + Sync + 'static, +{ + if let Err(error) = registry.register(Box::new(metric)) { + tracing::warn!( + ?error, + "envoy-client metric registration failed, using existing collector" + ); + } +} diff --git a/engine/sdks/rust/envoy-client/src/sqlite.rs b/engine/sdks/rust/envoy-client/src/sqlite.rs index 26e92623d5..d1530bc6fb 100644 --- a/engine/sdks/rust/envoy-client/src/sqlite.rs +++ b/engine/sdks/rust/envoy-client/src/sqlite.rs @@ -4,6 +4,7 @@ use tokio::sync::oneshot; use crate::connection::ws_send; use crate::envoy::EnvoyContext; use crate::kv::KV_EXPIRE_MS; +use crate::metrics::METRICS; use crate::utils::{EnvoyShutdownError, RemoteSqliteIndeterminateResultError}; #[derive(Clone)] @@ -12,6 +13,15 @@ pub enum SqliteRequest { Commit(protocol::SqliteCommitRequest), } +impl SqliteRequest { + pub fn kind(&self) -> &'static str { + match self { + SqliteRequest::GetPages(_) => "get_pages", + SqliteRequest::Commit(_) => "commit", + } + } +} + pub enum SqliteResponse { GetPages(protocol::SqliteGetPagesResponse), Commit(protocol::SqliteCommitResponse), @@ -36,6 +46,13 @@ impl RemoteSqliteRequest { RemoteSqliteRequest::Execute(_) => "execute", } } + + pub fn kind(&self) -> &'static str { + match self { + RemoteSqliteRequest::Exec(_) => "remote_exec", + RemoteSqliteRequest::Execute(_) => "remote_execute", + } + } } pub struct SqliteRequestEntry { @@ -68,6 +85,7 @@ pub async fn handle_sqlite_request( }; ctx.sqlite_requests.insert(request_id, entry); + METRICS.sqlite_requests_inflight.inc(); let ws_available = { let guard = ctx.shared.ws_tx.lock().await; @@ -95,6 +113,7 @@ pub async fn handle_remote_sqlite_request( }; ctx.remote_sqlite_requests.insert(request_id, entry); + METRICS.remote_sqlite_requests_inflight.inc(); let ws_available = { let guard = ctx.shared.ws_tx.lock().await; @@ -163,6 +182,7 @@ fn handle_sqlite_response( let request = ctx.sqlite_requests.remove(&request_id); if let Some(request) = request { + METRICS.sqlite_requests_inflight.dec(); let _ = request.response_tx.send(Ok(response)); } else { tracing::error!( @@ -182,6 +202,7 @@ fn handle_remote_sqlite_response( let request = ctx.remote_sqlite_requests.remove(&request_id); if let Some(request) = request { + METRICS.remote_sqlite_requests_inflight.dec(); let _ = request.response_tx.send(Ok(response)); } else { tracing::error!( @@ -310,6 +331,20 @@ pub fn cleanup_old_sqlite_requests(ctx: &mut EnvoyContext) { for request_id in to_delete { if let Some(request) = ctx.sqlite_requests.remove(&request_id) { + let kind = request.request.kind(); + let was_sent = if request.sent { "true" } else { "false" }; + METRICS + .sqlite_request_expired_total + .with_label_values(&[kind, was_sent]) + .inc(); + METRICS.sqlite_requests_inflight.dec(); + tracing::warn!( + request_id, + kind, + was_sent = request.sent, + age_ms = now.duration_since(request.timestamp).as_millis() as u64, + "sqlite request expired by cleanup; if was_sent=true this indicates an abandoned in-flight request" + ); let _ = request .response_tx .send(Err(anyhow::anyhow!("sqlite request timed out"))); @@ -329,6 +364,20 @@ pub fn cleanup_old_remote_sqlite_requests(ctx: &mut EnvoyContext) { for request_id in to_delete { if let Some(request) = ctx.remote_sqlite_requests.remove(&request_id) { + let kind = request.request.kind(); + let was_sent = if request.sent { "true" } else { "false" }; + METRICS + .sqlite_request_expired_total + .with_label_values(&[kind, was_sent]) + .inc(); + METRICS.remote_sqlite_requests_inflight.dec(); + tracing::warn!( + request_id, + kind, + was_sent = request.sent, + age_ms = now.duration_since(request.timestamp).as_millis() as u64, + "remote sqlite request expired by cleanup; if was_sent=true this indicates an abandoned in-flight request" + ); let _ = request .response_tx .send(Err(anyhow::anyhow!("remote sqlite request timed out"))); @@ -338,6 +387,7 @@ pub fn cleanup_old_remote_sqlite_requests(ctx: &mut EnvoyContext) { pub fn fail_sqlite_requests_with_shutdown(ctx: &mut EnvoyContext) { for (_id, request) in ctx.sqlite_requests.drain() { + METRICS.sqlite_requests_inflight.dec(); let _ = request .response_tx .send(Err(anyhow::anyhow!(EnvoyShutdownError))); @@ -346,6 +396,7 @@ pub fn fail_sqlite_requests_with_shutdown(ctx: &mut EnvoyContext) { pub fn fail_remote_sqlite_requests_with_shutdown(ctx: &mut EnvoyContext) { for (_id, request) in ctx.remote_sqlite_requests.drain() { + METRICS.remote_sqlite_requests_inflight.dec(); let _ = request .response_tx .send(Err(anyhow::anyhow!(EnvoyShutdownError))); @@ -362,6 +413,7 @@ pub fn fail_sent_remote_sqlite_requests_with_indeterminate_result(ctx: &mut Envo for request_id in request_ids { if let Some(request) = ctx.remote_sqlite_requests.remove(&request_id) { + METRICS.remote_sqlite_requests_inflight.dec(); let operation = request.request.operation(); tracing::warn!( request_id, @@ -471,6 +523,9 @@ mod tests { )), protocol_metadata: Arc::new(tokio::sync::Mutex::new(None)), shutting_down: std::sync::atomic::AtomicBool::new(false), + last_ping_ts: std::sync::atomic::AtomicI64::new(crate::time::now_millis()), + last_pong_sent_ts: std::sync::atomic::AtomicI64::new(crate::time::now_millis()), + ws_tx_depth: std::sync::atomic::AtomicI64::new(0), stopped_tx: tokio::sync::watch::channel(true).0, }); @@ -608,7 +663,7 @@ mod tests { tx, ) .await; - assert!(matches!(ws_rx.recv().await, Some(WsTxMessage::Send(_)))); + assert!(matches!(ws_rx.recv().await, Some(WsTxMessage::Send { .. }))); assert!( ctx.remote_sqlite_requests .get(&0) @@ -658,7 +713,7 @@ mod tests { *ctx.shared.ws_tx.lock().await = Some(ws_tx); process_unsent_remote_sqlite_requests(&mut ctx).await; - assert!(matches!(ws_rx.recv().await, Some(WsTxMessage::Send(_)))); + assert!(matches!(ws_rx.recv().await, Some(WsTxMessage::Send { .. }))); assert!( ctx.remote_sqlite_requests .get(&0) diff --git a/engine/sdks/rust/envoy-client/src/tunnel.rs b/engine/sdks/rust/envoy-client/src/tunnel.rs index 910b243530..39a13e9f9a 100644 --- a/engine/sdks/rust/envoy-client/src/tunnel.rs +++ b/engine/sdks/rust/envoy-client/src/tunnel.rs @@ -2,6 +2,7 @@ use rivet_envoy_protocol as protocol; use crate::connection::ws_send; use crate::envoy::{BufferedActorMessage, EnvoyContext}; +use crate::utils::display_id; fn make_ws_key(gateway_id: &protocol::GatewayId, request_id: &protocol::RequestId) -> [u8; 8] { let mut key = [0u8; 8]; @@ -21,6 +22,17 @@ pub struct HibernatingWebSocketMetadata { pub async fn handle_tunnel_message(ctx: &mut EnvoyContext, msg: protocol::ToEnvoyTunnelMessage) { let message_id = msg.message_id; + let message_index = message_id.message_index; + let message_kind = to_envoy_tunnel_message_kind_name(&msg.message_kind); + let inner_data_len = to_envoy_tunnel_message_inner_data_len(&msg.message_kind); + tracing::trace!( + gateway_id = %display_id(&message_id.gateway_id), + request_id = %display_id(&message_id.request_id), + message_index, + message_kind, + inner_data_len, + "received tunnel message from engine" + ); match msg.message_kind { protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestStart(req) => { handle_request_start(ctx, message_id, req).await; @@ -173,21 +185,75 @@ fn handle_ws_message( message_id: protocol::MessageId, msg: protocol::ToEnvoyWebSocketMessage, ) { + let data_len = msg.data.len(); + let binary = msg.binary; + let gateway_id = message_id.gateway_id; + let request_id = message_id.request_id; + let message_index = message_id.message_index; let actor_id = ctx .request_to_actor .get(&[&message_id.gateway_id, &message_id.request_id]) .cloned(); if let Some(actor_id) = &actor_id { if let Some(actor) = ctx.get_actor(actor_id, None) { - let _ = actor + tracing::trace!( + actor_id = %actor_id, + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "dispatching websocket message to actor task" + ); + if actor .handle - .send(crate::actor::ToActor::WsMsg { message_id, msg }); + .send(crate::actor::ToActor::WsMsg { message_id, msg }) + .is_ok() + { + tracing::trace!( + actor_id = %actor_id, + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "dispatched websocket message to actor task" + ); + } else { + tracing::warn!( + actor_id = %actor_id, + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "actor websocket task channel closed" + ); + } } else { + tracing::trace!( + actor_id = %actor_id, + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "buffering websocket message for actor task" + ); ctx.buffered_actor_messages .entry(actor_id.clone()) .or_default() .push(BufferedActorMessage::WsMsg { message_id, msg }); } + } else { + tracing::warn!( + gateway_id = %display_id(&gateway_id), + request_id = %display_id(&request_id), + message_index, + data_len, + binary, + "received websocket message for unknown tunnel request" + ); } } @@ -226,6 +292,34 @@ fn handle_ws_close( .remove(&make_ws_key(&message_id.gateway_id, &message_id.request_id)); } +fn to_envoy_tunnel_message_kind_name( + kind: &protocol::ToEnvoyTunnelMessageKind, +) -> &'static str { + match kind { + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestStart(_) => "ToEnvoyRequestStart", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestChunk(_) => "ToEnvoyRequestChunk", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestAbort => "ToEnvoyRequestAbort", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketOpen(_) => "ToEnvoyWebSocketOpen", + protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketMessage(_) => { + "ToEnvoyWebSocketMessage" + } + protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketClose(_) => "ToEnvoyWebSocketClose", + } +} + +fn to_envoy_tunnel_message_inner_data_len(kind: &protocol::ToEnvoyTunnelMessageKind) -> usize { + match kind { + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestStart(msg) => { + msg.body.as_ref().map_or(0, Vec::len) + } + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestChunk(msg) => msg.body.len(), + protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketMessage(msg) => msg.data.len(), + protocol::ToEnvoyTunnelMessageKind::ToEnvoyRequestAbort + | protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketOpen(_) + | protocol::ToEnvoyTunnelMessageKind::ToEnvoyWebSocketClose(_) => 0, + } +} + pub fn send_hibernatable_ws_message_ack( ctx: &mut EnvoyContext, gateway_id: protocol::GatewayId, @@ -258,6 +352,9 @@ pub async fn resend_buffered_tunnel_messages(ctx: &mut EnvoyContext) { ); let messages = std::mem::take(&mut ctx.buffered_messages); + crate::metrics::METRICS + .outbound_queue_depth + .sub(messages.len() as i64); for msg in messages { ws_send(&ctx.shared, protocol::ToRivet::ToRivetTunnelMessage(msg)).await; } diff --git a/engine/sdks/rust/envoy-client/src/utils.rs b/engine/sdks/rust/envoy-client/src/utils.rs index bd0c0671e7..5ee446a30c 100644 --- a/engine/sdks/rust/envoy-client/src/utils.rs +++ b/engine/sdks/rust/envoy-client/src/utils.rs @@ -14,6 +14,22 @@ pub fn id_to_str(id: &[u8]) -> String { hex::encode(id) } +pub fn display_id(id: &[u8]) -> DisplayId<'_> { + DisplayId(id) +} + +#[derive(Clone, Copy)] +pub struct DisplayId<'a>(&'a [u8]); + +impl std::fmt::Display for DisplayId<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for byte in self.0 { + write!(f, "{byte:02x}")?; + } + Ok(()) + } +} + /// Stringify an error for logging. pub fn stringify_error(error: &anyhow::Error) -> String { format!("{error:#}") diff --git a/examples/kitchen-sink/CLAUDE.md b/examples/kitchen-sink/CLAUDE.md index e1c9ca1ddd..9099e2db8c 100644 --- a/examples/kitchen-sink/CLAUDE.md +++ b/examples/kitchen-sink/CLAUDE.md @@ -2,9 +2,44 @@ ## Testing Against Production (Rivet Cloud) +### Rivet Cloud Managed-Pool Deploys + +- The `Rivet Deploy (kitchen-sink)` GitHub Action (`.github/workflows/rivet-deploy-kitchen-sink.yml`) builds `examples/kitchen-sink/Dockerfile` from the repo root and ships it to a Rivet Cloud managed pool via `rivet-dev/deploy-action@v1.1.0`. +- The workflow only triggers on changes under `examples/kitchen-sink/**`, the bundled `rivetkit-typescript/packages/**`, `engine/sdks/typescript/**`, and `shared/typescript/**` (everything the Dockerfile actually copies). +- The Dockerfile bakes `ENV PORT=8080` and `ENV KITCHEN_SINK_SERVERLESS_URL=cloud` so the container listens on 8080 and `server.ts` enters serverless mode (serves `/api/rivet/*` via the registry handler instead of calling `registry.start()`). Do NOT use `RIVET_RUN_ENGINE=1` — that flag starts an embedded engine, which conflicts with the engine endpoint that Rivet Cloud injects (`ZodError: cannot specify both startEngine and endpoint`). The `managed-pool-config.environment` field on the deploy-action is not currently honored by the cloud API, so do not rely on it; configure runtime env via `ENV` in the Dockerfile. +- The CI workflow prebuilds `rivetkit-napi.linux-x64-gnu.node` via `docker/build/linux-x64-gnu.Dockerfile` and the runtime Dockerfile copies the whole built `/app` workspace tree (not `pnpm deploy`). `pnpm deploy --legacy` leaves workspace packages as symlinks back to `/app/rivetkit-typescript/...` which then dangle in the runtime stage, so the kitchen-sink image keeps the workspace source + symlinked `node_modules` + chunked `.pnpm` store instead. Image size is large but registry-friendly because the `.pnpm` store is split into 8 chunked `COPY` layers (the registry 503s on a single large layer). +- Token is the `KITCHEN_SINK_RIVET_CLOUD_TOKEN` repo secret (a `cloud_api_*` token scoped to cloud-api.rivet.dev for this kitchen-sink project only). Do not confuse with the engine `pk_*` token used for actor/gateway calls. + ### Cloud Run Deploys -- Deploy the kitchen-sink to Cloud Run from an isolated temp build context that pins the published `rivetkit` preview version, so root workspace `resolutions` do not silently swap in local packages. +There are two Cloud Run services maintained for the kitchen-sink in `dev-projects-491221` / `us-east4`: + +- `kitchen-sink-staging` → engine namespace `kitchen-sink-gv34-staging-52gh` on `api.staging.rivet.dev`. +- `rivet-kitchen-sink` → engine namespace under `kitchen-sink-29a8-cloud-run-*` on `api.rivet.dev`. + +#### Deploying the current workspace (with local rivetkit changes) + +Use [`scripts/deploy-cloud-run.sh`](file:///home/nathan/r8/examples/kitchen-sink/scripts/deploy-cloud-run.sh). It builds [`Dockerfile`](file:///home/nathan/r8/examples/kitchen-sink/Dockerfile) from the monorepo root, tags with `manual-`, pushes to the `cloud-run-source-deploy` Artifact Registry repo, and updates both services (or just one with `--only staging|prod`). It also verifies `/api/rivet/health` after each update. + +```bash +# Build the napi binary once if it's missing. +cd rivetkit-typescript/packages/rivetkit-napi && pnpm build:release && cd - + +# Deploy. +examples/kitchen-sink/scripts/deploy-cloud-run.sh +``` + +Things that must be true for the deploy to actually start serving on Rivet Cloud: + +- Container listens on `$PORT` (default 8080). The [Dockerfile](file:///home/nathan/r8/examples/kitchen-sink/Dockerfile) bakes `ENV PORT=8080`. +- `server.ts` must enter serverless mode (`registry.handler(...)`, not `registry.start()`). The Dockerfile sets `ENV KITCHEN_SINK_SERVERLESS_URL=cloud` to force that. Do NOT use `RIVET_RUN_ENGINE=1` — it also turns on `startEngine`, which collides with the engine endpoint Rivet Cloud injects (`ZodError: cannot specify both startEngine and endpoint`). +- `serverlessPoolConfig()` in [src/index.ts](file:///home/nathan/r8/examples/kitchen-sink/src/index.ts) returns `undefined` whenever `_RIVET_COMPUTE=1` (Rivet Cloud managed compute) **or** `SANDBOX_MODE=serverless` (Cloud Run via Rivet's deploy pipeline) is set. The platform configures the runner pool itself; the in-process `configurePool` would try to hit `GET /datacenters`, which the per-namespace `sk_` token cannot do (`no permission to list datacenters in namespace any with target any`). +- Rivet-injected envs on Cloud Run: `RIVET_PUBLIC_ENDPOINT` (pk_) + `RIVET_ENDPOINT` (sk_) + `SANDBOX_MODE=serverless` + (on the managed-pool path) `RIVET_RUNNER_VERSION` + `_RIVET_COMPUTE=1`. Do not duplicate them in the image. + +#### Deploying a published rivetkit preview build instead + +When validating a published rivetkit preview (instead of the local workspace), build from an isolated temp context so the root `package.json` `resolutions` do not silently swap the published package back to the workspace copy: + - Copy `examples/kitchen-sink` to a temp directory and edit that temp copy instead of building from the monorepo root. - Pin the temp copy to the exact published preview packages you want to test, such as `rivetkit@0.0.0-pr.4667.33279e9` and `@rivetkit/react@0.0.0-pr.4667.33279e9`. - Build and push the image from that temp context, then update the target Cloud Run service to that image. @@ -49,7 +84,7 @@ export GW="https://api.rivet.dev/gateway" curl -s -X PUT "https://api.rivet.dev/actors?namespace=${RIVET_NS}" \ -H "Authorization: Bearer ${RIVET_TOKEN}" \ -H 'Content-Type: application/json' \ - -d '{"name":"","key":"","runner_name_selector":"default","crash_policy":"sleep"}' + -d '{"name":"","key":"","runner_name_selector":"k8s","crash_policy":"sleep"}' ``` This returns `{"actor":{"actor_id":"", ...}, "created": true/false}`. diff --git a/examples/kitchen-sink/Dockerfile b/examples/kitchen-sink/Dockerfile index fe0ebf5dd2..1203ecd5aa 100644 --- a/examples/kitchen-sink/Dockerfile +++ b/examples/kitchen-sink/Dockerfile @@ -1,20 +1,87 @@ -FROM node:22-slim +# Stage 1: install workspace deps + build kitchen-sink. +# The native rivetkit-napi .node is pre-built outside this image and copied in +# via examples/kitchen-sink/rivetkit-napi.linux-x64-gnu.node (see +# .github/workflows/rivet-deploy-kitchen-sink.yml). +FROM node:22-slim AS builder +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* RUN corepack enable && corepack prepare pnpm@10.13.1 --activate WORKDIR /app ENV NODE_OPTIONS=--max-old-space-size=7168 +ENV SKIP_WASM_BUILD=1 +ENV SKIP_NAPI_BUILD=1 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml turbo.json tsconfig.base.json tsup.base.ts ./ +COPY patches/ ./patches/ COPY examples/kitchen-sink/ ./examples/kitchen-sink/ COPY rivetkit-typescript/packages/ ./rivetkit-typescript/packages/ COPY engine/sdks/typescript/ ./engine/sdks/typescript/ COPY shared/typescript/ ./shared/typescript/ +# Move the pre-built napi .node into the workspace package so SKIP_NAPI_BUILD=1 +# at runtime resolves via the local-file branch in +# rivetkit-typescript/packages/rivetkit-napi/index.js. +RUN mv examples/kitchen-sink/rivetkit-napi.linux-x64-gnu.node \ + rivetkit-typescript/packages/rivetkit-napi/rivetkit-napi.linux-x64-gnu.node + RUN pnpm install --frozen-lockfile RUN mkdir -p rivetkit-openapi rivetkit-asyncapi -RUN pnpm build --filter=kitchen-sink -COPY examples/kitchen-sink/public ./examples/kitchen-sink/dist/public +RUN SKIP_WASM_BUILD=1 SKIP_NAPI_BUILD=1 pnpm build --filter=kitchen-sink +RUN mkdir -p ./examples/kitchen-sink/dist/public -WORKDIR /app/examples/kitchen-sink +# Split the pnpm content-addressed store into multiple subdirs so the final +# image layers stay small enough for registry.rivet.dev to accept. A single +# /app/node_modules/.pnpm layer is ~300MB compressed and the registry 503s +# on push around that size. +RUN cd /app/node_modules/.pnpm \ + && mkdir -p /pnpm-chunks/c0 /pnpm-chunks/c1 /pnpm-chunks/c2 /pnpm-chunks/c3 \ + /pnpm-chunks/c4 /pnpm-chunks/c5 /pnpm-chunks/c6 /pnpm-chunks/c7 \ + && i=0 \ + && for d in */; do \ + mv "$d" "/pnpm-chunks/c$((i % 8))/$d"; \ + i=$((i + 1)); \ + done +# Stage 2: minimal runtime image. +# We deliberately copy the whole built workspace (sans heavyweight pnpm cache, +# which we re-layer separately for registry-friendly chunk sizes) instead of +# using `pnpm deploy`. Workspace packages live under +# /app/rivetkit-typescript/packages/* and are wired into /app/node_modules via +# pnpm symlinks; reproducing that layout in /deploy via `pnpm deploy` proved +# more trouble than it was worth. +FROM node:22-slim +RUN corepack enable && corepack prepare pnpm@10.13.1 --activate +WORKDIR /app +ENV NODE_ENV=production + +# Source + workspace packages required by kitchen-sink at runtime. +COPY --from=builder /app/package.json /app/pnpm-workspace.yaml /app/turbo.json /app/tsconfig.base.json /app/tsup.base.ts ./ +COPY --from=builder /app/examples/kitchen-sink/ ./examples/kitchen-sink/ +COPY --from=builder /app/rivetkit-typescript/ ./rivetkit-typescript/ +COPY --from=builder /app/engine/sdks/typescript/ ./engine/sdks/typescript/ +COPY --from=builder /app/shared/typescript/ ./shared/typescript/ + +# node_modules tree with the symlink layout, minus the chunked .pnpm store. +COPY --from=builder /app/node_modules/ ./node_modules/ + +# Replay each pnpm CAS chunk back into .pnpm so all the symlinks resolve. +COPY --from=builder /pnpm-chunks/c0/ ./node_modules/.pnpm/ +COPY --from=builder /pnpm-chunks/c1/ ./node_modules/.pnpm/ +COPY --from=builder /pnpm-chunks/c2/ ./node_modules/.pnpm/ +COPY --from=builder /pnpm-chunks/c3/ ./node_modules/.pnpm/ +COPY --from=builder /pnpm-chunks/c4/ ./node_modules/.pnpm/ +COPY --from=builder /pnpm-chunks/c5/ ./node_modules/.pnpm/ +COPY --from=builder /pnpm-chunks/c6/ ./node_modules/.pnpm/ +COPY --from=builder /pnpm-chunks/c7/ ./node_modules/.pnpm/ + +WORKDIR /app/examples/kitchen-sink +ENV PORT=8080 +# Tell server.ts to enter serverless mode (serve /api/rivet/* via the registry +# handler, do NOT call registry.start()). The Rivet engine is provided by the +# managed pool; we only need this process to act as the runner. +# RIVET_RUN_ENGINE=1 cannot be used here because it would also start an +# embedded engine, conflicting with the cloud-supplied endpoint. +ENV KITCHEN_SINK_SERVERLESS_URL=cloud EXPOSE 8080 -CMD ["pnpm", "start"] +# Run the bundled server directly so node is pid1. Signals reach the rivetkit +# SIGTERM handler without going through pnpm/tsx wrappers. +CMD ["node", "--enable-source-maps", "dist-server/server.mjs"] diff --git a/examples/kitchen-sink/dist-server/server.mjs b/examples/kitchen-sink/dist-server/server.mjs new file mode 100644 index 0000000000..c3c5547ae3 --- /dev/null +++ b/examples/kitchen-sink/dist-server/server.mjs @@ -0,0 +1,13037 @@ +var __defProp = Object.defineProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; + +// src/index.ts +import { setup as setup2 } from "rivetkit"; + +// src/mode.ts +function resolveMode() { + const explicit = process.env.RIVET_KITCHEN_SINK_MODE; + if (explicit === "serverless" || explicit === "serverful" || explicit === "serverless-local") { + return explicit; + } + if (explicit !== void 0 && explicit !== "") { + throw new Error( + `RIVET_KITCHEN_SINK_MODE must be one of "serverless", "serverful", or "serverless-local" (got "${explicit}")` + ); + } + if (process.env.RIVET_RUN_ENGINE === "1") return "serverless-local"; + if (process.env.RIVET_SERVERLESS_URL !== void 0) return "serverless-local"; + if (process.env.KITCHEN_SINK_SERVERLESS_URL !== void 0) { + return "serverless-local"; + } + return "serverless"; +} + +// src/actors/counter/counter.ts +import { actor, event } from "rivetkit"; +var counter = actor({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 5e3 + }, + state: { count: 0 }, + events: { + newCount: event() + }, + onWebSocket(_c, websocket) { + websocket.addEventListener("message", (event21) => { + if (websocket.readyState !== 1) return; + websocket.send(event21.data); + }); + }, + actions: { + increment: (c, x) => { + c.state.count += x; + c.broadcast("newCount", c.state.count); + return c.state.count; + }, + setCount: (c, x) => { + c.state.count = x; + c.broadcast("newCount", x); + return c.state.count; + }, + getCount: (c) => { + return c.state.count; + }, + noop: (_c) => { + return { ok: true }; + }, + goToSleep: (c) => { + c.sleep(); + return { ok: true }; + } + } +}); + +// src/actors/counter/counter-conn.ts +import { actor as actor2, event as event2 } from "rivetkit"; +var counterConn = actor2({ + state: { + connectionCount: 0 + }, + events: { + newCount: event2() + }, + connState: { count: 0 }, + onConnect: (c, conn) => { + c.state.connectionCount += 1; + }, + onDisconnect: (c, conn) => { + c.state.connectionCount -= 1; + }, + actions: { + increment: (c, x) => { + c.conn.state.count += x; + c.broadcast("newCount", c.conn.state.count); + }, + setCount: (c, x) => { + c.conn.state.count = x; + c.broadcast("newCount", x); + }, + getCount: (c) => { + return c.conn.state.count; + }, + getConnectionCount: (c) => { + return c.state.connectionCount; + } + } +}); + +// src/actors/counter/conn-params.ts +import { actor as actor3, event as event3 } from "rivetkit"; +var counterWithParams = actor3({ + state: { count: 0, initializers: [] }, + events: { + newCount: event3() + }, + createConnState: (c, params) => { + return { + name: params.name || "anonymous" + }; + }, + onConnect: (c, conn) => { + c.state.initializers.push(conn.state.name); + }, + actions: { + increment: (c, x) => { + c.state.count += x; + c.broadcast("newCount", { + count: c.state.count, + by: c.conn.state.name + }); + return c.state.count; + }, + getInitializers: (c) => { + return c.state.initializers; + } + } +}); + +// src/actors/counter/lifecycle.ts +import { actor as actor4 } from "rivetkit"; +var counterWithLifecycle = actor4({ + state: { + count: 0, + events: [] + }, + createConnState: (c, params) => ({ + joinTime: Date.now() + }), + onWake: (c) => { + c.state.events.push("onWake"); + }, + onSleep: async (c) => { + c.state.events.push("onSleep:start"); + await new Promise((resolve) => setTimeout(resolve, 1e3)); + c.state.events.push("onSleep:end"); + }, + onBeforeConnect: (c, params) => { + if (params?.trackLifecycle) c.state.events.push("onBeforeConnect"); + }, + onConnect: (c, conn) => { + if (conn.params?.trackLifecycle) c.state.events.push("onConnect"); + }, + onDisconnect: (c, conn) => { + if (conn.params?.trackLifecycle) c.state.events.push("onDisconnect"); + }, + actions: { + getEvents: (c) => { + return c.state.events; + }, + increment: (c, x) => { + c.state.count += x; + return c.state.count; + } + } +}); + +// src/actors/counter/ping-pong-counter.ts +import { actor as actor5 } from "rivetkit"; +var pingPongCounter = actor5({ + state: { + pingCount: 0 + }, + onWebSocket(ctx, websocket) { + websocket.addEventListener("message", (event21) => { + const data = event21.data; + if (typeof data !== "string") return; + let parsed; + try { + parsed = JSON.parse(data); + } catch { + return; + } + if (parsed?.type === "ping") { + ctx.state.pingCount = ctx.state.pingCount + 1; + websocket.send( + JSON.stringify({ + type: "pong", + pingCount: ctx.state.pingCount, + timestamp: Date.now() + }) + ); + } + }); + }, + actions: { + getPingCount(c) { + return c.state.pingCount; + }, + resetPingCount(c) { + c.state.pingCount = 0; + return c.state.pingCount; + } + } +}); + +// src/actors/actions/action-inputs.ts +import { actor as actor6 } from "rivetkit"; +var inputActor = actor6({ + createState: (c, input) => { + return { + initialInput: input, + onCreateInput: void 0 + }; + }, + onCreate: (c, input) => { + c.state.onCreateInput = input; + }, + actions: { + getInputs: (c) => { + return { + initialInput: c.state.initialInput, + onCreateInput: c.state.onCreateInput + }; + } + } +}); + +// src/actors/actions/action-types.ts +import { actor as actor7, UserError } from "rivetkit"; +var syncActionActor = actor7({ + state: { value: 0 }, + actions: { + // Simple synchronous action that returns a value directly + increment: (c, amount = 1) => { + c.state.value += amount; + return c.state.value; + }, + // Synchronous action that returns an object + getInfo: (c) => { + return { + currentValue: c.state.value, + timestamp: Date.now() + }; + }, + // Synchronous action with no return value (void) + reset: (c) => { + c.state.value = 0; + } + } +}); +var asyncActionActor = actor7({ + state: { value: 0, data: null }, + actions: { + // Async action with a delay + delayedIncrement: async (c, amount = 1) => { + await Promise.resolve(); + c.state.value += amount; + return c.state.value; + }, + // Async action that simulates an API call + fetchData: async (c, id) => { + await Promise.resolve(); + const data = { id, timestamp: Date.now() }; + c.state.data = data; + return data; + }, + // Async action with error handling + asyncWithError: async (c, shouldError) => { + await Promise.resolve(); + if (shouldError) { + throw new UserError("Intentional error"); + } + return "Success"; + } + } +}); +var promiseActor = actor7({ + state: { results: [] }, + actions: { + // Action that returns a resolved promise + resolvedPromise: (c) => { + return Promise.resolve("resolved value"); + }, + // Action that returns a promise that resolves after a delay + delayedPromise: (c) => { + return new Promise((resolve) => { + c.state.results.push("delayed"); + resolve("delayed value"); + }); + }, + // Action that returns a rejected promise + rejectedPromise: (c) => { + return Promise.reject(new UserError("promised rejection")); + }, + // Action to check the collected results + getResults: (c) => { + return c.state.results; + } + } +}); + +// src/actors/actions/action-timeout.ts +import { actor as actor8 } from "rivetkit"; +var shortTimeoutActor = actor8({ + state: { value: 0 }, + options: { + actionTimeout: 50 + // 50ms timeout + }, + actions: { + quickAction: async (c) => { + return "quick response"; + }, + slowAction: async (c) => { + await new Promise((resolve) => setTimeout(resolve, 100)); + return "slow response"; + } + } +}); +var longTimeoutActor = actor8({ + state: { value: 0 }, + options: { + actionTimeout: 200 + // 200ms timeout + }, + actions: { + delayedAction: async (c) => { + await new Promise((resolve) => setTimeout(resolve, 100)); + return "delayed response"; + } + } +}); +var defaultTimeoutActor = actor8({ + state: { value: 0 }, + actions: { + normalAction: async (c) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return "normal response"; + } + } +}); +var syncTimeoutActor = actor8({ + state: { value: 0 }, + options: { + actionTimeout: 50 + // 50ms timeout + }, + actions: { + syncAction: (c) => { + return "sync response"; + } + } +}); + +// src/actors/actions/error-handling.ts +import { actor as actor9, UserError as UserError2 } from "rivetkit"; +var errorHandlingActor = actor9({ + state: { + errorLog: [] + }, + actions: { + // Action that throws a UserError with just a message + throwSimpleError: () => { + throw new UserError2("Simple error message"); + }, + // Action that throws a UserError with code and metadata + throwDetailedError: () => { + throw new UserError2("Detailed error message", { + code: "detailed_error", + metadata: { + reason: "test", + timestamp: Date.now() + } + }); + }, + // Action that throws an internal error + throwInternalError: () => { + throw new Error("This is an internal error"); + }, + // Action that returns successfully + successfulAction: () => { + return "success"; + }, + // Action that times out (simulated with a long delay) + timeoutAction: async (c) => { + return new Promise((resolve) => { + setTimeout(() => { + resolve("This should not be reached if timeout works"); + }, 1e4); + }); + }, + // Action with configurable delay to test timeout edge cases + delayedAction: async (c, delayMs) => { + return new Promise((resolve) => { + setTimeout(() => { + resolve(`Completed after ${delayMs}ms`); + }, delayMs); + }); + }, + // Log an error for inspection + logError: (c, error) => { + c.state.errorLog.push(error); + return c.state.errorLog; + }, + // Get the error log + getErrorLog: (c) => { + return c.state.errorLog; + }, + // Clear the error log + clearErrorLog: (c) => { + c.state.errorLog = []; + return true; + } + }, + options: { + actionTimeout: 500 + // 500ms timeout for actions + } +}); +var customTimeoutActor = actor9({ + state: {}, + actions: { + quickAction: async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return "Quick action completed"; + }, + slowAction: async () => { + await new Promise((resolve) => setTimeout(resolve, 300)); + return "Slow action completed"; + } + }, + options: { + actionTimeout: 200 + // 200ms timeout + } +}); + +// src/actors/state/actor-onstatechange.ts +import { actor as actor10 } from "rivetkit"; +var onStateChangeActor = actor10({ + state: { + value: 0 + }, + vars: { + changeCount: 0 + }, + actions: { + // Action that modifies state - should trigger onStateChange + setValue: (c, newValue) => { + c.state.value = newValue; + return c.state.value; + }, + // Action that modifies state multiple times - should trigger onStateChange for each change + incrementMultiple: (c, times) => { + for (let i = 0; i < times; i++) { + c.state.value++; + } + return c.state.value; + }, + // Action that doesn't modify state - should NOT trigger onStateChange + getValue: (c) => { + return c.state.value; + }, + // Action that reads and returns without modifying - should NOT trigger onStateChange + getDoubled: (c) => { + const doubled = c.state.value * 2; + return doubled; + }, + // Get the count of how many times onStateChange was called + getChangeCount: (c) => { + return c.vars.changeCount; + }, + // Reset change counter for testing + resetChangeCount: (c) => { + c.vars.changeCount = 0; + } + }, + // Track onStateChange calls + onStateChange: (c) => { + c.vars.changeCount++; + } +}); + +// src/actors/state/metadata.ts +import { actor as actor11 } from "rivetkit"; +var metadataActor = actor11({ + state: { + lastMetadata: null, + actorName: "", + // Store tags and region in state for testing since they may not be + // available in the context in all environments + storedTags: {}, + storedRegion: null + }, + onWake: (c) => { + c.state.actorName = c.name; + }, + actions: { + // Set up test tags - this will be called by tests to simulate tags + setupTestTags: (c, tags) => { + c.state.storedTags = tags; + return tags; + }, + // Set up test region - this will be called by tests to simulate region + setupTestRegion: (c, region) => { + c.state.storedRegion = region; + return region; + }, + // Get all available metadata + getMetadata: (c) => { + const metadata = { + name: c.name, + tags: c.state.storedTags, + region: c.state.storedRegion + }; + c.state.lastMetadata = metadata; + return metadata; + }, + // Get the actor name + getActorName: (c) => { + return c.name; + }, + // Get a specific tag by key + getTag: (c, key) => { + return c.state.storedTags[key] || null; + }, + // Get all tags + getTags: (c) => { + return c.state.storedTags; + }, + // Get the region + getRegion: (c) => { + return c.state.storedRegion; + }, + // Get the stored actor name (from onWake) + getStoredActorName: (c) => { + return c.state.actorName; + }, + // Get last retrieved metadata + getLastMetadata: (c) => { + return c.state.lastMetadata; + } + } +}); + +// src/actors/state/vars.ts +import { actor as actor12 } from "rivetkit"; +var staticVarActor = actor12({ + state: { value: 0 }, + connState: { hello: "world" }, + vars: { counter: 42, name: "test-actor" }, + actions: { + getVars: (c) => { + return c.vars; + }, + getName: (c) => { + return c.vars.name; + } + } +}); +var nestedVarActor = actor12({ + state: { value: 0 }, + connState: { hello: "world" }, + vars: { + counter: 42, + nested: { + value: "original", + array: [1, 2, 3], + obj: { key: "value" } + } + }, + actions: { + getVars: (c) => { + return c.vars; + }, + modifyNested: (c) => { + c.vars.nested.value = "modified"; + c.vars.nested.array.push(4); + c.vars.nested.obj.key = "new-value"; + return c.vars; + } + } +}); +var dynamicVarActor = actor12({ + state: { value: 0 }, + connState: { hello: "world" }, + createVars: () => { + return { + random: Math.random(), + computed: `Actor-${Math.floor(Math.random() * 1e3)}` + }; + }, + actions: { + getVars: (c) => { + return c.vars; + } + } +}); +var uniqueVarActor = actor12({ + state: { value: 0 }, + connState: { hello: "world" }, + createVars: () => { + return { + id: Math.floor(Math.random() * 1e6) + }; + }, + actions: { + getVars: (c) => { + return c.vars; + } + } +}); +var driverCtxActor = actor12({ + state: { value: 0 }, + connState: { hello: "world" }, + createVars: (c, driverCtx) => { + return { + hasDriverCtx: Boolean(driverCtx?.isTest) + }; + }, + actions: { + getVars: (c) => { + return c.vars; + } + } +}); + +// src/actors/state/kv.ts +import { actor as actor13 } from "rivetkit"; +var kvActor = actor13({ + actions: { + putText: async (c, key, value) => { + await c.kv.put(key, value); + return true; + }, + getText: async (c, key) => { + return await c.kv.get(key); + }, + listText: async (c, prefix) => { + const results = await c.kv.list(prefix, { keyType: "text" }); + return results.map(([key, value]) => ({ + key, + value + })); + }, + roundtripArrayBuffer: async (c, key, values) => { + const buffer = new Uint8Array(values).buffer; + await c.kv.put(key, buffer, { type: "arrayBuffer" }); + const result = await c.kv.get(key, { type: "arrayBuffer" }); + if (!result) { + return null; + } + return Array.from(new Uint8Array(result)); + } + } +}); + +// src/actors/state/large-payloads.ts +import { actor as actor14 } from "rivetkit"; +var largePayloadActor = actor14({ + state: {}, + actions: { + /** + * Accepts a large request payload and returns its size + */ + processLargeRequest: (c, data) => { + return { + itemCount: data.items.length, + firstItem: data.items[0], + lastItem: data.items[data.items.length - 1] + }; + }, + /** + * Returns a large response payload + */ + getLargeResponse: (c, itemCount) => { + const items = []; + for (let i = 0; i < itemCount; i++) { + items.push(`Item ${i} with some additional text to increase size`); + } + return { items }; + }, + /** + * Echo back the request data + */ + echo: (c, data) => { + return data; + } + } +}); +var largePayloadConnActor = actor14({ + state: {}, + connState: { + lastRequestSize: 0 + }, + actions: { + /** + * Accepts a large request payload and returns its size + */ + processLargeRequest: (c, data) => { + c.conn.state.lastRequestSize = data.items.length; + return { + itemCount: data.items.length, + firstItem: data.items[0], + lastItem: data.items[data.items.length - 1] + }; + }, + /** + * Returns a large response payload + */ + getLargeResponse: (c, itemCount) => { + const items = []; + for (let i = 0; i < itemCount; i++) { + items.push(`Item ${i} with some additional text to increase size`); + } + return { items }; + }, + /** + * Echo back the request data + */ + echo: (c, data) => { + return data; + }, + /** + * Get the last request size + */ + getLastRequestSize: (c) => { + return c.conn.state.lastRequestSize; + } + } +}); + +// src/actors/state/sqlite-raw.ts +import { actor as actor15 } from "rivetkit"; +import { db } from "rivetkit/db"; +var sqliteRawActor = actor15({ + db: db({ + onMigrate: async (db16) => { + await db16.execute(` + CREATE TABLE IF NOT EXISTS todos ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + completed INTEGER DEFAULT 0, + created_at INTEGER NOT NULL + ) + `); + } + }), + actions: { + addTodo: async (c, title) => { + const createdAt = Date.now(); + await c.db.execute( + "INSERT INTO todos (title, created_at) VALUES (?, ?)", + title, + createdAt + ); + return { title, createdAt }; + }, + getTodos: async (c) => { + return await c.db.execute("SELECT * FROM todos ORDER BY created_at DESC"); + }, + toggleTodo: async (c, id) => { + await c.db.execute( + "UPDATE todos SET completed = NOT completed WHERE id = ?", + id + ); + const rows = await c.db.execute("SELECT * FROM todos WHERE id = ?", id); + return rows[0]; + }, + deleteTodo: async (c, id) => { + await c.db.execute("DELETE FROM todos WHERE id = ?", id); + return { deleted: id }; + } + } +}); + +// src/actors/state/sqlite-drizzle/mod.ts +import { actor as actor16 } from "rivetkit"; +import { db as db2 } from "rivetkit/db/drizzle"; +import { eq } from "drizzle-orm"; + +// src/actors/state/sqlite-drizzle/schema.ts +var schema_exports = {}; +__export(schema_exports, { + todos: () => todos +}); +import { sqliteTable, text, integer } from "rivetkit/db/drizzle"; +var todos = sqliteTable("todos", { + id: integer("id").primaryKey({ autoIncrement: true }), + title: text("title").notNull(), + completed: integer("completed").default(0), + createdAt: integer("created_at").notNull() +}); + +// src/actors/state/sqlite-drizzle/drizzle/meta/_journal.json +var journal_default = { + version: "7", + dialect: "sqlite", + entries: [ + { + idx: 0, + version: "6", + when: 1770921282251, + tag: "0000_left_wrecking_crew", + breakpoints: true + } + ] +}; + +// src/actors/state/sqlite-drizzle/drizzle/0000_left_wrecking_crew.sql +var left_wrecking_crew_default = "CREATE TABLE `todos` (\n `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,\n `title` text NOT NULL,\n `completed` integer DEFAULT 0,\n `created_at` integer NOT NULL\n);\n"; + +// src/actors/state/sqlite-drizzle/drizzle/migrations.js +var migrations_default = { + journal: journal_default, + migrations: { + m0000: left_wrecking_crew_default + } +}; + +// src/actors/state/sqlite-drizzle/mod.ts +var { todos: todos2 } = schema_exports; +var sqliteDrizzleActor = actor16({ + db: db2({ schema: schema_exports, migrations: migrations_default }), + actions: { + addTodo: async (c, title) => { + const result = await c.db.insert(todos2).values({ + title, + createdAt: Date.now() + }).returning(); + return result[0]; + }, + getTodos: async (c) => { + return await c.db.select().from(todos2).orderBy(todos2.createdAt); + }, + toggleTodo: async (c, id) => { + const existing = await c.db.select().from(todos2).where(eq(todos2.id, id)); + if (!existing[0]) return null; + const newCompleted = existing[0].completed ? 0 : 1; + const result = await c.db.update(todos2).set({ completed: newCompleted }).where(eq(todos2.id, id)).returning(); + return result[0]; + }, + deleteTodo: async (c, id) => { + await c.db.delete(todos2).where(eq(todos2.id, id)); + return { deleted: id }; + } + } +}); + +// src/actors/state/parallelism-test.ts +import { actor as actor17, event as event4 } from "rivetkit"; +import { db as db3 } from "rivetkit/db"; +var parallelismTest = actor17({ + state: { + stateCount: 0 + }, + db: db3({ + onMigrate: async (db16) => { + await db16.execute(` + CREATE TABLE IF NOT EXISTS counter ( + id INTEGER PRIMARY KEY CHECK (id = 1), + count INTEGER NOT NULL DEFAULT 0 + ) + `); + await db16.execute(` + INSERT OR IGNORE INTO counter (id, count) VALUES (1, 0) + `); + } + }), + events: { + stateCountChanged: event4(), + sqliteCountChanged: event4() + }, + actions: { + incrementState: (c) => { + c.state.stateCount += 1; + c.broadcast("stateCountChanged", { count: c.state.stateCount }); + return { count: c.state.stateCount }; + }, + getStateCount: (c) => { + return { count: c.state.stateCount }; + }, + incrementSqlite: async (c) => { + await c.db.execute(`UPDATE counter SET count = count + 1 WHERE id = 1`); + const results = await c.db.execute( + `SELECT count FROM counter WHERE id = 1` + ); + const count = results[0].count; + c.broadcast("sqliteCountChanged", { count }); + return { count }; + }, + getSqliteCount: async (c) => { + const results = await c.db.execute( + `SELECT count FROM counter WHERE id = 1` + ); + return { count: results[0].count }; + } + }, + options: { + sleepTimeout: 3e4 + } +}); + +// src/actors/connections/conn-state.ts +import { actor as actor18, event as event5 } from "rivetkit"; +var connStateActor = actor18({ + state: { + sharedCounter: 0, + disconnectionCount: 0 + }, + events: { + userConnected: event5(), + userDisconnected: event5(), + directMessage: event5() + }, + // Define connection state + createConnState: (c, params) => { + return { + username: params?.username || "anonymous", + role: params?.role || "user", + counter: 0, + createdAt: Date.now(), + noCount: params?.noCount ?? false + }; + }, + // Lifecycle hook when a connection is established + onConnect: (c, conn) => { + c.broadcast("userConnected", { + id: conn.id, + username: "anonymous", + role: "user" + }); + }, + // Lifecycle hook when a connection is closed + onDisconnect: (c, conn) => { + if (!conn.state?.noCount) { + c.state.disconnectionCount += 1; + c.broadcast("userDisconnected", { + id: conn.id + }); + } + }, + actions: { + // Action to increment the connection's counter + incrementConnCounter: (c, amount = 1) => { + c.conn.state.counter += amount; + }, + // Action to increment the shared counter + incrementSharedCounter: (c, amount = 1) => { + c.state.sharedCounter += amount; + return c.state.sharedCounter; + }, + // Get the connection state + getConnectionState: (c) => { + return { id: c.conn.id, ...c.conn.state }; + }, + // Check all active connections + getConnectionIds: (c) => { + return c.conns.entries().filter((c2) => !c2[1].state?.noCount).map((x) => x[0]).toArray(); + }, + // Get disconnection count + getDisconnectionCount: (c) => { + return c.state.disconnectionCount; + }, + // Get all active connection states + getAllConnectionStates: (c) => { + return c.conns.entries().map(([id, conn]) => ({ id, ...conn.state })).toArray(); + }, + // Send message to a specific connection with matching ID + sendToConnection: (c, targetId, message) => { + if (c.conns.has(targetId)) { + c.conns.get(targetId).send("directMessage", { from: c.conn.id, message }); + return true; + } else { + return false; + } + }, + // Update connection state (simulated for tests) + updateConnection: (c, updates) => { + if (updates.username) c.conn.state.username = updates.username; + if (updates.role) c.conn.state.role = updates.role; + return c.conn.state; + }, + disconnectSelf: (c, reason) => { + c.conn.disconnect(reason ?? "test.disconnect"); + return true; + } + } +}); + +// src/actors/connections/reject-connection.ts +import { actor as actor19, UserError as UserError3 } from "rivetkit"; +var rejectConnectionActor = actor19({ + onBeforeConnect: async (_c, params) => { + if (params?.reject) { + await new Promise((resolve) => setTimeout(resolve, 500)); + throw new UserError3("Rejected connection", { + code: "rejected" + }); + } + }, + actions: { + ping: () => "pong" + } +}); + +// src/actors/connections/request-access.ts +import { actor as actor20 } from "rivetkit"; +var requestAccessActor = actor20({ + state: { + // Track request info from different hooks + onBeforeConnectRequest: { + hasRequest: false, + requestUrl: null, + requestMethod: null, + requestHeaders: {} + }, + createConnStateRequest: { + hasRequest: false, + requestUrl: null, + requestMethod: null, + requestHeaders: {} + }, + onRequestRequest: { + hasRequest: false, + requestUrl: null, + requestMethod: null, + requestHeaders: {} + }, + onWebSocketRequest: { + hasRequest: false, + requestUrl: null, + requestMethod: null, + requestHeaders: {} + } + }, + createConnState: (c, params) => { + let requestInfo = null; + if (params?.trackRequest && c.request) { + const headers = {}; + c.request.headers.forEach((value, key) => { + headers[key] = value; + }); + requestInfo = { + hasRequest: true, + requestUrl: c.request.url, + requestMethod: c.request.method, + requestHeaders: headers + }; + } + return { + trackRequest: params?.trackRequest || false, + requestInfo + }; + }, + onConnect: (c, conn) => { + if (conn.state.requestInfo) { + c.state.createConnStateRequest = conn.state.requestInfo; + } + }, + onBeforeConnect: (c, params) => { + if (params?.trackRequest) { + if (c.request) { + c.state.onBeforeConnectRequest.hasRequest = true; + c.state.onBeforeConnectRequest.requestUrl = c.request.url; + c.state.onBeforeConnectRequest.requestMethod = c.request.method; + const headers = {}; + c.request.headers.forEach((value, key) => { + headers[key] = value; + }); + c.state.onBeforeConnectRequest.requestHeaders = headers; + } else { + c.state.onBeforeConnectRequest.hasRequest = false; + } + } + }, + onRequest: (c, request) => { + c.state.onRequestRequest.hasRequest = true; + c.state.onRequestRequest.requestUrl = request.url; + c.state.onRequestRequest.requestMethod = request.method; + const headers = {}; + request.headers.forEach((value, key) => { + headers[key] = value; + }); + c.state.onRequestRequest.requestHeaders = headers; + return new Response( + JSON.stringify({ + hasRequest: true, + requestUrl: request.url, + requestMethod: request.method, + requestHeaders: headers + }), + { + status: 200, + headers: { "Content-Type": "application/json" } + } + ); + }, + onWebSocket: (c, websocket) => { + if (!c.request) throw "Missing request"; + c.state.onWebSocketRequest.hasRequest = true; + c.state.onWebSocketRequest.requestUrl = c.request.url; + c.state.onWebSocketRequest.requestMethod = c.request.method; + const headers = {}; + c.request.headers.forEach((value, key) => { + headers[key] = value; + }); + c.state.onWebSocketRequest.requestHeaders = headers; + websocket.send( + JSON.stringify({ + hasRequest: true, + requestUrl: c.request.url, + requestMethod: c.request.method, + requestHeaders: headers + }) + ); + websocket.addEventListener("message", (event21) => { + websocket.send(event21.data); + }); + }, + actions: { + getRequestInfo: (c) => { + return { + onBeforeConnect: c.state.onBeforeConnectRequest, + createConnState: c.state.createConnStateRequest, + onRequest: c.state.onRequestRequest, + onWebSocket: c.state.onWebSocketRequest + }; + } + } +}); + +// src/actors/http/raw-http.ts +import { Hono } from "hono"; +import { actor as actor21 } from "rivetkit"; +var rawHttpActor = actor21({ + state: { + requestCount: 0 + }, + onRequest(ctx, request) { + const url = new URL(request.url); + const method = request.method; + ctx.state.requestCount++; + if (url.pathname === "/api/hello") { + return new Response( + JSON.stringify({ message: "Hello from actor!" }), + { + headers: { "Content-Type": "application/json" } + } + ); + } + if (url.pathname === "/api/echo" && method === "POST") { + return new Response(request.body, { + headers: request.headers + }); + } + if (url.pathname === "/api/state") { + return new Response( + JSON.stringify({ + requestCount: ctx.state.requestCount + }), + { + headers: { "Content-Type": "application/json" } + } + ); + } + if (url.pathname === "/api/headers") { + const headers = {}; + request.headers.forEach((value, key) => { + headers[key] = value; + }); + return new Response(JSON.stringify(headers), { + headers: { "Content-Type": "application/json" } + }); + } + return new Response("Not Found", { status: 404 }); + }, + actions: {} +}); +var rawHttpNoHandlerActor = actor21({ + actions: {} +}); +var rawHttpVoidReturnActor = actor21({ + onRequest(ctx, request) { + return void 0; + }, + actions: {} +}); +var rawHttpHonoActor = actor21({ + createVars() { + const router = new Hono(); + router.get( + "/", + (c) => c.json({ message: "Welcome to Hono actor!" }) + ); + router.get( + "/users", + (c) => c.json([ + { id: 1, name: "Alice" }, + { id: 2, name: "Bob" } + ]) + ); + router.get("/users/:id", (c) => { + const id = c.req.param("id"); + return c.json({ + id: parseInt(id), + name: id === "1" ? "Alice" : "Bob" + }); + }); + router.post("/users", async (c) => { + const body = await c.req.json(); + return c.json({ id: 3, ...body }, 201); + }); + router.put("/users/:id", async (c) => { + const id = c.req.param("id"); + const body = await c.req.json(); + return c.json({ id: parseInt(id), ...body }); + }); + router.delete("/users/:id", (c) => { + const id = c.req.param("id"); + return c.json({ message: `User ${id} deleted` }); + }); + return { router }; + }, + onRequest(ctx, request) { + return ctx.vars.router.fetch(request); + }, + actions: {} +}); + +// src/actors/http/raw-http-request-properties.ts +import { actor as actor22 } from "rivetkit"; +var rawHttpRequestPropertiesActor = actor22({ + actions: {}, + onRequest(ctx, request) { + const url = new URL(request.url); + const method = request.method; + const headers = {}; + request.headers.forEach((value, key) => { + headers[key] = value; + }); + const handleBody = async () => { + if (!request.body) { + return null; + } + const contentType = request.headers.get("content-type") || ""; + try { + if (contentType.includes("application/json")) { + const text2 = await request.text(); + return text2 ? JSON.parse(text2) : null; + } else { + const text2 = await request.text(); + return text2 || null; + } + } catch (error) { + return null; + } + }; + if (method === "HEAD") { + return new Response(null, { + status: 200 + }); + } + if (method === "OPTIONS") { + return new Response(null, { + status: 204 + }); + } + return handleBody().then((body) => { + const responseData = { + // URL properties + url: request.url, + pathname: url.pathname, + search: url.search, + searchParams: Object.fromEntries(url.searchParams.entries()), + hash: url.hash, + // Method + method: request.method, + // Headers + headers, + // Body + body, + bodyText: typeof body === "string" ? body : body === null && request.body !== null ? "" : null, + // Additional properties that might be available + // Note: Some properties like cache, credentials, mode, etc. + // might not be available in all environments + cache: request.cache || null, + credentials: request.credentials || null, + mode: request.mode || null, + redirect: request.redirect || null, + referrer: request.referrer || null + }; + return new Response(JSON.stringify(responseData), { + headers: { "Content-Type": "application/json" } + }); + }); + } +}); + +// src/actors/http/raw-websocket.ts +import { actor as actor23 } from "rivetkit"; +var rawWebSocketActor = actor23({ + state: { + connectionCount: 0, + messageCount: 0 + }, + onWebSocket(ctx, websocket) { + ctx.state.connectionCount = ctx.state.connectionCount + 1; + console.log( + `[ACTOR] New connection, count: ${ctx.state.connectionCount}` + ); + websocket.send( + JSON.stringify({ + type: "welcome", + connectionCount: ctx.state.connectionCount + }) + ); + console.log("[ACTOR] Sent welcome message"); + websocket.addEventListener("message", (event21) => { + ctx.state.messageCount = ctx.state.messageCount + 1; + console.log( + `[ACTOR] Message received, total count: ${ctx.state.messageCount}, data:`, + event21.data + ); + const data = event21.data; + if (typeof data === "string") { + try { + const parsed = JSON.parse(data); + if (parsed.type === "ping") { + websocket.send( + JSON.stringify({ + type: "pong", + timestamp: Date.now() + }) + ); + } else if (parsed.type === "getStats") { + console.log( + `[ACTOR] Sending stats - connections: ${ctx.state.connectionCount}, messages: ${ctx.state.messageCount}` + ); + websocket.send( + JSON.stringify({ + type: "stats", + connectionCount: ctx.state.connectionCount, + messageCount: ctx.state.messageCount + }) + ); + } else if (parsed.type === "getRequestInfo") { + const url = ctx.request?.url || "ws://actor/websocket"; + const urlObj = new URL(url); + websocket.send( + JSON.stringify({ + type: "requestInfo", + url, + pathname: urlObj.pathname, + search: urlObj.search + }) + ); + } else { + websocket.send(data); + } + } catch { + websocket.send(data); + } + } else { + websocket.send(data); + } + }); + websocket.addEventListener("close", () => { + ctx.state.connectionCount = ctx.state.connectionCount - 1; + console.log( + `[ACTOR] Connection closed, count: ${ctx.state.connectionCount}` + ); + }); + }, + actions: { + getStats(ctx) { + return { + connectionCount: ctx.state.connectionCount, + messageCount: ctx.state.messageCount + }; + } + } +}); +var rawWebSocketBinaryActor = actor23({ + onWebSocket(ctx, websocket) { + websocket.addEventListener("message", (event21) => { + const data = event21.data; + if (data instanceof ArrayBuffer || data instanceof Uint8Array) { + const bytes = new Uint8Array(data); + const reversed = new Uint8Array(bytes.length); + for (let i = 0; i < bytes.length; i++) { + reversed[i] = bytes[bytes.length - 1 - i]; + } + websocket.send(reversed); + } + }); + }, + actions: {} +}); + +// src/actors/http/raw-fetch-counter.ts +import { Hono as Hono2 } from "hono"; +import { actor as actor24 } from "rivetkit"; +var rawFetchCounter = actor24({ + state: { + count: 0 + }, + createVars: () => { + return { router: createCounterRouter() }; + }, + onRequest: (c, request) => { + return c.vars.router.fetch(request, { actor: c }); + }, + actions: { + // ...actions... + } +}); +function createCounterRouter() { + const app2 = new Hono2(); + app2.get("/count", (c) => { + const { actor: actor61 } = c.env; + return c.json({ + count: actor61.state.count + }); + }); + app2.post("/increment", (c) => { + const { actor: actor61 } = c.env; + actor61.state.count++; + return c.json({ + count: actor61.state.count + }); + }); + return app2; +} + +// src/actors/http/raw-websocket-chat-room.ts +import { actor as actor25 } from "rivetkit"; +var rawWebSocketChatRoom = actor25({ + state: { + messages: [] + }, + createVars: () => { + return { + sockets: /* @__PURE__ */ new Set() + }; + }, + onWebSocket(ctx, socket) { + ctx.vars.sockets.add(socket); + socket.send( + JSON.stringify({ + type: "init", + messages: ctx.state.messages + }) + ); + socket.addEventListener("message", (event21) => { + try { + const data = JSON.parse(event21.data); + if (data.type === "message" && data.text) { + const message = { + id: crypto.randomUUID(), + text: data.text, + timestamp: Date.now() + }; + ctx.state.messages.push(message); + ctx.saveState({}); + if (ctx.state.messages.length > 50) { + ctx.state.messages.shift(); + } + const broadcast = JSON.stringify({ + type: "message", + ...message + }); + for (const ws of ctx.vars.sockets) { + if (ws.readyState === 1) { + ws.send(broadcast); + } + } + } + } catch (e) { + console.error("Failed to process message:", e); + } + }); + socket.addEventListener("close", () => { + ctx.vars.sockets.delete(socket); + }); + }, + actions: {} +}); + +// src/actors/http/raw-websocket-serverless-smoke.ts +import { actor as actor26 } from "rivetkit"; +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +var rawWebSocketServerlessSmoke = actor26({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 5e3 + }, + state: { + connectionCount: 0, + sleepCount: 0, + totalTickCount: 0, + totalMessageCount: 0 + }, + async onSleep(c) { + const delayMs = 10 + Math.floor(Math.random() * 1991); + c.state.sleepCount += 1; + c.log.info({ + msg: "raw websocket serverless smoke onSleep delay", + delayMs, + sleepCount: c.state.sleepCount + }); + await sleep(delayMs); + }, + onWebSocket(c, websocket) { + c.state.connectionCount += 1; + const connectionId = crypto.randomUUID(); + let index = 0; + const sendTick = () => { + if (websocket.readyState !== 1) return; + const timestamp = Date.now(); + const message = { + type: "tick", + connectionId, + index, + timestamp, + iso: new Date(timestamp).toISOString(), + totalTickCount: c.state.totalTickCount + }; + c.state.totalTickCount += 1; + index += 1; + websocket.send(JSON.stringify(message)); + }; + c.log.info({ + msg: "raw websocket serverless smoke connected", + connectionId, + connectionCount: c.state.connectionCount + }); + sendTick(); + const interval = setInterval(sendTick, 1e3); + websocket.addEventListener("message", (event21) => { + c.state.totalMessageCount += 1; + c.log.info({ + msg: "raw websocket serverless smoke received message", + connectionId, + totalMessageCount: c.state.totalMessageCount + }); + websocket.send( + JSON.stringify({ + type: "ack", + connectionId, + index, + timestamp: Date.now(), + received: event21.data + }) + ); + }); + websocket.addEventListener("close", () => { + clearInterval(interval); + c.state.connectionCount -= 1; + c.log.info({ + msg: "raw websocket serverless smoke disconnected", + connectionId, + connectionCount: c.state.connectionCount + }); + }); + }, + actions: { + getStats(c) { + return { + connectionCount: c.state.connectionCount, + sleepCount: c.state.sleepCount, + totalTickCount: c.state.totalTickCount, + totalMessageCount: c.state.totalMessageCount + }; + } + } +}); + +// src/actors/http/tunnel-stress.ts +import { actor as actor27 } from "rivetkit"; +var tunnelStress = actor27({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 5e3 + }, + state: { + connectionCount: 0, + messageCount: 0, + heartbeatCount: 0 + }, + onWebSocket(c, websocket) { + c.state.connectionCount += 1; + const connectionId = crypto.randomUUID(); + const sendHeartbeat = () => { + if (websocket.readyState !== 1) return; + c.state.heartbeatCount += 1; + websocket.send( + JSON.stringify({ + type: "heartbeat", + connectionId, + heartbeatCount: c.state.heartbeatCount, + timestamp: Date.now() + }) + ); + }; + const heartbeat = setInterval(sendHeartbeat, 1e3); + sendHeartbeat(); + websocket.addEventListener("message", async (event21) => { + if (typeof event21.data === "string") { + let parsed; + try { + parsed = JSON.parse(event21.data); + } catch { + parsed = void 0; + } + if (parsed && typeof parsed === "object" && parsed.type === "ping") { + const id = parsed.id; + if (websocket.readyState === 1) { + websocket.send( + JSON.stringify({ + type: "pong", + connectionId, + id, + timestamp: Date.now() + }) + ); + } + return; + } + } + c.state.messageCount += 1; + await c.kv.put("counter", String(c.state.messageCount)); + websocket.send( + JSON.stringify({ + type: "reply", + connectionId, + messageCount: c.state.messageCount, + timestamp: Date.now(), + received: event21.data + }) + ); + }); + websocket.addEventListener("close", () => { + clearInterval(heartbeat); + c.state.connectionCount -= 1; + }); + }, + actions: { + getStats(c) { + return { + connectionCount: c.state.connectionCount, + messageCount: c.state.messageCount, + heartbeatCount: c.state.heartbeatCount + }; + } + } +}); + +// src/actors/lifecycle/run.ts +import { actor as actor28, queue } from "rivetkit"; +var RUN_SLEEP_TIMEOUT = 500; +var runWithTicks = actor28({ + state: { + tickCount: 0, + lastTickAt: 0, + runStarted: false, + runExited: false + }, + run: async (c) => { + c.state.runStarted = true; + c.log.info("run handler started"); + while (!c.aborted) { + c.state.tickCount += 1; + c.state.lastTickAt = Date.now(); + c.log.info({ msg: "tick", tickCount: c.state.tickCount }); + await new Promise((resolve) => { + const timeout = setTimeout(resolve, 50); + c.abortSignal.addEventListener( + "abort", + () => { + clearTimeout(timeout); + resolve(); + }, + { once: true } + ); + }); + } + c.state.runExited = true; + c.log.info("run handler exiting gracefully"); + }, + actions: { + getState: (c) => ({ + tickCount: c.state.tickCount, + lastTickAt: c.state.lastTickAt, + runStarted: c.state.runStarted, + runExited: c.state.runExited + }) + }, + options: { + sleepTimeout: RUN_SLEEP_TIMEOUT + } +}); +var runWithQueueConsumer = actor28({ + state: { + messagesReceived: [], + runStarted: false + }, + queues: { + messages: queue() + }, + run: async (c) => { + c.state.runStarted = true; + c.log.info("run handler started, waiting for messages"); + for await (const message of c.queue.iter()) { + c.log.info({ msg: "received message", body: message.body }); + c.state.messagesReceived.push({ + name: message.name, + body: message.body + }); + } + c.log.info("run handler exiting gracefully"); + }, + actions: { + getState: (c) => ({ + messagesReceived: c.state.messagesReceived, + runStarted: c.state.runStarted + }), + sendMessage: async (c, body) => { + const client = c.client(); + const handle = client.runWithQueueConsumer.getForId(c.actorId); + await handle.send("messages", body); + return true; + } + }, + options: { + sleepTimeout: RUN_SLEEP_TIMEOUT + } +}); +var runWithEarlyExit = actor28({ + state: { + runStarted: false, + destroyCalled: false + }, + run: async (c) => { + c.state.runStarted = true; + c.log.info("run handler started, will exit after delay"); + await new Promise((resolve) => setTimeout(resolve, 200)); + c.log.info("run handler exiting early"); + }, + onDestroy: (c) => { + c.state.destroyCalled = true; + }, + actions: { + getState: (c) => ({ + runStarted: c.state.runStarted, + destroyCalled: c.state.destroyCalled + }) + }, + options: { + sleepTimeout: RUN_SLEEP_TIMEOUT + } +}); +var runWithError = actor28({ + state: { + runStarted: false, + destroyCalled: false + }, + run: async (c) => { + c.state.runStarted = true; + c.log.info("run handler started, will throw error"); + await new Promise((resolve) => setTimeout(resolve, 50)); + throw new Error("intentional error in run handler"); + }, + onDestroy: (c) => { + c.state.destroyCalled = true; + }, + actions: { + getState: (c) => ({ + runStarted: c.state.runStarted, + destroyCalled: c.state.destroyCalled + }) + }, + options: { + sleepTimeout: RUN_SLEEP_TIMEOUT + } +}); +var runWithoutHandler = actor28({ + state: { + wakeCount: 0 + }, + onWake: (c) => { + c.state.wakeCount += 1; + }, + actions: { + getState: (c) => ({ + wakeCount: c.state.wakeCount + }) + }, + options: { + sleepTimeout: RUN_SLEEP_TIMEOUT + } +}); + +// src/actors/lifecycle/sleep.ts +import { actor as actor29, event as event7 } from "rivetkit"; +import { promiseWithResolvers } from "rivetkit/utils"; +var SLEEP_TIMEOUT = 1e3; +var sleep2 = actor29({ + state: { startCount: 0, sleepCount: 0 }, + onWake: (c) => { + c.state.startCount += 1; + }, + onSleep: (c) => { + c.state.sleepCount += 1; + }, + actions: { + triggerSleep: (c) => { + c.sleep(); + }, + getCounts: (c) => { + return { + startCount: c.state.startCount, + sleepCount: c.state.sleepCount + }; + }, + setAlarm: async (c, duration) => { + await c.schedule.after(duration, "onAlarm"); + }, + onAlarm: (c) => { + c.log.info("alarm called"); + } + }, + options: { + sleepTimeout: SLEEP_TIMEOUT + } +}); +var sleepWithLongRpc = actor29({ + state: { startCount: 0, sleepCount: 0 }, + createVars: () => ({}), + events: { + waiting: event7() + }, + onWake: (c) => { + c.state.startCount += 1; + }, + onSleep: (c) => { + c.state.sleepCount += 1; + }, + actions: { + getCounts: (c) => { + return { + startCount: c.state.startCount, + sleepCount: c.state.sleepCount + }; + }, + longRunningRpc: async (c) => { + c.log.info("starting long running rpc"); + c.vars.longRunningResolve = promiseWithResolvers(() => { + }); + c.broadcast("waiting"); + await c.vars.longRunningResolve.promise; + c.log.info("finished long running rpc"); + }, + finishLongRunningRpc: (c) => c.vars.longRunningResolve?.resolve() + }, + options: { + sleepTimeout: SLEEP_TIMEOUT + } +}); +var sleepWithRawHttp = actor29({ + state: { startCount: 0, sleepCount: 0, requestCount: 0 }, + onWake: (c) => { + c.state.startCount += 1; + }, + onSleep: (c) => { + c.state.sleepCount += 1; + }, + onRequest: async (c, request) => { + c.state.requestCount += 1; + const url = new URL(request.url); + if (url.pathname === "/long-request") { + const duration = parseInt( + url.searchParams.get("duration") || "1000" + ); + c.log.info({ msg: "starting long fetch request", duration }); + await new Promise((resolve) => setTimeout(resolve, duration)); + c.log.info("finished long fetch request"); + return new Response(JSON.stringify({ completed: true }), { + headers: { "Content-Type": "application/json" } + }); + } + return new Response("Not Found", { status: 404 }); + }, + actions: { + getCounts: (c) => { + return { + startCount: c.state.startCount, + sleepCount: c.state.sleepCount, + requestCount: c.state.requestCount + }; + } + }, + options: { + sleepTimeout: SLEEP_TIMEOUT + } +}); +var sleepWithRawWebSocket = actor29({ + state: { startCount: 0, sleepCount: 0, connectionCount: 0 }, + onWake: (c) => { + c.state.startCount += 1; + }, + onSleep: (c) => { + c.state.sleepCount += 1; + }, + onWebSocket: (c, websocket) => { + c.state.connectionCount += 1; + c.log.info({ + msg: "websocket connected", + connectionCount: c.state.connectionCount + }); + websocket.send( + JSON.stringify({ + type: "connected", + connectionCount: c.state.connectionCount + }) + ); + websocket.addEventListener("message", (event21) => { + const data = event21.data; + if (typeof data === "string") { + try { + const parsed = JSON.parse(data); + if (parsed.type === "getCounts") { + websocket.send( + JSON.stringify({ + type: "counts", + startCount: c.state.startCount, + sleepCount: c.state.sleepCount, + connectionCount: c.state.connectionCount + }) + ); + } else if (parsed.type === "keepAlive") { + websocket.send(JSON.stringify({ type: "ack" })); + } + } catch { + websocket.send(data); + } + } + }); + websocket.addEventListener("close", () => { + c.state.connectionCount -= 1; + c.log.info({ + msg: "websocket disconnected", + connectionCount: c.state.connectionCount + }); + }); + }, + actions: { + getCounts: (c) => { + return { + startCount: c.state.startCount, + sleepCount: c.state.sleepCount, + connectionCount: c.state.connectionCount + }; + } + }, + options: { + sleepTimeout: SLEEP_TIMEOUT + } +}); +var sleepWithNoSleepOption = actor29({ + state: { startCount: 0, sleepCount: 0 }, + onWake: (c) => { + c.state.startCount += 1; + }, + onSleep: (c) => { + c.state.sleepCount += 1; + }, + actions: { + getCounts: (c) => { + return { + startCount: c.state.startCount, + sleepCount: c.state.sleepCount + }; + } + }, + options: { + sleepTimeout: SLEEP_TIMEOUT, + noSleep: true + } +}); + +// src/actors/lifecycle/scheduled.ts +import { actor as actor30, event as event8 } from "rivetkit"; +var scheduled = actor30({ + state: { + lastRun: 0, + scheduledCount: 0, + taskHistory: [] + }, + events: { + scheduled: event8(), + scheduledWithId: event8() + }, + actions: { + // Schedule using 'at' with specific timestamp + scheduleTaskAt: (c, timestamp) => { + c.schedule.at(timestamp, "onScheduledTask"); + return timestamp; + }, + // Schedule using 'after' with delay + scheduleTaskAfter: (c, delayMs) => { + c.schedule.after(delayMs, "onScheduledTask"); + return Date.now() + delayMs; + }, + // Schedule with a task ID for ordering tests + scheduleTaskAfterWithId: (c, taskId, delayMs) => { + c.schedule.after(delayMs, "onScheduledTaskWithId", taskId); + return { taskId, scheduledFor: Date.now() + delayMs }; + }, + // Original method for backward compatibility + scheduleTask: (c, delayMs) => { + const timestamp = Date.now() + delayMs; + c.schedule.at(timestamp, "onScheduledTask"); + return timestamp; + }, + // Getters for state + getLastRun: (c) => { + return c.state.lastRun; + }, + getScheduledCount: (c) => { + return c.state.scheduledCount; + }, + getTaskHistory: (c) => { + return c.state.taskHistory; + }, + clearHistory: (c) => { + c.state.taskHistory = []; + c.state.scheduledCount = 0; + c.state.lastRun = 0; + return true; + }, + // Scheduled task handlers + onScheduledTask: (c) => { + c.state.lastRun = Date.now(); + c.state.scheduledCount++; + c.broadcast("scheduled", { + time: c.state.lastRun, + count: c.state.scheduledCount + }); + }, + onScheduledTaskWithId: (c, taskId) => { + c.state.lastRun = Date.now(); + c.state.scheduledCount++; + c.state.taskHistory.push(taskId); + c.broadcast("scheduledWithId", { + taskId, + time: c.state.lastRun, + count: c.state.scheduledCount + }); + } + } +}); + +// src/actors/lifecycle/destroy.ts +import { actor as actor31 } from "rivetkit"; +var destroyObserver = actor31({ + state: { destroyedActors: [] }, + actions: { + notifyDestroyed: (c, actorKey) => { + c.state.destroyedActors.push(actorKey); + }, + wasDestroyed: (c, actorKey) => { + return c.state.destroyedActors.includes(actorKey); + }, + reset: (c) => { + c.state.destroyedActors = []; + } + } +}); +var destroyActor = actor31({ + state: { value: 0, key: "" }, + onWake: (c) => { + c.state.key = c.key.join("/"); + }, + onDestroy: async (c) => { + const client = c.client(); + const observer = client.destroyObserver.getOrCreate(["observer"]); + await observer.notifyDestroyed(c.state.key); + }, + actions: { + setValue: async (c, newValue) => { + c.state.value = newValue; + await c.saveState({ immediate: true }); + return c.state.value; + }, + getValue: (c) => { + return c.state.value; + }, + destroy: (c) => { + c.destroy(); + } + } +}); + +// src/actors/lifecycle/hibernation.ts +import { actor as actor32 } from "rivetkit"; +var HIBERNATION_SLEEP_TIMEOUT = 500; +var hibernationActor = actor32({ + state: { + sleepCount: 0, + wakeCount: 0 + }, + createConnState: (c) => { + return { + count: 0, + connectCount: 0, + disconnectCount: 0 + }; + }, + onWake: (c) => { + c.state.wakeCount += 1; + }, + onSleep: (c) => { + c.state.sleepCount += 1; + }, + onConnect: (c, conn) => { + conn.state.connectCount += 1; + }, + onDisconnect: (c, conn) => { + conn.state.disconnectCount += 1; + }, + actions: { + // Basic RPC that returns a simple value + ping: (c) => { + return "pong"; + }, + // Increment the connection's count + connIncrement: (c) => { + c.conn.state.count += 1; + return c.conn.state.count; + }, + // Get the connection's count + getConnCount: (c) => { + return c.conn.state.count; + }, + // Get the connection's lifecycle counts + getConnLifecycleCounts: (c) => { + return { + connectCount: c.conn.state.connectCount, + disconnectCount: c.conn.state.disconnectCount + }; + }, + // Get all connection IDs + getConnectionIds: (c) => { + return c.conns.entries().map((x) => x[0]).toArray(); + }, + // Get actor sleep/wake counts + getActorCounts: (c) => { + return { + sleepCount: c.state.sleepCount, + wakeCount: c.state.wakeCount + }; + }, + // Trigger sleep + triggerSleep: (c) => { + c.sleep(); + } + }, + options: { + sleepTimeout: HIBERNATION_SLEEP_TIMEOUT + } +}); + +// src/actors/queue/worker.ts +import { actor as actor33, event as event9, queue as queue2 } from "rivetkit"; +var worker = actor33({ + state: { + status: "idle", + processed: 0, + lastJob: null + }, + events: { + statusChanged: event9(), + jobProcessed: event9() + }, + queues: { + jobs: queue2() + }, + async run(c) { + c.state.status = "running"; + c.broadcast("statusChanged", { + status: c.state.status, + processed: c.state.processed + }); + for await (const job of c.queue.iter()) { + c.state.processed += 1; + c.state.lastJob = job.body; + c.broadcast("jobProcessed", { + processed: c.state.processed, + job: job.body + }); + } + c.state.status = "idle"; + }, + actions: { + getState(c) { + return { + status: c.state.status, + processed: c.state.processed, + lastJob: c.state.lastJob + }; + } + } +}); + +// src/actors/queue/worker-timeout.ts +import { actor as actor34, event as event10, queue as queue3 } from "rivetkit"; +var DEFAULT_TIMEOUT_MS = 2e3; +var workerTimeout = actor34({ + state: { + status: "idle", + processed: 0, + ticks: 0, + lastTickAt: null, + lastJob: null, + timeoutMs: DEFAULT_TIMEOUT_MS + }, + events: { + tick: event10(), + jobProcessed: event10() + }, + queues: { + jobs: queue3() + }, + run: async (c) => { + c.state.status = "running"; + while (!c.aborted) { + const message = await c.queue.next({ + names: ["jobs"], + timeout: c.state.timeoutMs + }); + if (!message) { + const at = Date.now(); + c.state.ticks += 1; + c.state.lastTickAt = at; + c.broadcast("tick", { + ticks: c.state.ticks, + at + }); + continue; + } + c.state.processed += 1; + c.state.lastJob = message.body; + c.broadcast("jobProcessed", { + processed: c.state.processed, + job: message.body + }); + } + c.state.status = "idle"; + }, + actions: { + enqueueJob: async (c, payload2) => { + const job = { + id: crypto.randomUUID(), + payload: payload2 + }; + await c.queue.send("jobs", job); + return job; + }, + setTimeoutMs: (c, timeoutMs) => { + c.state.timeoutMs = Math.max(100, Math.floor(timeoutMs)); + return c.state.timeoutMs; + }, + getState: (c) => c.state + } +}); + +// src/actors/workflow/workflow-fixtures.ts +import { actor as actor35, event as event11, queue as queue4 } from "rivetkit"; +import { Loop, workflow } from "rivetkit/workflow"; +var WORKFLOW_GUARD_KV_KEY = "__rivet_actor_workflow_guard_triggered"; +var WORKFLOW_QUEUE_NAME = "workflow-default"; +var WORKFLOW_TIMEOUT_QUEUE_NAME = "workflow-timeout"; +var workflowCounterActor = actor35({ + state: { + runCount: 0, + guardTriggered: false, + history: [] + }, + run: workflow(async (ctx) => { + await ctx.loop("counter", async (loopCtx) => { + try { + loopCtx.state; + } catch { + } + await loopCtx.step("increment", async () => { + incrementWorkflowCounter(loopCtx); + }); + await loopCtx.sleep("idle", 25); + return Loop.continue(void 0); + }); + }), + actions: { + getState: async (c) => { + const guardFlag = await c.kv.get(WORKFLOW_GUARD_KV_KEY); + if (guardFlag === "true") { + c.state.guardTriggered = true; + } + return c.state; + } + }, + options: { + sleepTimeout: 50 + } +}); +var workflowQueueActor = actor35({ + state: { + received: [] + }, + queues: { + [WORKFLOW_QUEUE_NAME]: queue4() + }, + run: workflow(async (ctx) => { + await ctx.loop("queue", async (loopCtx) => { + const message = await loopCtx.queue.next("queue-wait", { + names: [WORKFLOW_QUEUE_NAME], + completable: true + }); + if (!message.complete) { + return Loop.continue(void 0); + } + const complete = message.complete; + await loopCtx.step("store-message", async () => { + await storeWorkflowQueueMessage(loopCtx, message.body, complete); + }); + return Loop.continue(void 0); + }); + }), + actions: { + getMessages: (c) => c.state.received + } +}); +var workflowSleepActor = actor35({ + state: { + ticks: 0 + }, + run: workflow(async (ctx) => { + await ctx.loop("sleep", async (loopCtx) => { + await loopCtx.step("tick", async () => { + incrementWorkflowSleepTick(loopCtx); + }); + await loopCtx.sleep("delay", 40); + return Loop.continue(void 0); + }); + }), + actions: { + getState: (c) => c.state + }, + options: { + sleepTimeout: 50 + } +}); +var workflowQueueTimeoutActor = actor35({ + state: { + processed: 0, + ticks: 0, + lastTickAt: null, + lastJob: null, + timeoutMs: 2e3 + }, + events: { + tick: event11(), + jobProcessed: event11() + }, + queues: { + [WORKFLOW_TIMEOUT_QUEUE_NAME]: queue4() + }, + run: workflow(async (ctx) => { + await ctx.loop("queue-timeout-loop", async (loopCtx) => { + const timeoutMs = await loopCtx.step("read-timeout", async () => { + return readWorkflowTimeoutMs(loopCtx); + }); + const [message] = await loopCtx.queue.nextBatch("wait-job-or-timeout", { + names: [WORKFLOW_TIMEOUT_QUEUE_NAME], + timeout: timeoutMs + }); + if (!message) { + await loopCtx.step("tick", async () => { + recordWorkflowTimeoutTick(loopCtx); + }); + return Loop.continue(void 0); + } + await loopCtx.step("process-job", async () => { + processWorkflowTimeoutJob(loopCtx, message.body); + }); + return Loop.continue(void 0); + }); + }), + actions: { + enqueueJob: async (c, payload2) => { + const job = { id: crypto.randomUUID(), payload: payload2 }; + await c.queue.send(WORKFLOW_TIMEOUT_QUEUE_NAME, job); + return job; + }, + setTimeoutMs: (c, timeoutMs) => { + c.state.timeoutMs = Math.max(100, Math.floor(timeoutMs)); + return c.state.timeoutMs; + }, + getState: (c) => c.state + } +}); +function incrementWorkflowCounter(ctx) { + ctx.state.runCount += 1; + ctx.state.history.push(ctx.state.runCount); +} +async function storeWorkflowQueueMessage(ctx, body, complete) { + ctx.state.received.push(body); + await complete({ echo: body }); +} +function incrementWorkflowSleepTick(ctx) { + ctx.state.ticks += 1; +} +function readWorkflowTimeoutMs(ctx) { + return ctx.state.timeoutMs; +} +function recordWorkflowTimeoutTick(ctx) { + const at = Date.now(); + ctx.state.ticks += 1; + ctx.state.lastTickAt = at; + ctx.broadcast("tick", { + ticks: ctx.state.ticks, + at + }); +} +function processWorkflowTimeoutJob(ctx, job) { + ctx.state.processed += 1; + ctx.state.lastJob = job; + ctx.broadcast("jobProcessed", { + processed: ctx.state.processed, + job + }); +} + +// src/actors/workflow/timer.ts +import { actor as actor36, event as event12 } from "rivetkit"; +import { Loop as Loop2, workflow as workflow2 } from "rivetkit/workflow"; + +// src/actors/workflow/_helpers.ts +function actorCtx(ctx) { + return ctx; +} + +// src/actors/workflow/timer.ts +var timer = actor36({ + createState: (c, input) => ({ + id: c.key[0], + name: input?.name ?? "Timer", + durationMs: input?.durationMs ?? 1e4, + startedAt: Date.now() + }), + events: { + timerStarted: event12(), + timerCompleted: event12() + }, + actions: { + getTimer: (c) => c.state + }, + run: workflow2(async (ctx) => { + await ctx.loop("timer-loop", async (loopCtx) => { + const c = actorCtx(loopCtx); + const durationMs = await loopCtx.step("start-timer", async () => { + ctx.log.info({ + msg: "starting timer", + timerId: c.state.id, + durationMs: c.state.durationMs + }); + c.broadcast("timerStarted", c.state); + return c.state.durationMs; + }); + await loopCtx.sleep("countdown", durationMs); + await loopCtx.step("complete-timer", async () => { + c.state.completedAt = Date.now(); + c.broadcast("timerCompleted", c.state); + ctx.log.info({ msg: "timer completed", timerId: c.state.id }); + }); + return Loop2.break(void 0); + }); + }), + options: { + sleepTimeout: 1e3 + } +}); + +// src/actors/workflow/order.ts +import { actor as actor37, event as event13 } from "rivetkit"; +import { Loop as Loop3, workflow as workflow3 } from "rivetkit/workflow"; +async function simulateWork(name, failChance = 0.1) { + await new Promise( + (resolve) => setTimeout(resolve, 500 + Math.random() * 1e3) + ); + if (Math.random() < failChance) { + throw new Error(`${name} failed (simulated)`); + } +} +var order = actor37({ + createState: (c) => ({ + id: c.key[0], + status: "pending", + step: 0, + createdAt: Date.now() + }), + events: { + orderUpdated: event13() + }, + actions: { + getOrder: (c) => c.state + }, + run: workflow3(async (ctx) => { + await ctx.loop("process-order", async (loopCtx) => { + const c = actorCtx(loopCtx); + await loopCtx.step("validate", async () => { + ctx.log.info({ msg: "processing order", orderId: c.state.id }); + c.state.status = "validating"; + c.state.step = 1; + c.broadcast("orderUpdated", c.state); + await simulateWork("validation", 0.05); + }); + await loopCtx.step("charge", async () => { + c.state.status = "charging"; + c.state.step = 2; + c.broadcast("orderUpdated", c.state); + await simulateWork("payment", 0.1); + }); + await loopCtx.step("fulfill", async () => { + c.state.status = "fulfilling"; + c.state.step = 3; + c.broadcast("orderUpdated", c.state); + await simulateWork("fulfillment", 0.05); + }); + await loopCtx.step("complete", async () => { + c.state.status = "completed"; + c.state.step = 4; + c.state.completedAt = Date.now(); + c.broadcast("orderUpdated", c.state); + ctx.log.info({ msg: "order completed", orderId: c.state.id }); + }); + return Loop3.break(void 0); + }); + }) +}); + +// src/actors/workflow/batch.ts +import { actor as actor38, event as event14 } from "rivetkit"; +import { Loop as Loop4, workflow as workflow4 } from "rivetkit/workflow"; +function fetchBatch(cursor, batchSize, totalItems) { + const start = cursor * batchSize; + const end = Math.min(start + batchSize, totalItems); + const items = []; + for (let i = start; i < end; i++) { + items.push(i); + } + return { + items, + hasMore: end < totalItems + }; +} +var batch = actor38({ + createState: (c, input) => ({ + id: c.key[0], + totalItems: input?.totalItems ?? 50, + batchSize: input?.batchSize ?? 5, + status: "running", + processedTotal: 0, + currentBatch: 0, + batches: [], + startedAt: Date.now() + }), + events: { + batchProcessed: event14(), + stateChanged: event14(), + processingComplete: event14() + }, + actions: { + getJob: (c) => c.state + }, + run: workflow4(async (ctx) => { + await ctx.loop({ + name: "batch-loop", + state: { cursor: 0 }, + run: async (batchCtx, loopState) => { + const c = actorCtx(batchCtx); + const batch2 = await batchCtx.step("fetch-batch", async () => { + ctx.log.info({ + msg: "processing batch", + jobId: c.state.id, + cursor: loopState.cursor + }); + await new Promise((r) => setTimeout(r, 200 + Math.random() * 300)); + return fetchBatch(loopState.cursor, c.state.batchSize, c.state.totalItems); + }); + await batchCtx.step("process-batch", async () => { + await new Promise((r) => setTimeout(r, 300 + Math.random() * 500)); + const batchInfo = { + id: loopState.cursor, + count: batch2.items.length, + processedAt: Date.now() + }; + c.state.currentBatch = loopState.cursor; + c.state.processedTotal += batch2.items.length; + c.state.batches.push(batchInfo); + c.broadcast("batchProcessed", batchInfo); + c.broadcast("stateChanged", c.state); + ctx.log.info({ + msg: "batch processed", + jobId: c.state.id, + cursor: loopState.cursor, + count: batch2.items.length + }); + }); + if (!batch2.hasMore) { + await batchCtx.step("mark-complete", async () => { + c.state.status = "completed"; + c.state.completedAt = Date.now(); + c.broadcast("stateChanged", c.state); + c.broadcast("processingComplete", { + totalBatches: loopState.cursor + 1, + totalItems: c.state.processedTotal + }); + }); + return Loop4.break(loopState.cursor + 1); + } + return Loop4.continue({ cursor: loopState.cursor + 1 }); + } + }); + }) +}); + +// src/actors/workflow/approval.ts +import { actor as actor39, event as event15, queue as queue5 } from "rivetkit"; +import { Loop as Loop5, workflow as workflow5 } from "rivetkit/workflow"; +var QUEUE_DECISION = "decision"; +var APPROVAL_TIMEOUT_MS = 3e4; +var approval = actor39({ + createState: (c, input) => ({ + id: c.key[0], + title: input?.title ?? "Untitled Request", + description: input?.description ?? "", + status: "pending", + createdAt: Date.now() + }), + queues: { + decision: queue5() + }, + events: { + requestUpdated: event15(), + requestCreated: event15() + }, + actions: { + getRequest: (c) => c.state, + approve: async (c, approver) => { + if (c.state.status !== "pending") return; + c.state.deciding = true; + c.broadcast("requestUpdated", c.state); + await c.queue.send(QUEUE_DECISION, { approved: true, approver }); + }, + reject: async (c, approver) => { + if (c.state.status !== "pending") return; + c.state.deciding = true; + c.broadcast("requestUpdated", c.state); + await c.queue.send(QUEUE_DECISION, { approved: false, approver }); + } + }, + run: workflow5(async (ctx) => { + await ctx.loop("approval-loop", async (loopCtx) => { + const c = actorCtx(loopCtx); + await loopCtx.step("init-request", async () => { + ctx.log.info({ + msg: "waiting for approval decision", + requestId: c.state.id, + title: c.state.title + }); + c.broadcast("requestCreated", c.state); + }); + const [decisionMessage] = await loopCtx.queue.nextBatch( + "wait-decision", + { + names: [QUEUE_DECISION], + timeout: APPROVAL_TIMEOUT_MS + } + ); + const decision = decisionMessage?.body ?? null; + await loopCtx.step("update-status", async () => { + c.state.deciding = false; + if (decision === null) { + c.state.status = "timeout"; + ctx.log.info({ msg: "request timed out", requestId: c.state.id }); + } else if (decision.approved) { + c.state.status = "approved"; + c.state.decidedBy = decision.approver; + ctx.log.info({ + msg: "request approved", + requestId: c.state.id, + approver: decision.approver + }); + } else { + c.state.status = "rejected"; + c.state.decidedBy = decision.approver; + ctx.log.info({ + msg: "request rejected", + requestId: c.state.id, + approver: decision.approver + }); + } + c.state.decidedAt = Date.now(); + c.broadcast("requestUpdated", c.state); + }); + return Loop5.break(void 0); + }); + }) +}); + +// src/actors/workflow/dashboard.ts +import { actor as actor40, event as event16, queue as queue6 } from "rivetkit"; +import { Loop as Loop6, workflow as workflow6 } from "rivetkit/workflow"; +var QUEUE_REFRESH = "refresh"; +async function fetchUserStats() { + await new Promise((r) => setTimeout(r, 800 + Math.random() * 1200)); + return { + count: Math.floor(1e3 + Math.random() * 500), + activeToday: Math.floor(100 + Math.random() * 200), + newThisWeek: Math.floor(20 + Math.random() * 80) + }; +} +async function fetchOrderStats() { + await new Promise((r) => setTimeout(r, 600 + Math.random() * 1e3)); + const count = Math.floor(50 + Math.random() * 150); + const revenue = Math.floor(5e3 + Math.random() * 15e3); + return { + count, + revenue, + avgOrderValue: Math.round(revenue / count) + }; +} +async function fetchMetricsStats() { + await new Promise((r) => setTimeout(r, 400 + Math.random() * 800)); + return { + pageViews: Math.floor(1e4 + Math.random() * 5e4), + sessions: Math.floor(2e3 + Math.random() * 8e3), + bounceRate: Math.round(30 + Math.random() * 40) + }; +} +var dashboard = actor40({ + state: { + data: null, + loading: false, + branches: { + users: "pending", + orders: "pending", + metrics: "pending" + }, + lastRefresh: null + }, + queues: { + [QUEUE_REFRESH]: queue6() + }, + events: { + stateChanged: event16(), + refreshComplete: event16() + }, + actions: { + refresh: async (c) => { + if (!c.state.loading) { + c.state.loading = true; + c.state.branches = { + users: "pending", + orders: "pending", + metrics: "pending" + }; + c.broadcast("stateChanged", c.state); + await c.queue.send(QUEUE_REFRESH, {}); + } + }, + getState: (c) => c.state + }, + run: workflow6(async (ctx) => { + await ctx.loop("refresh-loop", async (loopCtx) => { + const c = actorCtx(loopCtx); + await loopCtx.queue.next("wait-refresh", { + names: [QUEUE_REFRESH] + }); + ctx.log.info({ msg: "starting dashboard refresh" }); + const results = await loopCtx.join("fetch-all", { + users: { + run: async (branchCtx) => { + const bc = actorCtx(branchCtx); + await branchCtx.step("mark-running", async () => { + bc.state.branches.users = "running"; + bc.broadcast("stateChanged", bc.state); + }); + const data = await branchCtx.step("fetch-users", async () => { + return await fetchUserStats(); + }); + await branchCtx.step("mark-complete", async () => { + bc.state.branches.users = "completed"; + bc.broadcast("stateChanged", bc.state); + }); + return data; + } + }, + orders: { + run: async (branchCtx) => { + const bc = actorCtx(branchCtx); + await branchCtx.step("mark-running", async () => { + bc.state.branches.orders = "running"; + bc.broadcast("stateChanged", bc.state); + }); + const data = await branchCtx.step("fetch-orders", async () => { + return await fetchOrderStats(); + }); + await branchCtx.step("mark-complete", async () => { + bc.state.branches.orders = "completed"; + bc.broadcast("stateChanged", bc.state); + }); + return data; + } + }, + metrics: { + run: async (branchCtx) => { + const bc = actorCtx(branchCtx); + await branchCtx.step("mark-running", async () => { + bc.state.branches.metrics = "running"; + bc.broadcast("stateChanged", bc.state); + }); + const data = await branchCtx.step("fetch-metrics", async () => { + return await fetchMetricsStats(); + }); + await branchCtx.step("mark-complete", async () => { + bc.state.branches.metrics = "completed"; + bc.broadcast("stateChanged", bc.state); + }); + return data; + } + } + }); + await loopCtx.step("save-data", async () => { + c.state.data = { + users: results.users, + orders: results.orders, + metrics: results.metrics, + fetchedAt: Date.now() + }; + c.state.loading = false; + c.state.lastRefresh = Date.now(); + c.broadcast("stateChanged", c.state); + c.broadcast("refreshComplete", c.state.data); + }); + ctx.log.info({ msg: "dashboard refresh complete" }); + return Loop6.continue(void 0); + }); + }) +}); + +// src/actors/workflow/race.ts +import { actor as actor41, event as event17 } from "rivetkit"; +import { Loop as Loop7, workflow as workflow7 } from "rivetkit/workflow"; +var race = actor41({ + createState: (c, input) => ({ + id: c.key[0], + workDurationMs: input?.workDurationMs ?? 2e3, + timeoutMs: input?.timeoutMs ?? 3e3, + status: "running", + startedAt: Date.now() + }), + events: { + raceStarted: event17(), + raceCompleted: event17() + }, + actions: { + getTask: (c) => c.state + }, + run: workflow7(async (ctx) => { + await ctx.loop("race-loop", async (loopCtx) => { + const c = actorCtx(loopCtx); + const { workDurationMs, timeoutMs, taskId } = await loopCtx.step( + "start-race", + async () => { + ctx.log.info({ + msg: "starting race", + taskId: c.state.id, + workDurationMs: c.state.workDurationMs, + timeoutMs: c.state.timeoutMs + }); + c.broadcast("raceStarted", c.state); + return { + workDurationMs: c.state.workDurationMs, + timeoutMs: c.state.timeoutMs, + taskId: c.state.id + }; + } + ); + const { winner, value } = await loopCtx.race("work-vs-timeout", [ + { + name: "work", + run: async (branchCtx) => { + await branchCtx.sleep("simulate-work", workDurationMs); + return await branchCtx.step("complete-work", async () => { + return `Result for task ${taskId}`; + }); + } + }, + { + name: "timeout", + run: async (branchCtx) => { + await branchCtx.sleep("timeout-wait", timeoutMs); + return null; + } + } + ]); + await loopCtx.step("save-result", async () => { + c.state.completedAt = Date.now(); + c.state.actualDurationMs = c.state.completedAt - c.state.startedAt; + if (winner === "work") { + c.state.status = "work_won"; + c.state.result = value; + ctx.log.info({ + msg: "work completed before timeout", + taskId: c.state.id, + durationMs: c.state.actualDurationMs + }); + } else { + c.state.status = "timeout_won"; + ctx.log.info({ + msg: "timeout won the race", + taskId: c.state.id, + durationMs: c.state.actualDurationMs + }); + } + c.broadcast("raceCompleted", c.state); + }); + return Loop7.break(void 0); + }); + }) +}); + +// src/actors/workflow/payment.ts +import { actor as actor42, event as event18 } from "rivetkit"; +import { Loop as Loop8, workflow as workflow8 } from "rivetkit/workflow"; +var payment = actor42({ + createState: (c, input) => ({ + id: c.key[0], + amount: input?.amount ?? 100, + shouldFail: input?.shouldFail ?? false, + status: "pending", + steps: [ + { name: "reserve-inventory", status: "pending" }, + { name: "charge-card", status: "pending" }, + { name: "complete-order", status: "pending" } + ], + startedAt: Date.now() + }), + events: { + transactionStarted: event18(), + transactionUpdated: event18(), + transactionCompleted: event18() + }, + actions: { + getTransaction: (c) => c.state + }, + run: workflow8(async (ctx) => { + await ctx.loop("payment-loop", async (loopCtx) => { + const c = actorCtx(loopCtx); + await loopCtx.step("init-payment", async () => { + ctx.log.info({ + msg: "starting payment processing", + txId: c.state.id, + amount: c.state.amount, + shouldFail: c.state.shouldFail + }); + c.broadcast("transactionStarted", c.state); + }); + await loopCtx.rollbackCheckpoint("payment-checkpoint"); + await loopCtx.step({ + name: "reserve-inventory", + run: async () => { + c.state.status = "reserving"; + const step = c.state.steps.find( + (s) => s.name === "reserve-inventory" + ); + if (step) { + step.status = "completed"; + step.completedAt = Date.now(); + } + c.broadcast("transactionUpdated", c.state); + await new Promise( + (r) => setTimeout(r, 500 + Math.random() * 500) + ); + ctx.log.info({ msg: "inventory reserved", txId: c.state.id }); + return { reserved: true }; + }, + rollback: async () => { + c.state.status = "rolling_back"; + const step = c.state.steps.find( + (s) => s.name === "reserve-inventory" + ); + if (step) { + step.status = "rolled_back"; + step.rolledBackAt = Date.now(); + } + ctx.log.info({ msg: "inventory released", txId: c.state.id }); + c.broadcast("transactionUpdated", c.state); + await new Promise((r) => setTimeout(r, 400)); + } + }); + await loopCtx.step({ + name: "charge-card", + run: async () => { + c.state.status = "charging"; + const step = c.state.steps.find((s) => s.name === "charge-card"); + if (step) { + step.status = "completed"; + step.completedAt = Date.now(); + } + c.broadcast("transactionUpdated", c.state); + await new Promise( + (r) => setTimeout(r, 500 + Math.random() * 500) + ); + if (c.state.shouldFail) { + throw new Error("Payment declined (simulated)"); + } + ctx.log.info({ msg: "card charged", txId: c.state.id }); + return { chargeId: `ch_${c.state.id}` }; + }, + rollback: async () => { + c.state.status = "rolling_back"; + const step = c.state.steps.find((s) => s.name === "charge-card"); + if (step) { + step.status = "rolled_back"; + step.rolledBackAt = Date.now(); + } + ctx.log.info({ msg: "charge refunded", txId: c.state.id }); + c.broadcast("transactionUpdated", c.state); + await new Promise((r) => setTimeout(r, 400)); + } + }); + await loopCtx.step("complete-order", async () => { + c.state.status = "completing"; + const step = c.state.steps.find((s) => s.name === "complete-order"); + if (step) step.status = "completed"; + c.broadcast("transactionUpdated", c.state); + await new Promise( + (r) => setTimeout(r, 300 + Math.random() * 300) + ); + c.state.status = "completed"; + c.state.completedAt = Date.now(); + ctx.log.info({ msg: "order completed", txId: c.state.id }); + c.broadcast("transactionCompleted", c.state); + }); + return Loop8.break(void 0); + }); + }) +}); + +// src/actors/workflow/history-examples.ts +import { actor as actor43, queue as queue7 } from "rivetkit"; +import { Loop as Loop9, workflow as workflow9 } from "rivetkit/workflow"; +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +var workflowHistorySimple = actor43({ + createState: (c) => ({ + id: c.key[0], + status: "pending" + }), + actions: { + getState: (c) => c.state + }, + run: workflow9(async (ctx) => { + await ctx.step("start", async () => { + const c = actorCtx(ctx); + c.state.status = "running"; + c.state.lastStep = "start"; + c.state.startedAt = Date.now(); + return { initialized: true }; + }); + await delay(700); + await ctx.step("process", async () => { + const c = actorCtx(ctx); + c.state.lastStep = "process"; + return { processed: true, items: 5 }; + }); + await delay(2200); + await ctx.step("validate", async () => { + const c = actorCtx(ctx); + c.state.lastStep = "validate"; + return { valid: true }; + }); + await delay(600); + await ctx.step("complete", async () => { + const c = actorCtx(ctx); + c.state.lastStep = "complete"; + c.state.status = "completed"; + c.state.completedAt = Date.now(); + c.state.output = { success: true, processedItems: 3 }; + return { success: true }; + }); + }) +}); +var LOOP_ITEMS = ["A", "B", "C"]; +var workflowHistoryLoop = actor43({ + createState: (c) => ({ + id: c.key[0], + status: "running", + processed: 0, + batches: [] + }), + actions: { + getState: (c) => c.state + }, + run: workflow9(async (ctx) => { + await ctx.step("init", async () => { + const c = actorCtx(ctx); + c.state.status = "running"; + return { batchSize: LOOP_ITEMS.length }; + }); + await ctx.loop({ + name: "batch-loop", + state: { index: 0 }, + commitInterval: 1, + historyEvery: 1, + historyKeep: LOOP_ITEMS.length, + run: async (loopCtx, loopState) => { + const item = LOOP_ITEMS[loopState.index]; + await loopCtx.step(`process-${loopState.index}`, async () => { + const c = actorCtx(loopCtx); + c.state.processed += 1; + c.state.batches.push({ index: loopState.index, item }); + return { item, status: "done" }; + }); + if (loopState.index >= LOOP_ITEMS.length - 1) { + return Loop9.break({ processed: LOOP_ITEMS.length }); + } + return Loop9.continue({ index: loopState.index + 1 }); + } + }); + await ctx.step("finalize", async () => { + const c = actorCtx(ctx); + c.state.status = "completed"; + c.state.completedAt = Date.now(); + return { allProcessed: true }; + }); + }) +}); +var workflowHistoryJoin = actor43({ + createState: (c) => ({ + id: c.key[0], + status: "pending" + }), + actions: { + getState: (c) => c.state + }, + run: workflow9(async (ctx) => { + await ctx.step("start", async () => { + const c = actorCtx(ctx); + c.state.status = "running"; + return { ready: true }; + }); + const results = await ctx.join("parallel-tasks", { + "fetch-api": { + run: async (branchCtx) => { + await branchCtx.step("task-a", async () => { + await delay(120); + return { fetched: true }; + }); + return { data: "api-response" }; + } + }, + "query-db": { + run: async (branchCtx) => { + await branchCtx.step("task-b", async () => { + await delay(200); + return { queried: true }; + }); + return { rows: 42 }; + } + }, + "check-cache": { + run: async (branchCtx) => { + await branchCtx.step("task-c", async () => { + await delay(60); + return { checked: true }; + }); + return { hit: true }; + } + } + }); + await ctx.step("merge-results", async () => { + const c = actorCtx(ctx); + c.state.status = "completed"; + c.state.result = { + api: results["fetch-api"].data, + rows: results["query-db"].rows, + cacheHit: results["check-cache"].hit + }; + return { merged: true }; + }); + }) +}); +var workflowHistoryRace = actor43({ + createState: (c) => ({ + id: c.key[0], + status: "running" + }), + actions: { + getState: (c) => c.state + }, + run: workflow9(async (ctx) => { + await ctx.step("begin", async () => { + const c = actorCtx(ctx); + c.state.status = "running"; + return { started: true }; + }); + const { winner, value } = await ctx.race("race-providers", [ + { + name: "provider-fast", + run: async (branchCtx) => { + await branchCtx.sleep("provider-fast-step", 50); + return { provider: "cdn-edge", latency: 12 }; + } + }, + { + name: "provider-slow", + run: async (branchCtx) => { + await branchCtx.sleep("provider-slow-step", 200); + return { provider: "origin", latency: 120 }; + } + } + ]); + await ctx.step("use-result", async () => { + const c = actorCtx(ctx); + c.state.status = "completed"; + c.state.winner = winner; + c.state.result = value.provider; + return { used: value.provider }; + }); + }) +}); +var QUEUE_ORDER_CREATED = "order:created"; +var QUEUE_ORDER_UPDATED = "order:updated"; +var QUEUE_ORDER_ITEM = "order:item"; +var QUEUE_ORDER_ARTIFACT = "order:artifact"; +var QUEUE_ORDER_READY = "order:ready"; +var QUEUE_ORDER_OPTIONAL = "order:optional"; +var FULL_WORKFLOW_MESSAGE_SEEDS = [ + { name: QUEUE_ORDER_CREATED, payload: { id: "order-1" } }, + { name: QUEUE_ORDER_UPDATED, payload: { id: "order-1", status: "paid" } }, + { name: QUEUE_ORDER_ITEM, payload: { sku: "sku-0", qty: 1 } }, + { name: QUEUE_ORDER_ITEM, payload: { sku: "sku-4", qty: 1 } }, + { name: QUEUE_ORDER_ARTIFACT, payload: { artifactId: "artifact-0" } }, + { name: QUEUE_ORDER_ARTIFACT, payload: { artifactId: "artifact-1" } }, + { name: QUEUE_ORDER_ARTIFACT, payload: { artifactId: "artifact-2" } }, + { name: QUEUE_ORDER_READY, payload: { batch: 3 } }, + { name: QUEUE_ORDER_READY, payload: { batch: 0 } }, + { name: QUEUE_ORDER_READY, payload: { batch: 2 } } +]; +var FULL_WORKFLOW_ITEMS = [ + { id: "item-1", basePrice: 100, tax: 8 }, + { id: "item-2", basePrice: 115, tax: 9 }, + { id: "item-3", basePrice: 130, tax: 10 }, + { id: "item-4", basePrice: 145, tax: 12 } +]; +var workflowHistoryFull = actor43({ + createState: (c) => ({ + id: c.key[0], + status: "pending", + seededMessages: false + }), + queues: { + [QUEUE_ORDER_CREATED]: queue7(), + [QUEUE_ORDER_UPDATED]: queue7(), + [QUEUE_ORDER_ITEM]: queue7(), + [QUEUE_ORDER_ARTIFACT]: queue7(), + [QUEUE_ORDER_READY]: queue7(), + [QUEUE_ORDER_OPTIONAL]: queue7() + }, + actions: { + getState: (c) => c.state, + seedMessages: async (c) => { + if (c.state.seededMessages) return; + for (const seed of FULL_WORKFLOW_MESSAGE_SEEDS) { + await c.queue.send(seed.name, seed.payload); + } + c.state.seededMessages = true; + } + }, + run: workflow9(async (ctx) => { + await ctx.step("bootstrap", async () => { + const c = actorCtx(ctx); + c.state.status = "running"; + c.state.lastStep = "bootstrap"; + c.state.startedAt = Date.now(); + return { + requestId: `req-${c.state.id}`, + startedAt: Date.now() + }; + }); + await ctx.step("validate-input", async () => { + const c = actorCtx(ctx); + c.state.lastStep = "validate-input"; + return true; + }); + await ctx.rollbackCheckpoint("checkpoint-after-validation"); + await ctx.step("load-user-profile", async () => { + const c = actorCtx(ctx); + c.state.lastStep = "load-user-profile"; + return { + id: "user-123", + tier: "standard", + flags: ["email-verified", "promo-eligible"] + }; + }); + await ctx.step("compute-discount", async () => { + const c = actorCtx(ctx); + c.state.lastStep = "compute-discount"; + return { percent: 5, reason: "tier-discount" }; + }); + await ctx.step("ephemeral-cache-check", async () => { + const c = actorCtx(ctx); + c.state.lastStep = "ephemeral-cache-check"; + return { cacheHit: false, tier: "standard" }; + }); + await ctx.rollbackCheckpoint("checkpoint-before-reserve"); + await ctx.loop({ + name: "process-items-loop", + state: { index: 0 }, + commitInterval: 1, + historyEvery: 1, + historyKeep: 2, + run: async (loopCtx, loopState) => { + const item = FULL_WORKFLOW_ITEMS[loopState.index]; + if (!item) { + return Loop9.break({ count: FULL_WORKFLOW_ITEMS.length }); + } + await loopCtx.step(`fetch-item-${loopState.index}`, async () => { + return { itemId: item.id, basePrice: item.basePrice }; + }); + await loopCtx.step(`compute-tax-${loopState.index}`, async () => { + return item.tax; + }); + await loopCtx.step( + `reserve-inventory-${loopState.index}`, + async () => ({ + reservationId: `res-${loopState.index}`, + itemId: item.id + }) + ); + if (loopState.index >= FULL_WORKFLOW_ITEMS.length - 1) { + return Loop9.break({ + count: FULL_WORKFLOW_ITEMS.length, + total: 504 + }); + } + return Loop9.continue({ index: loopState.index + 1 }); + } + }); + await ctx.sleep("short-cooldown", 40); + await ctx.sleep("cooldown-sleep", 60); + await ctx.sleep("wait-until-deadline", 45); + await ctx.step("compute-deadlines", async () => { + const readyBy = Date.now() + 800; + const readyBatchBy = Date.now() + 1100; + return { readyBy, readyBatchBy }; + }); + await ctx.queue.next("listen-order-created", { + names: [QUEUE_ORDER_CREATED] + }); + await ctx.queue.nextBatch("listen-order-updated-timeout", { + names: [QUEUE_ORDER_UPDATED], + timeout: 250 + }); + await ctx.queue.nextBatch("listen-batch-two", { + names: [QUEUE_ORDER_ITEM], + count: 2 + }); + await ctx.queue.nextBatch("listen-artifacts-timeout", { + names: [QUEUE_ORDER_ARTIFACT], + count: 3, + timeout: 300 + }); + await ctx.queue.nextBatch("listen-optional", { + names: [QUEUE_ORDER_OPTIONAL], + timeout: 200 + }); + await ctx.queue.nextBatch("listen-until", { + names: [QUEUE_ORDER_READY], + timeout: 300 + }); + await ctx.queue.nextBatch("listen-batch-until", { + names: [QUEUE_ORDER_READY], + count: 2, + timeout: 400 + }); + await ctx.join("join-dependencies", { + inventory: { + run: async (branchCtx) => { + const reserved = await branchCtx.step( + "inventory-audit", + async () => 4 + ); + await branchCtx.sleep("join-inventory-sleep", 35); + return { + reserved, + checked: 4, + notes: ["inventory-ok", "items=4"] + }; + } + }, + pricing: { + run: async (branchCtx) => { + const method = await branchCtx.step( + "pricing-method", + async () => "promo" + ); + return { + subtotal: 504, + discount: 25, + total: 479, + method + }; + } + }, + shipping: { + run: async (branchCtx) => { + const zone = await branchCtx.step( + "shipping-zone", + async () => "us-east" + ); + await branchCtx.sleep("join-shipping-sleep", 35); + return { method: "ground", etaDays: 4, zone }; + } + } + }); + await ctx.race("race-fulfillment", [ + { + name: "race-fast", + run: async (branchCtx) => { + await branchCtx.sleep("race-fast-sleep", 70); + return { method: "express", cost: 18, etaDays: 1 }; + } + }, + { + name: "race-slow", + run: async (branchCtx) => { + await branchCtx.sleep("race-slow-sleep", 250); + return { method: "ground", cost: 8, etaDays: 4 }; + } + } + ]); + await ctx.removed("legacy-step-placeholder", "step"); + await ctx.step("finalize", async () => { + const c = actorCtx(ctx); + c.state.lastStep = "finalize"; + c.state.status = "completed"; + c.state.completedAt = Date.now(); + return true; + }); + }) +}); +var workflowHistoryInProgress = actor43({ + createState: (c, input) => ({ + id: c.key[0], + status: "running", + processingDurationMs: input?.processingDurationMs ?? 3e4, + progress: 0 + }), + actions: { + getState: (c) => c.state + }, + run: workflow9(async (ctx) => { + await ctx.step("init", async () => { + const c = actorCtx(ctx); + c.state.startedAt = Date.now(); + c.state.progress = 10; + return { initialized: true }; + }); + await ctx.step("fetch-data", async () => { + const c = actorCtx(ctx); + c.state.progress = 25; + return { fetched: true, records: 100 }; + }); + await ctx.step("process", async () => { + const c = actorCtx(ctx); + c.state.progress = 42; + await delay(c.state.processingDurationMs); + c.state.status = "completed"; + c.state.completedAt = Date.now(); + return { processed: true }; + }); + }) +}); +var RETRY_MAX_RETRIES = 20; +var workflowHistoryRetrying = actor43({ + createState: (c) => ({ + id: c.key[0], + status: "running", + attempts: 0, + succeedAfter: 999 + }), + actions: { + getState: (c) => c.state, + allowSuccess: (c, afterAttempt) => { + c.state.succeedAfter = afterAttempt ?? c.state.attempts + 1; + } + }, + run: workflow9(async (ctx) => { + await ctx.step("start", async () => { + const c = actorCtx(ctx); + c.state.status = "running"; + return { ready: true }; + }); + await ctx.step({ + name: "api-call", + maxRetries: RETRY_MAX_RETRIES, + retryBackoffBase: 250, + retryBackoffMax: 1500, + run: async () => { + const c = actorCtx(ctx); + c.state.attempts += 1; + if (c.state.attempts < c.state.succeedAfter) { + const error = new Error("Connection timeout after 5000ms"); + c.state.lastError = error.message; + throw error; + } + c.state.status = "completed"; + c.state.lastError = void 0; + return { success: true }; + } + }); + }) +}); +var FAILED_MAX_RETRIES = 3; +var workflowHistoryFailed = actor43({ + createState: (c) => ({ + id: c.key[0], + status: "running", + attempts: 0 + }), + actions: { + getState: (c) => c.state + }, + run: workflow9(async (ctx) => { + await ctx.step("init", async () => { + const c = actorCtx(ctx); + c.state.status = "running"; + return { initialized: true }; + }); + await ctx.step("validate", async () => { + return { valid: true }; + }); + await ctx.step({ + name: "process", + maxRetries: FAILED_MAX_RETRIES, + retryBackoffBase: 200, + retryBackoffMax: 800, + run: async () => { + const c = actorCtx(ctx); + c.state.attempts += 1; + const error = new Error( + "Database connection failed: ECONNREFUSED" + ); + c.state.lastError = error.message; + throw error; + } + }); + }) +}); + +// src/actors/inter-actor/cross-actor-actions.ts +import { actor as actor44 } from "rivetkit"; +var inventory = actor44({ + // Each item has its own inventory actor instance + createState: (_c, input) => ({ + itemName: input?.itemName ?? "Widget", + stock: input?.initialStock ?? 100, + reservations: [] + }), + actions: { + // Check current stock + getStock: (c) => ({ + itemName: c.state.itemName, + stock: c.state.stock + }), + // Reserve items for checkout (called by checkout actor) + reserveItems: (c, checkoutId, quantity) => { + if (c.state.stock < quantity) { + return { + success: false, + message: `Insufficient stock. Available: ${c.state.stock}, Requested: ${quantity}`, + availableStock: c.state.stock + }; + } + c.state.stock -= quantity; + c.state.reservations.push(checkoutId); + return { + success: true, + message: `Reserved ${quantity} items for checkout ${checkoutId}`, + remainingStock: c.state.stock + }; + }, + // Release reserved items if checkout is cancelled + releaseItems: (c, checkoutId, quantity) => { + const index = c.state.reservations.indexOf(checkoutId); + if (index > -1) { + c.state.reservations.splice(index, 1); + c.state.stock += quantity; + } + return { + success: true, + remainingStock: c.state.stock + }; + } + } +}); +var checkout = actor44({ + createState: (_c, input) => ({ + customerName: input?.customerName ?? "Guest", + items: [], + completed: false + }), + actions: { + // Add item to checkout and reserve from inventory + addItem: async (c, itemId, quantity) => { + const inventoryActor = c.client().inventory.getOrCreate([itemId]); + const itemInfo = await inventoryActor.getStock(); + const reservation = await inventoryActor.reserveItems( + c.actorId, + // Use checkout ID as reservation ID + quantity + ); + if (!reservation.success) { + return { + success: false, + message: reservation.message + }; + } + c.state.items.push({ + itemId, + itemName: itemInfo.itemName, + quantity + }); + return { + success: true, + message: `Added ${quantity} ${itemInfo.itemName} to checkout`, + remainingStock: reservation.remainingStock + }; + }, + // Get checkout summary + getSummary: (c) => ({ + customerName: c.state.customerName, + items: c.state.items, + completed: c.state.completed, + totalItems: c.state.items.reduce( + (sum, item) => sum + item.quantity, + 0 + ) + }), + // Complete the checkout + completeCheckout: (c) => { + c.state.completed = true; + return { + success: true, + message: "Checkout completed successfully" + }; + }, + // Cancel checkout and release all reservations + cancelCheckout: async (c) => { + for (const item of c.state.items) { + const inventoryActor = c.client().inventory.getOrCreate([item.itemId]); + await inventoryActor.releaseItems(c.actorId, item.quantity); + } + c.state.items = []; + return { + success: true, + message: "Checkout cancelled, items returned to inventory" + }; + } + } +}); + +// src/actors/testing/inline-client.ts +import { actor as actor45 } from "rivetkit"; +function isDynamicSandboxRuntime() { + return false; +} +async function waitForConnectionOpen(connection) { + if (connection.connStatus === "connected") { + return; + } + await new Promise((resolve, reject) => { + const unsubscribeOpen = connection.onOpen(() => { + unsubscribeOpen(); + unsubscribeError(); + resolve(); + }); + const unsubscribeError = connection.onError((error) => { + unsubscribeOpen(); + unsubscribeError(); + reject(error); + }); + }); +} +var inlineClientActor = actor45({ + state: { messages: [] }, + actions: { + // Action that uses client to call another actor (stateless) + callCounterIncrement: async (c, amount) => { + const client = c.client(); + const result = await client.counter.getOrCreate(["inline-test"]).increment(amount); + c.state.messages.push( + `Called counter.increment(${amount}), result: ${result}` + ); + return result; + }, + // Action that uses client to get counter state (stateless) + getCounterState: async (c) => { + const client = c.client(); + const count = await client.counter.getOrCreate(["inline-test"]).getCount(); + c.state.messages.push(`Got counter state: ${count}`); + return count; + }, + // Action that uses client with .connect() for stateful communication + connectToCounterAndIncrement: async (c, amount) => { + const client = c.client(); + const handle = client.counter.getOrCreate(["inline-test-stateful"]); + if (isDynamicSandboxRuntime()) { + const events2 = []; + const result12 = await handle.increment(amount); + events2.push(result12); + const result22 = await handle.increment(amount * 2); + events2.push(result22); + c.state.messages.push( + `Connected to counter, incremented by ${amount} and ${amount * 2}, results: ${result12}, ${result22}, events: ${JSON.stringify(events2)}` + ); + return { result1: result12, result2: result22, events: events2 }; + } + await handle.getCount(); + const connection = handle.connect(); + await waitForConnectionOpen(connection); + const events = []; + connection.on("newCount", (count) => { + events.push(count); + }); + const result1 = await connection.increment(amount); + const result2 = await connection.increment(amount * 2); + await connection.dispose(); + c.state.messages.push( + `Connected to counter, incremented by ${amount} and ${amount * 2}, results: ${result1}, ${result2}, events: ${JSON.stringify(events)}` + ); + return { result1, result2, events }; + }, + // Get all messages from this actor's state + getMessages: (c) => { + return c.state.messages; + }, + // Clear messages + clearMessages: (c) => { + c.state.messages = []; + } + } +}); + +// src/actors/testing/test-counter.ts +import { actor as actor46 } from "rivetkit"; +var testCounter = actor46({ + state: { count: 0 }, + actions: { + increment: (c, amount = 1) => { + c.state.count += amount; + return c.state.count; + }, + getCount: (c) => { + return c.state.count; + }, + reset: (c) => { + c.state.count = 0; + return c.state.count; + } + } +}); + +// src/actors/testing/test-counter-sqlite.ts +import { actor as actor47 } from "rivetkit"; +import { db as db4 } from "rivetkit/db"; +var testCounterSqlite = actor47({ + db: db4({ + onMigrate: async (db16) => { + await db16.execute(` + CREATE TABLE IF NOT EXISTS counter ( + id INTEGER PRIMARY KEY CHECK (id = 1), + value INTEGER NOT NULL DEFAULT 0 + ) + `); + await db16.execute( + "INSERT OR IGNORE INTO counter (id, value) VALUES (1, 0)" + ); + } + }), + actions: { + increment: async (c, amount = 1) => { + await c.db.execute( + "UPDATE counter SET value = value + ? WHERE id = 1", + amount + ); + const rows = await c.db.execute( + "SELECT value FROM counter WHERE id = 1" + ); + return rows[0].value; + }, + getCount: async (c) => { + const rows = await c.db.execute( + "SELECT value FROM counter WHERE id = 1" + ); + return rows[0].value; + }, + reset: async (c) => { + await c.db.execute("UPDATE counter SET value = 0 WHERE id = 1"); + return 0; + } + } +}); + +// src/actors/testing/test-sqlite-load.ts +import { actor as actor48 } from "rivetkit"; +import { db as db5 } from "rivetkit/db"; +var testSqliteLoad = actor48({ + db: db5({ + onMigrate: async (db16) => { + await db16.execute(` + CREATE TABLE IF NOT EXISTS schema_version ( + id INTEGER PRIMARY KEY CHECK (id = 1), + version INTEGER NOT NULL DEFAULT 50 + ) + `); + await db16.execute( + "INSERT OR IGNORE INTO schema_version (id, version) VALUES (1, 50)" + ); + await db16.execute(` + CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + email TEXT, + created_at INTEGER NOT NULL + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS products ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + price REAL NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS orders ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + total REAL NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + created_at INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS order_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id INTEGER NOT NULL, + product_id INTEGER NOT NULL, + quantity INTEGER NOT NULL DEFAULT 1, + price REAL NOT NULL, + FOREIGN KEY (order_id) REFERENCES orders(id), + FOREIGN KEY (product_id) REFERENCES products(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS categories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + description TEXT + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS product_categories ( + product_id INTEGER NOT NULL, + category_id INTEGER NOT NULL, + PRIMARY KEY (product_id, category_id), + FOREIGN KEY (product_id) REFERENCES products(id), + FOREIGN KEY (category_id) REFERENCES categories(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS reviews ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + product_id INTEGER NOT NULL, + rating INTEGER NOT NULL CHECK (rating BETWEEN 1 AND 5), + comment TEXT, + created_at INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id), + FOREIGN KEY (product_id) REFERENCES products(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS addresses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + street TEXT NOT NULL, + city TEXT NOT NULL, + state TEXT, + zip TEXT, + country TEXT NOT NULL DEFAULT 'US', + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS payments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id INTEGER NOT NULL, + amount REAL NOT NULL, + method TEXT NOT NULL DEFAULT 'card', + status TEXT NOT NULL DEFAULT 'pending', + processed_at INTEGER, + FOREIGN KEY (order_id) REFERENCES orders(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS inventory ( + product_id INTEGER PRIMARY KEY, + quantity INTEGER NOT NULL DEFAULT 0, + reserved INTEGER NOT NULL DEFAULT 0, + last_restocked_at INTEGER, + FOREIGN KEY (product_id) REFERENCES products(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS coupons ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + code TEXT NOT NULL UNIQUE, + discount_percent REAL NOT NULL, + max_uses INTEGER, + used_count INTEGER NOT NULL DEFAULT 0, + expires_at INTEGER + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS shipping ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id INTEGER NOT NULL, + address_id INTEGER NOT NULL, + carrier TEXT, + tracking_number TEXT, + shipped_at INTEGER, + delivered_at INTEGER, + FOREIGN KEY (order_id) REFERENCES orders(id), + FOREIGN KEY (address_id) REFERENCES addresses(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS product_tags ( + product_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + PRIMARY KEY (product_id, tag_id), + FOREIGN KEY (product_id) REFERENCES products(id), + FOREIGN KEY (tag_id) REFERENCES tags(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS wishlists ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + name TEXT NOT NULL DEFAULT 'Default', + created_at INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS wishlist_items ( + wishlist_id INTEGER NOT NULL, + product_id INTEGER NOT NULL, + added_at INTEGER NOT NULL, + PRIMARY KEY (wishlist_id, product_id), + FOREIGN KEY (wishlist_id) REFERENCES wishlists(id), + FOREIGN KEY (product_id) REFERENCES products(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS notifications ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + type TEXT NOT NULL, + message TEXT NOT NULL, + read INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS audit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL, + entity_id INTEGER NOT NULL, + action TEXT NOT NULL, + details TEXT, + performed_at INTEGER NOT NULL + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + user_id INTEGER NOT NULL, + token TEXT NOT NULL UNIQUE, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_users_email ON users(email)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_orders_user ON orders(user_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_orders_status ON orders(status)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_reviews_product ON reviews(product_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_reviews_user ON reviews(user_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_notifications_user ON notifications(user_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_audit_entity ON audit_log(entity_type, entity_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_sessions_token ON sessions(token)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_inventory_quantity ON inventory(quantity)" + ); + await db16.execute(` + CREATE TABLE IF NOT EXISTS returns ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id INTEGER NOT NULL, + reason TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'requested', + requested_at INTEGER NOT NULL, + processed_at INTEGER, + FOREIGN KEY (order_id) REFERENCES orders(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS return_items ( + return_id INTEGER NOT NULL, + order_item_id INTEGER NOT NULL, + quantity INTEGER NOT NULL, + PRIMARY KEY (return_id, order_item_id), + FOREIGN KEY (return_id) REFERENCES returns(id), + FOREIGN KEY (order_item_id) REFERENCES order_items(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS suppliers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + contact_email TEXT, + country TEXT + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS product_suppliers ( + product_id INTEGER NOT NULL, + supplier_id INTEGER NOT NULL, + cost REAL NOT NULL, + lead_time_days INTEGER, + PRIMARY KEY (product_id, supplier_id), + FOREIGN KEY (product_id) REFERENCES products(id), + FOREIGN KEY (supplier_id) REFERENCES suppliers(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS price_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + product_id INTEGER NOT NULL, + old_price REAL NOT NULL, + new_price REAL NOT NULL, + changed_at INTEGER NOT NULL, + FOREIGN KEY (product_id) REFERENCES products(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS user_preferences ( + user_id INTEGER PRIMARY KEY, + theme TEXT NOT NULL DEFAULT 'dark', + language TEXT NOT NULL DEFAULT 'en', + notifications_enabled INTEGER NOT NULL DEFAULT 1, + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS cart ( + user_id INTEGER NOT NULL, + product_id INTEGER NOT NULL, + quantity INTEGER NOT NULL DEFAULT 1, + added_at INTEGER NOT NULL, + PRIMARY KEY (user_id, product_id), + FOREIGN KEY (user_id) REFERENCES users(id), + FOREIGN KEY (product_id) REFERENCES products(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS saved_searches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + query TEXT NOT NULL, + filters TEXT, + created_at INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS product_images ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + product_id INTEGER NOT NULL, + url TEXT NOT NULL, + alt_text TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (product_id) REFERENCES products(id) + ) + `); + await db16.execute(` + CREATE TABLE IF NOT EXISTS discounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + product_id INTEGER NOT NULL, + discount_percent REAL NOT NULL, + starts_at INTEGER NOT NULL, + ends_at INTEGER, + FOREIGN KEY (product_id) REFERENCES products(id) + ) + `); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_order_items_order ON order_items(order_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_order_items_product ON order_items(product_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_payments_order ON payments(order_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_shipping_order ON shipping(order_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_returns_order ON returns(order_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_price_history_product ON price_history(product_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_product_images_product ON product_images(product_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_discounts_product ON discounts(product_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_cart_user ON cart(user_id)" + ); + await db16.execute( + "CREATE INDEX IF NOT EXISTS idx_saved_searches_user ON saved_searches(user_id)" + ); + await db16.execute(` + CREATE TABLE IF NOT EXISTS counter ( + id INTEGER PRIMARY KEY CHECK (id = 1), + value INTEGER NOT NULL DEFAULT 0 + ) + `); + await db16.execute( + "INSERT OR IGNORE INTO counter (id, value) VALUES (1, 0)" + ); + } + }), + actions: { + increment: async (c, amount = 1) => { + await c.db.execute( + "UPDATE counter SET value = value + ? WHERE id = 1", + amount + ); + const rows = await c.db.execute( + "SELECT value FROM counter WHERE id = 1" + ); + return rows[0].value; + }, + getCount: async (c) => { + const rows = await c.db.execute( + "SELECT value FROM counter WHERE id = 1" + ); + return rows[0].value; + }, + reset: async (c) => { + await c.db.execute("UPDATE counter SET value = 0 WHERE id = 1"); + return 0; + }, + runLoadTest: async (c) => { + const now = Date.now(); + const results = []; + await c.db.execute( + "INSERT INTO users (name, email, created_at) VALUES (?, ?, ?)", + "Load Test User", + `load-${now}@test.com`, + now + ); + results.push("inserted user"); + const users = await c.db.execute( + "SELECT * FROM users WHERE email = ?", + `load-${now}@test.com` + ); + results.push(`fetched user: ${users[0].name}`); + const userId = users[0].id; + await c.db.execute( + "INSERT INTO products (name, price, created_at) VALUES (?, ?, ?)", + "Test Widget", + 29.99, + now + ); + results.push("inserted product"); + const products = await c.db.execute("SELECT * FROM products LIMIT 10"); + results.push(`fetched ${products.length} products`); + const productId = products[0].id; + await c.db.execute( + "INSERT OR IGNORE INTO categories (name, description) VALUES (?, ?)", + `test-cat-${now}`, + "A test category" + ); + results.push("inserted category"); + const categories = await c.db.execute("SELECT * FROM categories"); + results.push(`fetched ${categories.length} categories`); + await c.db.execute( + "INSERT INTO orders (user_id, total, status, created_at) VALUES (?, ?, ?, ?)", + userId, + 29.99, + "pending", + now + ); + results.push("inserted order"); + const orders = await c.db.execute( + "SELECT * FROM orders WHERE user_id = ?", + userId + ); + results.push(`fetched ${orders.length} orders for user`); + const orderId = orders[0].id; + await c.db.execute( + "INSERT INTO order_items (order_id, product_id, quantity, price) VALUES (?, ?, ?, ?)", + orderId, + productId, + 2, + 29.99 + ); + results.push("inserted order item"); + await c.db.execute( + "INSERT OR REPLACE INTO inventory (product_id, quantity, reserved, last_restocked_at) VALUES (?, ?, ?, ?)", + productId, + 100, + 2, + now + ); + results.push("inserted inventory"); + await c.db.execute( + "INSERT INTO reviews (user_id, product_id, rating, comment, created_at) VALUES (?, ?, ?, ?, ?)", + userId, + productId, + 5, + "Great product!", + now + ); + results.push("inserted review"); + const reviews = await c.db.execute( + "SELECT r.*, u.name as reviewer FROM reviews r JOIN users u ON r.user_id = u.id WHERE r.product_id = ?", + productId + ); + results.push(`fetched ${reviews.length} reviews`); + await c.db.execute( + "INSERT INTO notifications (user_id, type, message, created_at) VALUES (?, ?, ?, ?)", + userId, + "order", + "Your order has been placed", + now + ); + results.push("inserted notification"); + await c.db.execute( + "INSERT INTO audit_log (entity_type, entity_id, action, details, performed_at) VALUES (?, ?, ?, ?, ?)", + "order", + orderId, + "created", + `Order created by user ${userId}`, + now + ); + results.push("inserted audit log"); + const orderStats = await c.db.execute( + "SELECT status, COUNT(*) as count, SUM(total) as total_value FROM orders GROUP BY status" + ); + results.push( + `order stats: ${orderStats.length} statuses` + ); + await c.db.execute( + "INSERT INTO addresses (user_id, street, city, state, zip, country) VALUES (?, ?, ?, ?, ?, ?)", + userId, + "123 Test St", + "Testville", + "CA", + "90210", + "US" + ); + results.push("inserted address"); + const orderDetails = await c.db.execute(` + SELECT o.id, o.status, o.total, u.name as customer, COUNT(oi.id) as item_count + FROM orders o + JOIN users u ON o.user_id = u.id + LEFT JOIN order_items oi ON oi.order_id = o.id + GROUP BY o.id + LIMIT 10 + `); + results.push( + `fetched ${orderDetails.length} order details` + ); + await c.db.execute( + "UPDATE orders SET status = ? WHERE id = ?", + "completed", + orderId + ); + results.push("updated order status"); + const version = await c.db.execute( + "SELECT version FROM schema_version WHERE id = 1" + ); + results.push( + `schema version: ${version[0].version}` + ); + const tableCounts = await c.db.execute(` + SELECT 'users' as tbl, COUNT(*) as cnt FROM users + UNION ALL SELECT 'products', COUNT(*) FROM products + UNION ALL SELECT 'orders', COUNT(*) FROM orders + UNION ALL SELECT 'reviews', COUNT(*) FROM reviews + UNION ALL SELECT 'categories', COUNT(*) FROM categories + `); + results.push( + `table counts: ${tableCounts.length} tables checked` + ); + return { + queriesRun: 20, + results + }; + } + } +}); + +// src/actors/testing/test-sqlite-bench.ts +import { actor as actor49 } from "rivetkit"; +import { db as db6 } from "rivetkit/db"; +var CHAT_LOG_CHUNK_BYTES = 4 * 1024; +var CHAT_LOG_INSERT_BATCH_SIZE = 50; +function buildChatLogMessage(seq, targetBytes) { + const prefix = `message-${seq}: `; + return prefix + "x".repeat(Math.max(0, targetBytes - prefix.length)); +} +async function seedChatLog(database, targetBytes) { + const threadId = `chat-${crypto.randomUUID()}`; + const createdAtBase = Date.now(); + let remainingBytes = targetBytes; + let rows = 0; + await database.execute("BEGIN"); + try { + while (remainingBytes > 0) { + const placeholders = []; + const args = []; + for (let batchIndex = 0; batchIndex < CHAT_LOG_INSERT_BATCH_SIZE && remainingBytes > 0; batchIndex++) { + const contentBytes = Math.min(CHAT_LOG_CHUNK_BYTES, remainingBytes); + const seq = rows; + const role = seq % 2 === 0 ? "user" : "assistant"; + placeholders.push("(?, ?, ?, ?, ?, ?, ?)"); + args.push( + threadId, + seq, + role, + buildChatLogMessage(seq, contentBytes), + contentBytes, + Math.ceil(contentBytes / 4), + createdAtBase + seq + ); + remainingBytes -= contentBytes; + rows++; + } + await database.execute( + `INSERT INTO chat_log (thread_id, seq, role, content, content_bytes, token_estimate, created_at) VALUES ${placeholders.join(", ")}`, + ...args + ); + } + await database.execute("COMMIT"); + } catch (err) { + await database.execute("ROLLBACK"); + throw err; + } + return { threadId, rows, totalBytes: targetBytes }; +} +var testSqliteBench = actor49({ + options: { + actionTimeout: 3e5 + }, + db: db6({ + onMigrate: async (database) => { + await database.execute(`CREATE TABLE IF NOT EXISTS bench ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + key TEXT NOT NULL, + value TEXT NOT NULL, + num INTEGER NOT NULL DEFAULT 0, + payload BLOB, + created_at INTEGER NOT NULL DEFAULT 0 + )`); + await database.execute("CREATE INDEX IF NOT EXISTS idx_bench_key ON bench(key)"); + await database.execute("CREATE INDEX IF NOT EXISTS idx_bench_num ON bench(num)"); + await database.execute(`CREATE TABLE IF NOT EXISTS bench_json ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + data TEXT NOT NULL DEFAULT '{}' + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS bench_secondary ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bench_id INTEGER NOT NULL, + label TEXT NOT NULL, + score REAL NOT NULL DEFAULT 0, + FOREIGN KEY (bench_id) REFERENCES bench(id) + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS chat_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + thread_id TEXT NOT NULL, + seq INTEGER NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + content_bytes INTEGER NOT NULL, + token_estimate INTEGER NOT NULL, + created_at INTEGER NOT NULL DEFAULT 0 + )`); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_chat_log_thread_seq ON chat_log(thread_id, seq DESC)" + ); + } + }), + actions: { + noop: (_c) => ({ ok: true }), + goToSleep: (c) => { + c.sleep(); + return { ok: true }; + }, + insertSingle: async (c, n) => { + const t0 = performance.now(); + for (let i = 0; i < n; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `k-${i}`, + `v-${i}`, + i, + Date.now() + ); + } + return { ms: performance.now() - t0, ops: n }; + }, + insertTx: async (c, n) => { + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < n; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `k-${i}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: n }; + }, + insertBatch: async (c, n) => { + const t0 = performance.now(); + const placeholders = Array.from({ length: n }, () => "(?, ?, ?, ?)").join(", "); + const args = []; + for (let i = 0; i < n; i++) { + args.push(`k-${i}`, `v-${i}`, i, Date.now()); + } + await c.db.execute(`INSERT INTO bench (key, value, num, created_at) VALUES ${placeholders}`, ...args); + return { ms: performance.now() - t0, ops: n }; + }, + pointRead: async (c, n) => { + await c.db.execute("INSERT INTO bench (key, value, num, created_at) VALUES ('pr', 'pr', 0, 0)"); + const rows = await c.db.execute("SELECT id FROM bench WHERE key = 'pr' LIMIT 1"); + const id = rows[0].id; + const t0 = performance.now(); + for (let i = 0; i < n; i++) { + await c.db.execute("SELECT * FROM bench WHERE id = ?", id); + } + return { ms: performance.now() - t0, ops: n }; + }, + fullScan: async (c, seedRows) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < seedRows; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `scan-${i}`, + `val-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + const rows = await c.db.execute("SELECT * FROM bench"); + return { ms: performance.now() - t0, seedMs, rows: rows.length }; + }, + rangeScanIndexed: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 500; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `rs-${i}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + const rows = await c.db.execute("SELECT * FROM bench WHERE num BETWEEN 100 AND 300"); + return { ms: performance.now() - t0, seedMs, rows: rows.length }; + }, + rangeScanUnindexed: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 500; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `ru-${i}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + const rows = await c.db.execute("SELECT * FROM bench WHERE value BETWEEN 'v-100' AND 'v-300'"); + return { ms: performance.now() - t0, seedMs, rows: rows.length }; + }, + bulkUpdate: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 200; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `bu-${i}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + await c.db.execute("UPDATE bench SET value = 'updated', num = num + 1000 WHERE key LIKE 'bu-%'"); + return { ms: performance.now() - t0, seedMs, ops: 200 }; + }, + bulkDelete: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 200; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `bd-${i}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + await c.db.execute("DELETE FROM bench WHERE key LIKE 'bd-%'"); + return { ms: performance.now() - t0, seedMs, ops: 200 }; + }, + hotRowUpdates: async (c, n) => { + await c.db.execute("INSERT INTO bench (key, value, num, created_at) VALUES ('hot', 'v', 0, 0)"); + const rows = await c.db.execute("SELECT id FROM bench WHERE key = 'hot' LIMIT 1"); + const id = rows[0].id; + const t0 = performance.now(); + for (let i = 0; i < n; i++) { + await c.db.execute("UPDATE bench SET num = ? WHERE id = ?", i, id); + } + return { ms: performance.now() - t0, ops: n }; + }, + vacuumAfterDelete: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 500; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `vac-${i}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + await c.db.execute("DELETE FROM bench WHERE key LIKE 'vac-%'"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + await c.db.execute("VACUUM"); + return { ms: performance.now() - t0, seedMs }; + }, + largePayloadInsert: async (c, n) => { + const blob = "x".repeat(32 * 1024); + const t0 = performance.now(); + for (let i = 0; i < n; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, payload, created_at) VALUES (?, ?, ?, ?, ?)", + `lp-${i}`, + `v-${i}`, + i, + blob, + Date.now() + ); + } + return { ms: performance.now() - t0, ops: n }; + }, + mixedOltp: async (c) => { + const t0 = performance.now(); + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + "oltp", + "initial", + 0, + Date.now() + ); + const rows = await c.db.execute("SELECT * FROM bench WHERE key = 'oltp' LIMIT 1"); + const id = rows[0].id; + await c.db.execute("UPDATE bench SET value = 'updated', num = 1 WHERE id = ?", id); + await c.db.execute("SELECT * FROM bench WHERE id = ?", id); + return { ms: performance.now() - t0, ops: 4 }; + }, + jsonInsertAndQuery: async (c) => { + const t0 = performance.now(); + for (let i = 0; i < 50; i++) { + await c.db.execute( + "INSERT INTO bench_json (data) VALUES (?)", + JSON.stringify({ name: `item-${i}`, tags: ["a", "b"], score: Math.random() * 100 }) + ); + } + const rows = await c.db.execute( + "SELECT id, json_extract(data, '$.name') as name, json_extract(data, '$.score') as score FROM bench_json ORDER BY json_extract(data, '$.score') DESC LIMIT 10" + ); + return { ms: performance.now() - t0, ops: 51, rows: rows.length }; + }, + jsonEachAgg: async (c) => { + await c.db.execute( + "INSERT INTO bench_json (data) VALUES (?)", + JSON.stringify({ items: Array.from({ length: 100 }, (_, i) => ({ id: i, val: i * 10 })) }) + ); + const t0 = performance.now(); + const rows = await c.db.execute( + "SELECT SUM(json_extract(value, '$.val')) as total FROM bench_json, json_each(json_extract(data, '$.items')) LIMIT 1" + ); + return { ms: performance.now() - t0, total: rows[0].total }; + }, + complexAggregation: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 200; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `grp-${i % 10}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + const rows = await c.db.execute( + "SELECT key, COUNT(*) as cnt, AVG(num) as avg_num, MIN(num) as min_num, MAX(num) as max_num FROM bench WHERE key LIKE 'grp-%' GROUP BY key ORDER BY cnt DESC" + ); + return { ms: performance.now() - t0, seedMs, groups: rows.length }; + }, + complexSubquery: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 200; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `sq-${i}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + const rows = await c.db.execute( + "SELECT * FROM bench WHERE num > (SELECT AVG(num) FROM bench) ORDER BY num DESC LIMIT 50" + ); + return { ms: performance.now() - t0, seedMs, rows: rows.length }; + }, + complexJoin: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 200; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `j-${i}`, + `v-${i}`, + i, + Date.now() + ); + await c.db.execute( + "INSERT INTO bench_secondary (bench_id, label, score) VALUES (?, ?, ?)", + i + 1, + `label-${i}`, + Math.random() * 100 + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + const rows = await c.db.execute( + "SELECT b.key, b.num, s.label, s.score FROM bench b INNER JOIN bench_secondary s ON s.bench_id = b.id WHERE b.key LIKE 'j-%' ORDER BY s.score DESC LIMIT 200" + ); + return { ms: performance.now() - t0, seedMs, rows: rows.length }; + }, + complexCteWindow: async (c) => { + const t0Seed = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 200; i++) { + await c.db.execute( + "INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)", + `cte-${i % 10}`, + `v-${i}`, + i, + Date.now() + ); + } + await c.db.execute("COMMIT"); + const seedMs = performance.now() - t0Seed; + const t0 = performance.now(); + const rows = await c.db.execute(` + WITH ranked AS ( + SELECT key, num, ROW_NUMBER() OVER (PARTITION BY key ORDER BY num DESC) as rn, + AVG(num) OVER (PARTITION BY key) as avg_num + FROM bench + WHERE key LIKE 'cte-%' + ) + SELECT * FROM ranked WHERE rn <= 3 ORDER BY key, rn + `); + return { ms: performance.now() - t0, seedMs, rows: rows.length }; + }, + migrationTables: async (c, n) => { + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < n; i++) { + await c.db.execute(`CREATE TABLE IF NOT EXISTS mig_${i} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + data TEXT NOT NULL DEFAULT '' + )`); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: n }; + }, + chatLogInsert: async (c, totalBytes) => { + const t0 = performance.now(); + const seeded = await seedChatLog(c.db, totalBytes); + return { ms: performance.now() - t0, ops: seeded.rows, bytes: seeded.totalBytes }; + }, + chatLogSelectLimit: async (c, totalBytes) => { + const seeded = await seedChatLog(c.db, totalBytes); + const t0 = performance.now(); + const rows = await c.db.execute( + "SELECT seq, role, substr(content, 1, 128) AS preview FROM chat_log ORDER BY created_at DESC LIMIT 100" + ); + return { + ms: performance.now() - t0, + ops: rows.length, + rows: rows.length, + bytes: seeded.totalBytes + }; + }, + chatLogSelectIndexed: async (c, totalBytes) => { + const seeded = await seedChatLog(c.db, totalBytes); + const lowerBound = Math.max(0, seeded.rows - 100); + const t0 = performance.now(); + const rows = await c.db.execute( + "SELECT seq, role, content_bytes FROM chat_log WHERE thread_id = ? AND seq >= ? ORDER BY seq DESC LIMIT 100", + seeded.threadId, + lowerBound + ); + return { + ms: performance.now() - t0, + ops: rows.length, + rows: rows.length, + bytes: seeded.totalBytes + }; + }, + chatLogCount: async (c, totalBytes) => { + const seeded = await seedChatLog(c.db, totalBytes); + const t0 = performance.now(); + const rows = await c.db.execute( + "SELECT COUNT(*) AS count FROM chat_log WHERE thread_id = ?", + seeded.threadId + ); + return { + ms: performance.now() - t0, + ops: 1, + count: rows[0].count, + bytes: seeded.totalBytes + }; + }, + chatLogSum: async (c, totalBytes) => { + const seeded = await seedChatLog(c.db, totalBytes); + const t0 = performance.now(); + const rows = await c.db.execute( + "SELECT SUM(content_bytes) AS total_bytes FROM chat_log WHERE thread_id = ?", + seeded.threadId + ); + return { + ms: performance.now() - t0, + ops: 1, + totalBytes: rows[0].total_bytes ?? 0, + bytes: seeded.totalBytes + }; + }, + largeTxInsert500KB: async (c) => { + const targetBytes = 500 * 1024; + const rowSize = 4 * 1024; + const rowCount = Math.ceil(targetBytes / rowSize); + await c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < rowCount; i++) { + await c.db.execute( + "INSERT INTO large_tx (payload) VALUES (randomblob(?))", + rowSize + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize }; + }, + largeTxInsert1MB: async (c) => { + const targetBytes = 1024 * 1024; + const rowSize = 4 * 1024; + const rowCount = Math.ceil(targetBytes / rowSize); + await c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < rowCount; i++) { + await c.db.execute( + "INSERT INTO large_tx (payload) VALUES (randomblob(?))", + rowSize + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize }; + }, + // 1 MiB total, 4096 × 256 B rows. Max NAPI crossings. + largeTxInsert1MBTinyRows: async (c) => { + const targetBytes = 1024 * 1024; + const rowSize = 256; + const rowCount = Math.ceil(targetBytes / rowSize); + await c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < rowCount; i++) { + await c.db.execute( + "INSERT INTO large_tx (payload) VALUES (randomblob(?))", + rowSize + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize }; + }, + // 1 MiB total, 256 × 4 KiB rows. Same shape as largeTxInsert1MB; kept as a sanity duplicate. + largeTxInsert1MBMediumRows: async (c) => { + const targetBytes = 1024 * 1024; + const rowSize = 4 * 1024; + const rowCount = Math.ceil(targetBytes / rowSize); + await c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < rowCount; i++) { + await c.db.execute( + "INSERT INTO large_tx (payload) VALUES (randomblob(?))", + rowSize + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize }; + }, + // 1 MiB total, 1 × 1 MiB row. One NAPI crossing, exercises SQLite overflow-page chain. + largeTxInsert1MBOneRow: async (c) => { + const rowSize = 1024 * 1024; + await c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + await c.db.execute( + "INSERT INTO large_tx (payload) VALUES (randomblob(?))", + rowSize + ); + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: 1, bytes: rowSize }; + }, + largeTxInsert5MB: async (c) => { + const targetBytes = 5 * 1024 * 1024; + const rowSize = 4 * 1024; + const rowCount = Math.ceil(targetBytes / rowSize); + await c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < rowCount; i++) { + await c.db.execute( + "INSERT INTO large_tx (payload) VALUES (randomblob(?))", + rowSize + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize }; + }, + largeTxInsert10MB: async (c) => { + const targetBytes = 10 * 1024 * 1024; + const rowSize = 4 * 1024; + const rowCount = Math.ceil(targetBytes / rowSize); + await c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < rowCount; i++) { + await c.db.execute( + "INSERT INTO large_tx (payload) VALUES (randomblob(?))", + rowSize + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize }; + }, + largeTxInsert50MB: async (c) => { + const targetBytes = 50 * 1024 * 1024; + const rowSize = 4 * 1024; + const rowCount = Math.ceil(targetBytes / rowSize); + await c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < rowCount; i++) { + await c.db.execute( + "INSERT INTO large_tx (payload) VALUES (randomblob(?))", + rowSize + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize }; + }, + // Stress test: insert 1000 rows, delete them all, repeat 10 times. + // Tests freelist reuse and space reclamation patterns. + churnInsertDelete: async (c) => { + await c.db.execute(`CREATE TABLE IF NOT EXISTS churn ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + const t0 = performance.now(); + const cycles = 10; + const perCycle = 1e3; + for (let cycle = 0; cycle < cycles; cycle++) { + await c.db.execute("BEGIN"); + for (let i = 0; i < perCycle; i++) { + await c.db.execute( + "INSERT INTO churn (payload) VALUES (randomblob(1024))" + ); + } + await c.db.execute("DELETE FROM churn"); + await c.db.execute("COMMIT"); + } + return { + ms: performance.now() - t0, + ops: cycles * perCycle, + cycles + }; + }, + // Interleave inserts, updates, deletes in same transaction. Tests how + // the VFS handles mixed page dirtying patterns. + mixedOltpLarge: async (c) => { + await c.db.execute(`CREATE TABLE IF NOT EXISTS mixed_oltp ( + id INTEGER PRIMARY KEY, + value INTEGER NOT NULL, + data BLOB NOT NULL + )`); + await c.db.execute("DELETE FROM mixed_oltp"); + await c.db.execute("BEGIN"); + for (let i = 0; i < 500; i++) { + await c.db.execute( + "INSERT INTO mixed_oltp (id, value, data) VALUES (?, ?, randomblob(1024))", + i, + i * 2 + ); + } + await c.db.execute("COMMIT"); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 500; i++) { + await c.db.execute( + "INSERT INTO mixed_oltp (id, value, data) VALUES (?, ?, randomblob(1024))", + 500 + i, + i * 3 + ); + await c.db.execute( + "UPDATE mixed_oltp SET value = value + 1 WHERE id = ?", + i + ); + if (i % 5 === 0) { + await c.db.execute( + "DELETE FROM mixed_oltp WHERE id = ?", + i - 50 >= 0 ? i - 50 : i + ); + } + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: 500 * 2 + 100 }; + }, + // Growing aggregation: insert then SELECT SUM after each batch. + // Tests cache invalidation and read-after-write patterns. + growingAggregation: async (c) => { + await c.db.execute(`CREATE TABLE IF NOT EXISTS agg_test ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + value INTEGER NOT NULL + )`); + await c.db.execute("DELETE FROM agg_test"); + const t0 = performance.now(); + const batches = 20; + const perBatch = 100; + let lastSum = 0; + for (let batch2 = 0; batch2 < batches; batch2++) { + await c.db.execute("BEGIN"); + for (let i = 0; i < perBatch; i++) { + await c.db.execute( + "INSERT INTO agg_test (value) VALUES (?)", + batch2 * perBatch + i + ); + } + await c.db.execute("COMMIT"); + const rows = await c.db.execute( + "SELECT SUM(value) AS s FROM agg_test" + ); + lastSum = rows[0]?.s ?? 0; + } + return { + ms: performance.now() - t0, + ops: batches * perBatch, + batches, + lastSum + }; + }, + // Create index on already-populated table. Tests large rewrite patterns. + indexCreationOnLargeTable: async (c) => { + await c.db.execute(`CREATE TABLE IF NOT EXISTS idx_test ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + key TEXT NOT NULL, + value INTEGER NOT NULL + )`); + await c.db.execute("DROP INDEX IF EXISTS idx_test_key"); + await c.db.execute("DELETE FROM idx_test"); + await c.db.execute("BEGIN"); + for (let i = 0; i < 1e4; i++) { + await c.db.execute( + "INSERT INTO idx_test (key, value) VALUES (?, ?)", + `key-${i % 1e3}-${i}`, + i + ); + } + await c.db.execute("COMMIT"); + const t0 = performance.now(); + await c.db.execute("CREATE INDEX idx_test_key ON idx_test(key)"); + return { ms: performance.now() - t0, ops: 1e4 }; + }, + // Update 1000 different rows in separate UPDATEs in one transaction. + // Stresses B-tree navigation and page dirtying. + bulkUpdate1000Rows: async (c) => { + await c.db.execute(`CREATE TABLE IF NOT EXISTS bulk_update ( + id INTEGER PRIMARY KEY, + value INTEGER NOT NULL + )`); + await c.db.execute("DELETE FROM bulk_update"); + await c.db.execute("BEGIN"); + for (let i = 0; i < 1e3; i++) { + await c.db.execute( + "INSERT INTO bulk_update (id, value) VALUES (?, ?)", + i, + i + ); + } + await c.db.execute("COMMIT"); + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 1e3; i++) { + await c.db.execute( + "UPDATE bulk_update SET value = value + 1 WHERE id = ?", + i + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: 1e3 }; + }, + // Delete everything then re-insert. Tests truncate+regrow cycle. + truncateAndRegrow: async (c) => { + await c.db.execute(`CREATE TABLE IF NOT EXISTS regrow ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload BLOB NOT NULL + )`); + await c.db.execute("BEGIN"); + for (let i = 0; i < 500; i++) { + await c.db.execute( + "INSERT INTO regrow (payload) VALUES (randomblob(1024))" + ); + } + await c.db.execute("COMMIT"); + const t0 = performance.now(); + await c.db.execute("DELETE FROM regrow"); + await c.db.execute("BEGIN"); + for (let i = 0; i < 500; i++) { + await c.db.execute( + "INSERT INTO regrow (payload) VALUES (randomblob(1024))" + ); + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: 500 }; + }, + // Many small tables vs one large. Tests schema page growth. + manySmallTables: async (c) => { + const t0 = performance.now(); + await c.db.execute("BEGIN"); + for (let i = 0; i < 50; i++) { + await c.db.execute( + `CREATE TABLE IF NOT EXISTS small_t_${i} (id INTEGER PRIMARY KEY, value INTEGER)` + ); + for (let j = 0; j < 10; j++) { + await c.db.execute( + `INSERT INTO small_t_${i} (id, value) VALUES (?, ?)`, + j, + i * j + ); + } + } + await c.db.execute("COMMIT"); + return { ms: performance.now() - t0, ops: 50 * 10, tables: 50 }; + } + } +}); + +// src/actors/testing/sqlite-cold-start-bench.ts +import { randomBytes } from "crypto"; +import { actor as actor50 } from "rivetkit"; +import { db as db7 } from "rivetkit/db"; +var DEFAULT_TARGET_BYTES = 50 * 1024 * 1024; +var DEFAULT_ROW_BYTES = 16 * 1024; +var DEFAULT_BATCH_ROWS = 8; +var DEFAULT_TRANSACTION_BYTES = 64 * 1024; +var READ_BATCH_ROWS = 64; +var REVERSE_PROBE_ROWS = 32 * 1024; +var PAYLOAD_TABLE = "cold_start_payload"; +var REVERSE_PROBE_TABLE = "cold_start_reverse_probe"; +function positiveInteger(value, fallback, name) { + const resolved = value ?? fallback; + if (!Number.isInteger(resolved) || resolved < 1) { + throw new Error(`${name} must be a positive integer`); + } + return resolved; +} +function randomAsciiString(bytes) { + return randomBytes(Math.ceil(bytes / 2)).toString("hex").slice(0, bytes); +} +async function readPayloads(database, direction = "forward") { + const t0 = performance.now(); + const [bounds] = await database.execute( + ` + SELECT + MIN(id) AS min_id, + MAX(id) AS max_id, + COUNT(*) AS rows, + 0 AS bytes, + 0 AS expected_bytes + FROM ${PAYLOAD_TABLE} + ` + ); + if (!bounds) throw new Error("read query returned no rows"); + let rows = 0; + let bytes = 0; + let expectedBytes = 0; + let chunks = 0; + const minId = bounds.min_id ?? 0; + const maxId = bounds.max_id ?? 0; + if (direction === "backward") { + const [probeBounds] = await database.execute( + ` + SELECT + MIN(id) AS min_id, + MAX(id) AS max_id, + COUNT(*) AS rows, + 0 AS bytes, + 0 AS expected_bytes + FROM ${REVERSE_PROBE_TABLE} + ` + ); + if (!probeBounds) throw new Error("reverse probe query returned no rows"); + const probeMinId = probeBounds.min_id ?? 0; + const probeMaxId = probeBounds.max_id ?? 0; + for (let upperId = probeMaxId; upperId >= probeMinId && upperId > 0; upperId -= READ_BATCH_ROWS) { + const lowerId = Math.max(probeMinId, upperId - READ_BATCH_ROWS + 1); + const chunkRows = await database.execute( + ` + SELECT + marker AS bytes, + marker AS expected_bytes + FROM ${REVERSE_PROBE_TABLE} + WHERE id BETWEEN ? AND ? + ORDER BY id DESC + `, + lowerId, + upperId + ); + for (const row of chunkRows) { + rows += 1; + bytes += row.bytes; + expectedBytes += row.expected_bytes; + } + chunks += 1; + } + return { + ms: performance.now() - t0, + ops: rows, + rows, + bytes, + expectedBytes, + chunks, + readBatchRows: READ_BATCH_ROWS, + direction + }; + } + for (let lowerId = minId; lowerId <= maxId; lowerId += READ_BATCH_ROWS) { + const upperId = lowerId + READ_BATCH_ROWS - 1; + const [chunk] = await database.execute( + ` + SELECT + COUNT(*) AS rows, + COALESCE(SUM(length(payload)), 0) AS bytes, + COALESCE(SUM(payload_bytes), 0) AS expected_bytes + FROM ${PAYLOAD_TABLE} + WHERE id BETWEEN ? AND ? + `, + lowerId, + upperId + ); + if (!chunk) throw new Error("chunked read query returned no rows"); + rows += chunk.rows; + bytes += chunk.bytes; + expectedBytes += chunk.expected_bytes; + chunks += 1; + } + return { + ms: performance.now() - t0, + ops: rows, + rows, + bytes, + expectedBytes, + chunks, + readBatchRows: READ_BATCH_ROWS + }; +} +var sqliteColdStartBench = actor50({ + options: { + actionTimeout: 6e5 + }, + db: db7({ + onMigrate: async (database) => { + await database.execute(` + CREATE TABLE IF NOT EXISTS cold_start_payload ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload TEXT NOT NULL, + payload_bytes INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS cold_start_reverse_probe ( + id INTEGER PRIMARY KEY, + marker INTEGER NOT NULL + ) + `); + } + }), + actions: { + reset: async (c) => { + await c.db.execute(`DELETE FROM ${PAYLOAD_TABLE}`); + await c.db.execute(`DELETE FROM ${REVERSE_PROBE_TABLE}`); + return { ok: true }; + }, + writeRandomStrings: async (c, input = {}) => { + const targetBytes = positiveInteger( + input.targetBytes, + DEFAULT_TARGET_BYTES, + "targetBytes" + ); + const rowBytes = positiveInteger(input.rowBytes, DEFAULT_ROW_BYTES, "rowBytes"); + const batchRows = positiveInteger( + input.batchRows, + DEFAULT_BATCH_ROWS, + "batchRows" + ); + const transactionBytes = positiveInteger( + input.transactionBytes, + DEFAULT_TRANSACTION_BYTES, + "transactionBytes" + ); + const createdAt = Date.now(); + let remainingBytes = targetBytes; + let rows = 0; + let transactions = 0; + let randomStringMs = 0; + let sqliteInsertMs = 0; + let commitMs = 0; + let inTransaction = false; + const wallT0 = performance.now(); + try { + while (remainingBytes > 0) { + let transactionRemainingBytes = Math.min( + transactionBytes, + remainingBytes + ); + await c.db.execute("BEGIN"); + inTransaction = true; + transactions += 1; + while (transactionRemainingBytes > 0) { + const placeholders = []; + const args = []; + const generateT0 = performance.now(); + for (let batchIndex = 0; batchIndex < batchRows && transactionRemainingBytes > 0 && remainingBytes > 0; batchIndex += 1) { + const payloadBytes = Math.min( + rowBytes, + transactionRemainingBytes, + remainingBytes + ); + placeholders.push("(?, ?, ?)"); + args.push( + randomAsciiString(payloadBytes), + payloadBytes, + createdAt + rows + ); + transactionRemainingBytes -= payloadBytes; + remainingBytes -= payloadBytes; + rows += 1; + } + randomStringMs += performance.now() - generateT0; + const insertT0 = performance.now(); + await c.db.execute( + `INSERT INTO ${PAYLOAD_TABLE} (payload, payload_bytes, created_at) VALUES ${placeholders.join(", ")}`, + ...args + ); + sqliteInsertMs += performance.now() - insertT0; + } + const commitT0 = performance.now(); + await c.db.execute("COMMIT"); + commitMs += performance.now() - commitT0; + inTransaction = false; + } + await c.db.execute("BEGIN"); + inTransaction = true; + for (let lowerId = 1; lowerId <= REVERSE_PROBE_ROWS; lowerId += 256) { + const upperId = Math.min(REVERSE_PROBE_ROWS, lowerId + 255); + const placeholders = []; + const args = []; + for (let id = lowerId; id <= upperId; id += 1) { + placeholders.push("(?, ?)"); + args.push(id, 1); + } + const insertT0 = performance.now(); + await c.db.execute( + `INSERT INTO ${REVERSE_PROBE_TABLE} (id, marker) VALUES ${placeholders.join(", ")}`, + ...args + ); + sqliteInsertMs += performance.now() - insertT0; + } + const reverseCommitT0 = performance.now(); + await c.db.execute("COMMIT"); + commitMs += performance.now() - reverseCommitT0; + inTransaction = false; + return { + ms: sqliteInsertMs + commitMs, + writeWallMs: performance.now() - wallT0, + randomStringMs, + sqliteInsertMs, + commitMs, + ops: rows, + rows, + transactions, + bytes: targetBytes, + rowBytes, + batchRows, + transactionBytes, + reverseProbeRows: REVERSE_PROBE_ROWS + }; + } catch (err) { + if (inTransaction) { + await c.db.execute("ROLLBACK"); + } + throw err; + } + }, + readAll: async (c) => { + return readPayloads(c.db); + }, + readAllReverse: async (c) => { + return readPayloads(c.db, "backward"); + }, + wakeSqlite: async (c) => { + const t0 = performance.now(); + const [row] = await c.db.execute( + `SELECT COUNT(*) AS rows FROM ${PAYLOAD_TABLE} WHERE id = -1` + ); + return { + ms: performance.now() - t0, + rows: row?.rows ?? 0 + }; + }, + goToSleep: (c) => { + c.sleep(); + return { ok: true }; + } + } +}); + +// src/actors/testing/sqlite-realworld-bench.ts +import { actor as actor51 } from "rivetkit"; +import { db as db8 } from "rivetkit/db"; +var DEFAULT_ROW_BYTES2 = 2 * 1024; +var ORDER_BATCH_ROWS = 50; +var DOC_BATCH_ROWS = 75; +var LEDGER_BATCH_ROWS = 100; +var POINT_LOOKUP_OPS = 1e3; +var RANGE_CHUNK_ROWS = 512; +var SETUP_TRANSACTION_ROWS = 128; +var FEED_PAGE_ROWS = 100; +var CHAT_LOG_CHUNK_BYTES2 = 4 * 1024; +var CHAT_LOG_INSERT_BATCH_SIZE2 = 50; +var CHAT_THREAD_ID = "rw-chat-main"; +var SQL_RUSH_MSGS_COUNT = 2500; +var SQL_RUSH_TOOL_REFS_COUNT = 240; +var SQL_RUSH_EVENTS_COUNT = 700; +var SQL_RUSH_KV_COUNT = 40; +var SQL_RUSH_TOOLS_COUNT = 41; +var SQL_RUSH_META_COUNT = 12; +var WORKLOADS = [ + "small-rowid-point", + "small-schema-read", + "small-range-scan", + "rowid-range-forward", + "rowid-range-backward", + "secondary-index-covering-range", + "secondary-index-scattered-table", + "aggregate-status", + "aggregate-time-bucket", + "aggregate-tenant-time-range", + "parallel-read-aggregates", + "parallel-read-write-transition", + "feed-order-by-limit", + "feed-pagination-adjacent", + "join-order-items", + "random-point-lookups", + "hot-index-cold-table", + "ledger-without-rowid-range", + "chat-log-select-limit", + "chat-log-select-indexed", + "chat-log-count", + "chat-log-sum", + "chat-tool-read-fanout", + "chat-tool-script", + "write-batch-after-wake", + "update-hot-partition", + "delete-churn-range-read", + "migration-create-indexes-large", + "migration-create-indexes-skewed-large", + "migration-table-rebuild-large", + "migration-add-column-large", + "migration-ddl-small" +]; +function positiveInteger2(value, fallback, name) { + const resolved = value ?? fallback; + if (!Number.isInteger(resolved) || resolved < 1) { + throw new Error(`${name} must be a positive integer`); + } + return resolved; +} +function assertWorkload(workload) { + if (!WORKLOADS.includes(workload)) { + throw new Error(`unknown SQLite benchmark workload: ${workload}`); + } +} +function pseudoRandom(value) { + return Math.imul(value ^ 2654435769, 2246822507) >>> 0; +} +function paddedHex(value) { + return pseudoRandom(value).toString(16).padStart(8, "0"); +} +function payload(prefix, bytes) { + return prefix + "x".repeat(Math.max(0, bytes - prefix.length)); +} +function typedRows(rows) { + return rows; +} +async function queryPageCount(database) { + const [row] = typedRows(await database.execute("PRAGMA page_count")); + return row?.page_count ?? 0; +} +async function resetCommerce(database) { + await database.execute("DELETE FROM rw_order_items"); + await database.execute("DELETE FROM rw_orders"); + await database.execute("DELETE FROM rw_customers"); + await database.execute("DELETE FROM rw_events"); +} +async function resetDocs(database) { + await database.execute("DELETE FROM rw_docs"); +} +async function resetLedger(database) { + await database.execute("DELETE FROM rw_ledger"); +} +async function resetChatLog(database) { + await database.execute("DELETE FROM rw_chat_log"); +} +async function resetSqlRush(database) { + await database.execute("DELETE FROM tool_refs"); + await database.execute("DELETE FROM msgs"); + await database.execute("DELETE FROM events"); + await database.execute("DELETE FROM kv"); + await database.execute("DELETE FROM tools"); + await database.execute("DELETE FROM meta"); +} +async function resetMigration(database) { + await database.execute("DROP INDEX IF EXISTS idx_rw_migration_source_account"); + await database.execute("DROP INDEX IF EXISTS idx_rw_migration_source_created"); + await database.execute("DROP INDEX IF EXISTS idx_rw_migration_source_status_total"); + await database.execute("DROP INDEX IF EXISTS idx_rw_migration_source_skew_account"); + await database.execute("DROP INDEX IF EXISTS idx_rw_migration_source_skew_status"); + await database.execute("DROP TABLE IF EXISTS rw_migration_source_rebuilt"); + await database.execute("DROP TABLE IF EXISTS rw_migration_source"); + await database.execute("DROP TABLE IF EXISTS rw_migration_audit"); + await database.execute("DROP TABLE IF EXISTS rw_migration_empty"); +} +async function withTransaction(database, fn) { + let inTransaction = false; + await database.execute("BEGIN"); + inTransaction = true; + try { + await fn(); + await database.execute("COMMIT"); + inTransaction = false; + } catch (err) { + if (inTransaction) { + await database.execute("ROLLBACK").catch(() => void 0); + } + throw err; + } +} +async function seedCommerce(database, targetBytes, rowBytes) { + await resetCommerce(database); + const rows = Math.max(1, Math.ceil(targetBytes / rowBytes)); + const customerCount = Math.max(32, Math.ceil(rows / 16)); + const startedAt = performance.now(); + await withTransaction(database, async () => { + for (let offset = 0; offset < customerCount; offset += ORDER_BATCH_ROWS) { + const placeholders = []; + const args = []; + const batchEnd = Math.min(customerCount, offset + ORDER_BATCH_ROWS); + for (let i = offset; i < batchEnd; i += 1) { + placeholders.push("(?, ?, ?, ?, ?)"); + args.push( + i + 1, + `acct-${i % 64}`, + `user-${paddedHex(i)}@example.test`, + ["free", "pro", "team", "enterprise"][i % 4], + ["iad", "sfo", "fra", "sin"][i % 4] + ); + } + await database.execute( + `INSERT INTO rw_customers (id, account_id, email, plan, region) VALUES ${placeholders.join(", ")}`, + ...args + ); + } + }); + for (let txStart = 0; txStart < rows; txStart += SETUP_TRANSACTION_ROWS) { + const txEnd = Math.min(rows, txStart + SETUP_TRANSACTION_ROWS); + await withTransaction(database, async () => { + for (let offset = txStart; offset < txEnd; offset += ORDER_BATCH_ROWS) { + const orderPlaceholders = []; + const orderArgs = []; + const itemPlaceholders = []; + const itemArgs = []; + const eventPlaceholders = []; + const eventArgs = []; + const batchEnd = Math.min(txEnd, offset + ORDER_BATCH_ROWS); + for (let i = offset; i < batchEnd; i += 1) { + const id = i + 1; + const customerId = pseudoRandom(i) % customerCount + 1; + const createdAt = 17e11 + i * 1e3; + const status = ["pending", "paid", "shipped", "refunded"][i % 4]; + const totalCents = 500 + pseudoRandom(i + 17) % 25e3; + const note = payload(`order-${id}-${status}:`, rowBytes); + orderPlaceholders.push("(?, ?, ?, ?, ?, ?, ?)"); + orderArgs.push( + id, + customerId, + createdAt, + status, + totalCents, + i % 128, + note + ); + for (let item = 0; item < 2; item += 1) { + itemPlaceholders.push("(?, ?, ?, ?, ?)"); + itemArgs.push( + id, + `sku-${paddedHex(i + item).slice(0, 6)}`, + 1 + (i + item) % 5, + 100 + pseudoRandom(i + item + 31) % 5e3, + item + ); + } + eventPlaceholders.push("(?, ?, ?, ?, ?)"); + eventArgs.push( + `acct-${customerId % 64}`, + ["click", "purchase", "refund", "shipment"][i % 4], + createdAt, + `order:${id}`, + payload(`event-${id}:`, Math.min(rowBytes, 512)) + ); + } + await database.execute( + `INSERT INTO rw_orders (id, customer_id, created_at, status, total_cents, shard, note) VALUES ${orderPlaceholders.join(", ")}`, + ...orderArgs + ); + await database.execute( + `INSERT INTO rw_order_items (order_id, sku, quantity, price_cents, line_no) VALUES ${itemPlaceholders.join(", ")}`, + ...itemArgs + ); + await database.execute( + `INSERT INTO rw_events (account_id, event_type, created_at, entity_key, properties) VALUES ${eventPlaceholders.join(", ")}`, + ...eventArgs + ); + } + }); + } + return { + rows, + targetBytes, + rowBytes, + setupMs: performance.now() - startedAt, + pageCount: await queryPageCount(database) + }; +} +async function seedDocs(database, targetBytes, rowBytes) { + await resetDocs(database); + const rows = Math.max(1, Math.ceil(targetBytes / rowBytes)); + const startedAt = performance.now(); + for (let txStart = 0; txStart < rows; txStart += SETUP_TRANSACTION_ROWS) { + const txEnd = Math.min(rows, txStart + SETUP_TRANSACTION_ROWS); + await withTransaction(database, async () => { + for (let offset = txStart; offset < txEnd; offset += DOC_BATCH_ROWS) { + const placeholders = []; + const args = []; + const batchEnd = Math.min(txEnd, offset + DOC_BATCH_ROWS); + for (let i = offset; i < batchEnd; i += 1) { + const rank = pseudoRandom(i); + const body = payload(`doc-${i}-${rank}:`, rowBytes); + placeholders.push("(?, ?, ?, ?, ?)"); + args.push( + `doc-${paddedHex(i)}`, + rank, + `tenant-${rank % 128}`, + body, + rowBytes + ); + } + await database.execute( + `INSERT INTO rw_docs (external_key, row_rank, tenant_id, body, body_bytes) VALUES ${placeholders.join(", ")}`, + ...args + ); + } + }); + } + return { + rows, + targetBytes, + rowBytes, + setupMs: performance.now() - startedAt, + pageCount: await queryPageCount(database) + }; +} +async function seedLedger(database, targetBytes, rowBytes) { + await resetLedger(database); + const rows = Math.max(1, Math.ceil(targetBytes / rowBytes)); + const startedAt = performance.now(); + for (let txStart = 0; txStart < rows; txStart += SETUP_TRANSACTION_ROWS) { + const txEnd = Math.min(rows, txStart + SETUP_TRANSACTION_ROWS); + await withTransaction(database, async () => { + for (let offset = txStart; offset < txEnd; offset += LEDGER_BATCH_ROWS) { + const placeholders = []; + const args = []; + const batchEnd = Math.min(txEnd, offset + LEDGER_BATCH_ROWS); + for (let i = offset; i < batchEnd; i += 1) { + const accountId = `acct-${String(i % 256).padStart(4, "0")}`; + const entryId = Math.floor(i / 256) + 1; + placeholders.push("(?, ?, ?, ?, ?)"); + args.push( + accountId, + entryId, + (i % 2 === 0 ? 1 : -1) * (100 + i % 1e4), + 17e11 + i * 1e3, + payload(`ledger-${accountId}-${entryId}:`, Math.min(rowBytes, 512)) + ); + } + await database.execute( + `INSERT INTO rw_ledger (account_id, entry_id, amount_cents, created_at, memo) VALUES ${placeholders.join(", ")}`, + ...args + ); + } + }); + } + return { + rows, + targetBytes, + rowBytes, + setupMs: performance.now() - startedAt, + pageCount: await queryPageCount(database) + }; +} +function buildChatLogMessage2(seq, targetBytes) { + const prefix = `message-${seq}: `; + return prefix + "x".repeat(Math.max(0, targetBytes - prefix.length)); +} +async function seedChatLog2(database, targetBytes) { + await resetChatLog(database); + const createdAtBase = 17e11; + let remainingBytes = targetBytes; + let rows = 0; + const startedAt = performance.now(); + await withTransaction(database, async () => { + while (remainingBytes > 0) { + const placeholders = []; + const args = []; + for (let batchIndex = 0; batchIndex < CHAT_LOG_INSERT_BATCH_SIZE2 && remainingBytes > 0; batchIndex += 1) { + const contentBytes = Math.min(CHAT_LOG_CHUNK_BYTES2, remainingBytes); + const seq = rows; + const role = seq % 2 === 0 ? "user" : "assistant"; + placeholders.push("(?, ?, ?, ?, ?, ?, ?)"); + args.push( + CHAT_THREAD_ID, + seq, + role, + buildChatLogMessage2(seq, contentBytes), + contentBytes, + Math.ceil(contentBytes / 4), + createdAtBase + seq + ); + remainingBytes -= contentBytes; + rows += 1; + } + await database.execute( + `INSERT INTO rw_chat_log (thread_id, seq, role, content, content_bytes, token_estimate, created_at) VALUES ${placeholders.join(", ")}`, + ...args + ); + } + }); + return { + rows, + targetBytes, + rowBytes: CHAT_LOG_CHUNK_BYTES2, + setupMs: performance.now() - startedAt, + pageCount: await queryPageCount(database) + }; +} +async function batchInsert(database, sql, rows, batchSize) { + if (rows.length === 0) return; + const colsPerRow = rows[0]?.length ?? 0; + if (colsPerRow === 0) return; + const placeholder = `(${"?,".repeat(colsPerRow).slice(0, -1)})`; + for (let i = 0; i < rows.length; i += batchSize) { + const chunk = rows.slice(i, i + batchSize); + const values = new Array(chunk.length).fill(placeholder).join(","); + const args = []; + for (const row of chunk) args.push(...row); + await database.execute(`${sql} VALUES ${values}`, ...args); + } +} +async function seedSqlRush(database, targetBytes) { + await resetSqlRush(database); + const now = 17e11; + const startedAt = performance.now(); + await withTransaction(database, async () => { + const msgsRows = []; + for (let i = 0; i < SQL_RUSH_MSGS_COUNT; i += 1) { + msgsRows.push([ + i === 0 ? null : i, + i % 3 === 0 ? "user" : "assistant", + payload("msg:", 512), + 0, + now - (SQL_RUSH_MSGS_COUNT - i) * 1e3 + ]); + } + await batchInsert( + database, + "INSERT INTO msgs (parent, role, content, cancelled, created_at)", + msgsRows, + 50 + ); + const toolRefsRows = []; + for (let i = 0; i < SQL_RUSH_TOOL_REFS_COUNT; i += 1) { + toolRefsRows.push([ + i + 1, + `tool_${i % 20}`, + `call_${i}`, + i % 5 === 0 ? "pending" : "done" + ]); + } + await batchInsert( + database, + "INSERT INTO tool_refs (msg_id, tool_name, tool_call_id, status)", + toolRefsRows, + 100 + ); + const eventsRows = []; + for (let i = 0; i < SQL_RUSH_EVENTS_COUNT; i += 1) { + eventsRows.push([ + i + 1, + `event_${i % 8}`, + payload("event:", 256), + now - (SQL_RUSH_EVENTS_COUNT - i) * 100 + ]); + } + await batchInsert( + database, + "INSERT INTO events (seq, event_type, payload, created_at)", + eventsRows, + 100 + ); + const kvRows = []; + for (let i = 0; i < SQL_RUSH_KV_COUNT; i += 1) { + kvRows.push([`kv_${i}`, payload("kv:", 128), now]); + } + await batchInsert(database, "INSERT INTO kv (key, value, updated_at)", kvRows, 40); + const toolsRows = []; + for (let i = 0; i < SQL_RUSH_TOOLS_COUNT; i += 1) { + toolsRows.push(["exec-1", `tool_${i}`, payload("tool:", 1024), now]); + } + await batchInsert( + database, + "INSERT INTO tools (executor_id, name, spec, updated_at)", + toolsRows, + 41 + ); + const metaRows = []; + for (let i = 0; i < SQL_RUSH_META_COUNT; i += 1) { + metaRows.push([`key_${i}`, payload("meta:", 64)]); + } + await batchInsert(database, "INSERT INTO meta (key, value)", metaRows, 12); + }); + return { + rows: SQL_RUSH_MSGS_COUNT + SQL_RUSH_TOOL_REFS_COUNT + SQL_RUSH_EVENTS_COUNT + SQL_RUSH_KV_COUNT + SQL_RUSH_TOOLS_COUNT + SQL_RUSH_META_COUNT, + targetBytes, + rowBytes: 0, + setupMs: performance.now() - startedAt, + pageCount: await queryPageCount(database) + }; +} +async function seedMigrationSource(database, targetBytes, rowBytes, skewed = false) { + await resetMigration(database); + await database.execute(`CREATE TABLE rw_migration_source ( + id INTEGER PRIMARY KEY, + account_id TEXT NOT NULL, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + total_cents INTEGER NOT NULL, + body TEXT NOT NULL + )`); + const rows = Math.max(1, Math.ceil(targetBytes / rowBytes)); + const startedAt = performance.now(); + for (let txStart = 0; txStart < rows; txStart += SETUP_TRANSACTION_ROWS) { + const txEnd = Math.min(rows, txStart + SETUP_TRANSACTION_ROWS); + await withTransaction(database, async () => { + for (let offset = txStart; offset < txEnd; offset += ORDER_BATCH_ROWS) { + const placeholders = []; + const args = []; + const batchEnd = Math.min(txEnd, offset + ORDER_BATCH_ROWS); + for (let i = offset; i < batchEnd; i += 1) { + const accountId = skewed ? `acct-${i % 10 === 0 ? i % 512 : i % 8}` : `acct-${pseudoRandom(i) % 512}`; + const status = skewed ? i % 20 === 0 ? "failed" : "open" : ["open", "closed", "failed", "pending"][i % 4]; + placeholders.push("(?, ?, ?, ?, ?, ?)"); + args.push( + i + 1, + accountId, + status, + 17e11 + i * 1e3, + 100 + pseudoRandom(i + 41) % 5e4, + payload(`migration-${i}:`, rowBytes) + ); + } + await database.execute( + `INSERT INTO rw_migration_source (id, account_id, status, created_at, total_cents, body) VALUES ${placeholders.join(", ")}`, + ...args + ); + } + }); + } + return { + rows, + targetBytes, + rowBytes, + setupMs: performance.now() - startedAt, + pageCount: await queryPageCount(database) + }; +} +async function readRowidRange(database, direction) { + const [count] = typedRows( + await database.execute("SELECT COUNT(*) AS rows FROM rw_orders") + ); + const rows = count?.rows ?? 0; + let bytes = 0; + let scannedRows = 0; + if (direction === "backward") { + for (let upper = rows; upper > 0; upper -= RANGE_CHUNK_ROWS) { + const lower = Math.max(1, upper - RANGE_CHUNK_ROWS + 1); + const chunk = typedRows( + await database.execute( + `SELECT length(note) AS bytes FROM rw_orders WHERE id BETWEEN ? AND ? ORDER BY id DESC`, + lower, + upper + ) + ); + for (const row of chunk) { + bytes += row.bytes; + scannedRows += 1; + } + } + return { rows: scannedRows, bytes }; + } + for (let lower = 1; lower <= rows; lower += RANGE_CHUNK_ROWS) { + const upper = lower + RANGE_CHUNK_ROWS - 1; + const [chunk] = typedRows( + await database.execute( + `SELECT COUNT(*) AS rows, COALESCE(SUM(length(note)), 0) AS bytes FROM rw_orders WHERE id BETWEEN ? AND ?`, + lower, + upper + ) + ); + bytes += chunk?.bytes ?? 0; + scannedRows += chunk?.rows ?? 0; + } + return { rows: scannedRows, bytes }; +} +var sqliteRealworldBench = actor51({ + options: { + actionTimeout: 12e5, + sleepGracePeriod: 3e4 + }, + db: db8({ + onMigrate: async (database) => { + await database.execute(`CREATE TABLE IF NOT EXISTS rw_customers ( + id INTEGER PRIMARY KEY, + account_id TEXT NOT NULL, + email TEXT NOT NULL, + plan TEXT NOT NULL, + region TEXT NOT NULL + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS rw_orders ( + id INTEGER PRIMARY KEY, + customer_id INTEGER NOT NULL, + created_at INTEGER NOT NULL, + status TEXT NOT NULL, + total_cents INTEGER NOT NULL, + shard INTEGER NOT NULL, + note TEXT NOT NULL + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS rw_order_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id INTEGER NOT NULL, + sku TEXT NOT NULL, + quantity INTEGER NOT NULL, + price_cents INTEGER NOT NULL, + line_no INTEGER NOT NULL + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS rw_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + account_id TEXT NOT NULL, + event_type TEXT NOT NULL, + created_at INTEGER NOT NULL, + entity_key TEXT NOT NULL, + properties TEXT NOT NULL + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS rw_docs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + external_key TEXT NOT NULL UNIQUE, + row_rank INTEGER NOT NULL, + tenant_id TEXT NOT NULL, + body TEXT NOT NULL, + body_bytes INTEGER NOT NULL + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS rw_ledger ( + account_id TEXT NOT NULL, + entry_id INTEGER NOT NULL, + amount_cents INTEGER NOT NULL, + created_at INTEGER NOT NULL, + memo TEXT NOT NULL, + PRIMARY KEY (account_id, entry_id) + ) WITHOUT ROWID`); + await database.execute(`CREATE TABLE IF NOT EXISTS rw_chat_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + thread_id TEXT NOT NULL, + seq INTEGER NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + content_bytes INTEGER NOT NULL, + token_estimate INTEGER NOT NULL, + created_at INTEGER NOT NULL DEFAULT 0 + )`); + await database.execute( + "CREATE TABLE IF NOT EXISTS msgs (id INTEGER PRIMARY KEY AUTOINCREMENT, parent INTEGER, role TEXT NOT NULL, content TEXT NOT NULL, cancelled INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL)" + ); + await database.execute( + "CREATE TABLE IF NOT EXISTS tool_refs (id INTEGER PRIMARY KEY AUTOINCREMENT, msg_id INTEGER NOT NULL, tool_name TEXT NOT NULL, tool_call_id TEXT NOT NULL, status TEXT NOT NULL)" + ); + await database.execute( + "CREATE TABLE IF NOT EXISTS events (seq INTEGER PRIMARY KEY, event_type TEXT NOT NULL, payload TEXT NOT NULL, created_at INTEGER NOT NULL)" + ); + await database.execute( + "CREATE TABLE IF NOT EXISTS kv (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at INTEGER NOT NULL)" + ); + await database.execute( + "CREATE TABLE IF NOT EXISTS tools (id INTEGER PRIMARY KEY AUTOINCREMENT, executor_id TEXT NOT NULL, name TEXT NOT NULL, spec TEXT NOT NULL, updated_at INTEGER NOT NULL)" + ); + await database.execute( + "CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_rw_orders_customer_created ON rw_orders(customer_id, created_at DESC)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_rw_orders_status_created ON rw_orders(status, created_at)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_rw_orders_created ON rw_orders(created_at DESC)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_rw_order_items_order ON rw_order_items(order_id)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_rw_events_account_created ON rw_events(account_id, created_at)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_rw_docs_external_rank ON rw_docs(external_key, row_rank)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_rw_docs_tenant_rank ON rw_docs(tenant_id, row_rank)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_rw_chat_log_thread_seq ON rw_chat_log(thread_id, seq DESC)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_msgs_parent_role_cancelled_created_at ON msgs (parent, role, cancelled, created_at)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_tool_refs_msg_id ON tool_refs (msg_id)" + ); + } + }), + actions: { + inspectCacheConfig: async (c) => { + const [cacheSize] = typedRows( + await c.db.execute("PRAGMA cache_size") + ); + const [pageSize] = typedRows( + await c.db.execute("PRAGMA page_size") + ); + return { + sqliteCacheSizePragma: cacheSize?.cache_size ?? null, + sqlitePageSize: pageSize?.page_size ?? null, + pageCount: await queryPageCount(c.db) + }; + }, + setupWorkload: async (c, input) => { + assertWorkload(input.workload); + const rowBytes = positiveInteger2(input.rowBytes, DEFAULT_ROW_BYTES2, "rowBytes"); + if (input.workload === "migration-ddl-small") { + await resetMigration(c.db); + return { + rows: 0, + targetBytes: 0, + rowBytes, + setupMs: 0, + pageCount: await queryPageCount(c.db) + }; + } + const targetBytes = positiveInteger2( + input.targetBytes, + 8 * 1024 * 1024, + "targetBytes" + ); + switch (input.workload) { + case "small-rowid-point": + case "small-schema-read": + case "small-range-scan": + case "rowid-range-forward": + case "rowid-range-backward": + case "aggregate-status": + case "aggregate-time-bucket": + case "aggregate-tenant-time-range": + case "parallel-read-aggregates": + case "parallel-read-write-transition": + case "feed-order-by-limit": + case "feed-pagination-adjacent": + case "join-order-items": + case "random-point-lookups": + case "write-batch-after-wake": + case "update-hot-partition": + case "delete-churn-range-read": + return seedCommerce(c.db, targetBytes, rowBytes); + case "secondary-index-covering-range": + case "secondary-index-scattered-table": + case "hot-index-cold-table": + return seedDocs(c.db, targetBytes, rowBytes); + case "ledger-without-rowid-range": + return seedLedger(c.db, targetBytes, rowBytes); + case "chat-log-select-limit": + case "chat-log-select-indexed": + case "chat-log-count": + case "chat-log-sum": + case "chat-tool-read-fanout": + return seedChatLog2(c.db, targetBytes); + case "chat-tool-script": + return seedSqlRush(c.db, targetBytes); + case "migration-create-indexes-large": + return seedMigrationSource(c.db, targetBytes, rowBytes); + case "migration-create-indexes-skewed-large": + return seedMigrationSource(c.db, targetBytes, rowBytes, true); + case "migration-table-rebuild-large": + case "migration-add-column-large": + return seedMigrationSource(c.db, targetBytes, rowBytes); + } + }, + runWorkload: async (c, input) => { + assertWorkload(input.workload); + const t0 = performance.now(); + let details; + switch (input.workload) { + case "small-rowid-point": { + let bytes = 0; + for (let i = 0; i < 50; i += 1) { + const id = i % 16 + 1; + const [row] = typedRows( + await c.db.execute( + "SELECT length(note) AS bytes FROM rw_orders WHERE id = ?", + id + ) + ); + bytes += row?.bytes ?? 0; + } + details = { ops: 50, bytes }; + break; + } + case "small-schema-read": { + const tables = await c.db.execute( + "SELECT name, type FROM sqlite_master WHERE type IN ('table', 'index') ORDER BY name" + ); + const columns = await c.db.execute("PRAGMA table_info(rw_orders)"); + const [count] = typedRows( + await c.db.execute("SELECT COUNT(*) AS rows FROM rw_orders") + ); + details = { + objects: tables.length, + columns: columns.length, + rows: count?.rows ?? 0 + }; + break; + } + case "small-range-scan": + case "rowid-range-forward": { + details = await readRowidRange(c.db, "forward"); + break; + } + case "rowid-range-backward": { + details = await readRowidRange(c.db, "backward"); + break; + } + case "secondary-index-covering-range": { + const rows = typedRows( + await c.db.execute( + `SELECT external_key, row_rank FROM rw_docs + WHERE external_key BETWEEN 'doc-00000000' AND 'doc-ffffffff' + ORDER BY external_key` + ) + ); + let checksum2 = 0; + for (const row of rows) checksum2 = checksum2 + row.row_rank >>> 0; + details = { rows: rows.length, checksum: checksum2 }; + break; + } + case "secondary-index-scattered-table": { + const rows = typedRows( + await c.db.execute( + `SELECT body_bytes AS bytes FROM rw_docs + WHERE external_key BETWEEN 'doc-00000000' AND 'doc-ffffffff' + ORDER BY external_key` + ) + ); + let bytes = 0; + for (const row of rows) bytes += row.bytes; + details = { rows: rows.length, bytes }; + break; + } + case "aggregate-status": { + const rows = typedRows( + await c.db.execute( + `SELECT status, COUNT(*) AS rows, SUM(total_cents) AS total + FROM rw_orders + GROUP BY status + ORDER BY status` + ) + ); + details = { + groups: rows.length, + rows: rows.reduce((sum, row) => sum + row.rows, 0), + total: rows.reduce((sum, row) => sum + row.total, 0) + }; + break; + } + case "aggregate-time-bucket": { + const rows = typedRows( + await c.db.execute( + `SELECT (created_at / 300000) AS bucket, COUNT(*) AS rows, SUM(total_cents) AS total + FROM rw_orders + GROUP BY bucket + ORDER BY bucket` + ) + ); + details = { + buckets: rows.length, + rows: rows.reduce((sum, row) => sum + row.rows, 0), + total: rows.reduce((sum, row) => sum + row.total, 0) + }; + break; + } + case "aggregate-tenant-time-range": { + const rows = typedRows( + await c.db.execute( + `SELECT e.event_type, COUNT(*) AS rows, SUM(o.total_cents) AS total + FROM rw_events e + JOIN rw_orders o ON o.id = CAST(substr(e.entity_key, 7) AS INTEGER) + WHERE e.account_id = ? AND e.created_at BETWEEN ? AND ? + GROUP BY e.event_type + ORDER BY e.event_type`, + "acct-7", + 17e11, + 17e11 + 864e5 + ) + ); + details = { + groups: rows.length, + rows: rows.reduce((sum, row) => sum + row.rows, 0), + total: rows.reduce((sum, row) => sum + row.total, 0) + }; + break; + } + case "parallel-read-aggregates": { + const [ + statusRows, + bucketRows, + tenantRows, + joinRows + ] = await Promise.all([ + c.db.execute( + `SELECT status, COUNT(*) AS rows, SUM(total_cents) AS total + FROM rw_orders + GROUP BY status + ORDER BY status` + ), + c.db.execute( + `SELECT (created_at / 300000) AS bucket, COUNT(*) AS rows, SUM(total_cents) AS total + FROM rw_orders + GROUP BY bucket + ORDER BY bucket` + ), + c.db.execute( + `SELECT e.event_type, COUNT(*) AS rows, SUM(o.total_cents) AS total + FROM rw_events e + JOIN rw_orders o ON o.id = CAST(substr(e.entity_key, 7) AS INTEGER) + WHERE e.account_id = ? AND e.created_at BETWEEN ? AND ? + GROUP BY e.event_type + ORDER BY e.event_type`, + "acct-7", + 17e11, + 17e11 + 864e5 + ), + c.db.execute( + `SELECT o.status, COUNT(*) AS rows, SUM(oi.quantity * oi.price_cents) AS total + FROM rw_orders o + JOIN rw_order_items oi ON oi.order_id = o.id + GROUP BY o.status + ORDER BY o.status` + ) + ]); + const aggregates = [ + ...typedRows(statusRows), + ...typedRows(bucketRows), + ...typedRows(tenantRows), + ...typedRows(joinRows) + ]; + details = { + ops: 4, + groups: aggregates.length, + rows: aggregates.reduce((sum, row) => sum + row.rows, 0), + total: aggregates.reduce((sum, row) => sum + row.total, 0) + }; + break; + } + case "parallel-read-write-transition": { + const readStatus = c.db.execute( + `SELECT status, COUNT(*) AS rows, SUM(total_cents) AS total + FROM rw_orders + GROUP BY status + ORDER BY status` + ); + const readJoin = c.db.execute( + `SELECT o.status, COUNT(*) AS rows, SUM(oi.quantity * oi.price_cents) AS total + FROM rw_orders o + JOIN rw_order_items oi ON oi.order_id = o.id + GROUP BY o.status + ORDER BY o.status` + ); + const writeHotShard = c.db.execute( + "UPDATE rw_orders SET total_cents = total_cents + 1 WHERE shard BETWEEN 0 AND 7" + ); + const readAfterWrite = c.db.execute( + "SELECT COUNT(*) AS rows FROM rw_orders WHERE shard BETWEEN 0 AND 7" + ); + const [statusRows, joinRows, , shardRows] = await Promise.all([ + readStatus, + readJoin, + writeHotShard, + readAfterWrite + ]); + const aggregates = [ + ...typedRows(statusRows), + ...typedRows(joinRows) + ]; + const [shardCount] = typedRows(shardRows); + details = { + ops: 4, + readOps: 3, + writeOps: 1, + groups: aggregates.length, + rows: aggregates.reduce((sum, row) => sum + row.rows, 0) + (shardCount?.rows ?? 0), + total: aggregates.reduce((sum, row) => sum + row.total, 0) + }; + break; + } + case "feed-order-by-limit": { + const rows = await c.db.execute( + `SELECT id, customer_id, created_at, status, total_cents + FROM rw_orders + WHERE created_at >= ? + ORDER BY created_at DESC + LIMIT 1000`, + 17e11 + ); + details = { rows: rows.length }; + break; + } + case "feed-pagination-adjacent": { + const firstPage = typedRows( + await c.db.execute( + `SELECT created_at + FROM rw_orders + WHERE created_at >= ? + ORDER BY created_at DESC + LIMIT ?`, + 17e11, + FEED_PAGE_ROWS + ) + ); + const cursor = firstPage.at(-1)?.created_at ?? 17e11; + const secondPage = await c.db.execute( + `SELECT id, customer_id, created_at, status, total_cents + FROM rw_orders + WHERE created_at < ? + ORDER BY created_at DESC + LIMIT ?`, + cursor, + FEED_PAGE_ROWS + ); + details = { firstPageRows: firstPage.length, rows: secondPage.length }; + break; + } + case "join-order-items": { + const rows = typedRows( + await c.db.execute( + `SELECT o.status, COUNT(*) AS rows, SUM(oi.quantity * oi.price_cents) AS total + FROM rw_orders o + JOIN rw_order_items oi ON oi.order_id = o.id + GROUP BY o.status + ORDER BY o.status` + ) + ); + details = { + groups: rows.length, + rows: rows.reduce((sum, row) => sum + row.rows, 0), + total: rows.reduce((sum, row) => sum + row.total, 0) + }; + break; + } + case "random-point-lookups": { + const [count] = typedRows( + await c.db.execute("SELECT COUNT(*) AS rows FROM rw_orders") + ); + const rows = Math.max(1, count?.rows ?? 1); + let bytes = 0; + for (let i = 0; i < POINT_LOOKUP_OPS; i += 1) { + const id = pseudoRandom(i) % rows + 1; + const [row] = typedRows( + await c.db.execute( + "SELECT length(note) AS bytes FROM rw_orders WHERE id = ?", + id + ) + ); + bytes += row?.bytes ?? 0; + } + details = { ops: POINT_LOOKUP_OPS, bytes }; + break; + } + case "hot-index-cold-table": { + const indexRows = typedRows( + await c.db.execute( + `SELECT id + FROM rw_docs + WHERE tenant_id = ? + ORDER BY row_rank + LIMIT 1000`, + "tenant-7" + ) + ); + let bytes = 0; + for (const row of indexRows) { + const [doc] = typedRows( + await c.db.execute( + "SELECT body_bytes AS bytes FROM rw_docs WHERE id = ?", + row.id + ) + ); + bytes += doc?.bytes ?? 0; + } + details = { rows: indexRows.length, bytes }; + break; + } + case "ledger-without-rowid-range": { + const rows = typedRows( + await c.db.execute( + `SELECT account_id, entry_id, amount_cents, length(memo) AS bytes + FROM rw_ledger + WHERE account_id BETWEEN 'acct-0040' AND 'acct-0180' + ORDER BY account_id, entry_id` + ) + ); + let bytes = 0; + for (const row of rows) bytes += row.bytes; + details = { rows: rows.length, bytes }; + break; + } + case "chat-log-select-limit": { + const rows = await c.db.execute( + "SELECT seq, role, substr(content, 1, 128) AS preview FROM rw_chat_log ORDER BY created_at DESC LIMIT 100" + ); + details = { rows: rows.length }; + break; + } + case "chat-log-select-indexed": { + const expectedRows = Math.max( + 1, + Math.ceil( + positiveInteger2(input.targetBytes, CHAT_LOG_CHUNK_BYTES2, "targetBytes") / CHAT_LOG_CHUNK_BYTES2 + ) + ); + const lowerBound = Math.max(0, expectedRows - 100); + const rows = await c.db.execute( + "SELECT seq, role, content_bytes FROM rw_chat_log WHERE thread_id = ? AND seq >= ? ORDER BY seq DESC LIMIT 100", + CHAT_THREAD_ID, + lowerBound + ); + details = { rows: rows.length }; + break; + } + case "chat-log-count": { + const [row] = typedRows( + await c.db.execute( + "SELECT COUNT(*) AS count FROM rw_chat_log WHERE thread_id = ?", + CHAT_THREAD_ID + ) + ); + details = { ops: 1, rows: row?.count ?? 0 }; + break; + } + case "chat-log-sum": { + const [row] = typedRows( + await c.db.execute( + "SELECT SUM(content_bytes) AS total_bytes FROM rw_chat_log WHERE thread_id = ?", + CHAT_THREAD_ID + ) + ); + details = { ops: 1, bytes: row?.total_bytes ?? 0 }; + break; + } + case "chat-tool-read-fanout": { + const expectedRows = Math.max( + 1, + Math.ceil( + positiveInteger2(input.targetBytes, CHAT_LOG_CHUNK_BYTES2, "targetBytes") / CHAT_LOG_CHUNK_BYTES2 + ) + ); + const lowerBound = Math.max(0, expectedRows - 100); + const [limitRows, indexedRows, countRows, sumRows] = await Promise.all([ + c.db.execute( + "SELECT seq, role, substr(content, 1, 128) AS preview FROM rw_chat_log ORDER BY created_at DESC LIMIT 100" + ), + c.db.execute( + "SELECT seq, role, content_bytes FROM rw_chat_log WHERE thread_id = ? AND seq >= ? ORDER BY seq DESC LIMIT 100", + CHAT_THREAD_ID, + lowerBound + ), + c.db.execute( + "SELECT COUNT(*) AS count FROM rw_chat_log WHERE thread_id = ?", + CHAT_THREAD_ID + ), + c.db.execute( + "SELECT SUM(content_bytes) AS total_bytes FROM rw_chat_log WHERE thread_id = ?", + CHAT_THREAD_ID + ) + ]); + const [countRow] = typedRows(countRows); + const [sumRow] = typedRows(sumRows); + details = { + ops: 4, + limitRows: limitRows.length, + indexedRows: indexedRows.length, + rows: countRow?.count ?? 0, + bytes: sumRow?.total_bytes ?? 0 + }; + break; + } + case "chat-tool-script": { + const [ + msgsRows, + toolRefsRows, + eventsRows, + kvRows, + toolsRows, + metaRows, + unresolvedRows + ] = await Promise.all([ + c.db.execute( + "SELECT id, role, length(content) AS bytes FROM msgs WHERE parent IS NOT NULL AND role = ? AND cancelled = 0 ORDER BY created_at DESC LIMIT 50", + "assistant" + ), + c.db.execute( + "SELECT id, tool_name, status FROM tool_refs WHERE status = ? ORDER BY id DESC LIMIT 50", + "pending" + ), + c.db.execute( + "SELECT seq, event_type, length(payload) AS bytes FROM events WHERE seq > ? ORDER BY seq ASC LIMIT 100", + 600 + ), + c.db.execute( + "SELECT key, length(value) AS bytes FROM kv ORDER BY updated_at DESC LIMIT 20" + ), + c.db.execute( + "SELECT id, name, length(spec) AS bytes FROM tools WHERE executor_id = ? ORDER BY updated_at DESC", + "exec-1" + ), + c.db.execute("SELECT key, length(value) AS bytes FROM meta"), + c.db.execute(`SELECT m.id, m.role, count(tr.id) AS pending_refs + FROM msgs m + LEFT JOIN tool_refs tr ON tr.msg_id = m.id AND tr.status = 'pending' + WHERE m.role = 'assistant' AND m.cancelled = 0 + GROUP BY m.id + ORDER BY m.created_at DESC + LIMIT 100`) + ]); + details = { + ops: 7, + msgsRows: msgsRows.length, + toolRefsRows: toolRefsRows.length, + eventsRows: eventsRows.length, + kvRows: kvRows.length, + toolsRows: toolsRows.length, + metaRows: metaRows.length, + unresolvedRows: unresolvedRows.length + }; + break; + } + case "write-batch-after-wake": { + const [count] = typedRows( + await c.db.execute("SELECT COUNT(*) AS rows FROM rw_orders") + ); + const startId = (count?.rows ?? 0) + 1; + await c.db.execute("BEGIN"); + for (let offset = 0; offset < 1e3; offset += ORDER_BATCH_ROWS) { + const placeholders = []; + const args = []; + for (let i = offset; i < offset + ORDER_BATCH_ROWS; i += 1) { + const id = startId + i; + placeholders.push("(?, ?, ?, ?, ?, ?, ?)"); + args.push( + id, + i % 128 + 1, + 18e11 + i, + "pending", + 1e3 + i, + i % 128, + payload(`wake-insert-${id}:`, DEFAULT_ROW_BYTES2) + ); + } + await c.db.execute( + `INSERT INTO rw_orders (id, customer_id, created_at, status, total_cents, shard, note) VALUES ${placeholders.join(", ")}`, + ...args + ); + } + await c.db.execute("COMMIT"); + details = { rows: 1e3 }; + break; + } + case "update-hot-partition": { + await c.db.execute( + "UPDATE rw_orders SET total_cents = total_cents + 1 WHERE shard BETWEEN 0 AND 15" + ); + const [count] = typedRows( + await c.db.execute( + "SELECT COUNT(*) AS rows FROM rw_orders WHERE shard BETWEEN 0 AND 15" + ) + ); + details = { rows: count?.rows ?? 0 }; + break; + } + case "delete-churn-range-read": { + await c.db.execute("DELETE FROM rw_orders WHERE shard BETWEEN 0 AND 15"); + const result = await readRowidRange(c.db, "forward"); + details = { + ...result, + deletedShardCount: 16 + }; + break; + } + case "migration-create-indexes-large": { + await c.db.execute( + "CREATE INDEX idx_rw_migration_source_account ON rw_migration_source(account_id)" + ); + await c.db.execute( + "CREATE INDEX idx_rw_migration_source_created ON rw_migration_source(created_at)" + ); + await c.db.execute( + "CREATE INDEX idx_rw_migration_source_status_total ON rw_migration_source(status, total_cents)" + ); + details = { indexes: 3 }; + break; + } + case "migration-create-indexes-skewed-large": { + await c.db.execute( + "CREATE INDEX idx_rw_migration_source_skew_account ON rw_migration_source(account_id, created_at)" + ); + await c.db.execute( + "CREATE INDEX idx_rw_migration_source_skew_status ON rw_migration_source(status, total_cents)" + ); + details = { indexes: 2, skewed: true }; + break; + } + case "migration-table-rebuild-large": { + await c.db.execute(`CREATE TABLE rw_migration_source_rebuilt ( + id INTEGER PRIMARY KEY, + account_id TEXT NOT NULL, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + total_cents INTEGER NOT NULL, + body TEXT NOT NULL, + archived_at INTEGER + )`); + await c.db.execute(`INSERT INTO rw_migration_source_rebuilt ( + id, account_id, status, created_at, total_cents, body, archived_at + ) + SELECT id, account_id, status, created_at, total_cents, body, NULL + FROM rw_migration_source`); + await c.db.execute("DROP TABLE rw_migration_source"); + await c.db.execute( + "ALTER TABLE rw_migration_source_rebuilt RENAME TO rw_migration_source" + ); + details = { rebuilt: true }; + break; + } + case "migration-add-column-large": { + await c.db.execute( + "ALTER TABLE rw_migration_source ADD COLUMN archived_at INTEGER" + ); + details = { alters: 1, rewritesRows: false }; + break; + } + case "migration-ddl-small": { + await c.db.execute(`CREATE TABLE rw_migration_empty ( + id INTEGER PRIMARY KEY, + tenant_id TEXT NOT NULL, + created_at INTEGER NOT NULL + )`); + await c.db.execute("ALTER TABLE rw_migration_empty ADD COLUMN status TEXT"); + await c.db.execute( + "CREATE INDEX idx_rw_migration_empty_tenant_created ON rw_migration_empty(tenant_id, created_at)" + ); + await c.db.execute(`CREATE TABLE rw_migration_audit ( + id INTEGER PRIMARY KEY, + migration_name TEXT NOT NULL, + applied_at INTEGER NOT NULL + )`); + details = { tables: 2, indexes: 1, alters: 1 }; + break; + } + } + const ms = performance.now() - t0; + return { + ms, + workload: input.workload, + ...details, + pageCount: await queryPageCount(c.db) + }; + }, + goToSleep: (c) => { + c.sleep(); + return { ok: true }; + } + } +}); + +// src/actors/testing/raw-sqlite-fuzzer.ts +import { actor as actor52 } from "rivetkit"; +import { db as db9 } from "rivetkit/db"; +var ACCOUNT_COUNT = 8; +var ACCOUNT_INITIAL_BALANCE = 1e5; +var DEFAULT_KEY_SPACE = 64; +var DEFAULT_MAX_PAYLOAD_BYTES = 8 * 1024; +var DEFAULT_GROWTH_TARGET_BYTES = 1024 * 1024; +var LARGE_WRITE_CHUNK_BYTES = 96 * 1024; +var PAGE_BOUNDARY_SIZES = [ + 1, + 4095, + 4096, + 4097, + 8191, + 8192, + 8193, + 32768, + 65535, + 65536, + 98304, + 131072 +]; +function hashSeed(input) { + let hash = 2166136261; + for (let i = 0; i < input.length; i += 1) { + hash ^= input.charCodeAt(i); + hash = Math.imul(hash, 16777619); + } + return hash >>> 0; +} +function makeRng(seed) { + let state = hashSeed(seed) || 2654435769; + return () => { + state = state + 1831565813 >>> 0; + let t = state; + t = Math.imul(t ^ t >>> 15, t | 1); + t ^= t + Math.imul(t ^ t >>> 7, t | 61); + return ((t ^ t >>> 14) >>> 0) / 4294967296; + }; +} +function intBetween(rng, min, max) { + return min + Math.floor(rng() * (max - min + 1)); +} +function checksum(input) { + let hash = 2166136261; + for (let i = 0; i < input.length; i += 1) { + hash ^= input.charCodeAt(i); + hash = Math.imul(hash, 16777619); + } + return hash >>> 0; +} +function payloadFor(seed, phase, index, bytes) { + const prefix = `${seed}:${phase}:${index}:`; + if (bytes <= prefix.length) return prefix.slice(0, bytes); + return prefix + "x".repeat(bytes - prefix.length); +} +async function queryOne(database, sql, ...args) { + const rows = await database.execute(sql, ...args); + return rows[0]; +} +async function transaction(database, fn) { + await database.execute("BEGIN"); + try { + const result = await fn(); + await database.execute("COMMIT"); + return result; + } catch (err) { + await database.execute("ROLLBACK").catch(() => void 0); + throw err; + } +} +async function recordProbe(database, phase, scenario, name, expected, actual, mismatch) { + await database.execute( + `INSERT INTO fuzz_probe_results ( + phase, scenario, name, expected, actual, mismatch, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + phase, + scenario, + name, + String(expected), + String(actual), + mismatch ? 1 : 0, + Date.now() + ); +} +function firstColumn(row) { + if (!row || typeof row !== "object") return void 0; + const values = Object.values(row); + return values[0]; +} +async function ensureAccounts(database) { + await database.execute("BEGIN"); + try { + for (let i = 0; i < ACCOUNT_COUNT; i += 1) { + await database.execute( + "INSERT OR IGNORE INTO fuzz_accounts (id, balance) VALUES (?, ?)", + `acct-${i}`, + ACCOUNT_INITIAL_BALANCE + ); + } + await database.execute("COMMIT"); + } catch (err) { + await database.execute("ROLLBACK").catch(() => void 0); + throw err; + } +} +async function recordItemEvent(database, phase, localIndex, kind, itemKey, present, value, version, updateCount, payload2, applied) { + await database.execute( + `INSERT INTO fuzz_item_events ( + phase, local_index, kind, item_key, present, value, version, + update_count, payload_checksum, payload_bytes, applied, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + phase, + localIndex, + kind, + itemKey, + present ? 1 : 0, + value, + version, + updateCount, + checksum(payload2), + payload2.length, + applied ? 1 : 0, + Date.now() + ); +} +async function upsertLiveItem(database, row, payload2) { + await database.execute( + `INSERT INTO fuzz_items ( + item_key, value, version, update_count, payload, payload_checksum, + payload_bytes, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(item_key) DO UPDATE SET + value = excluded.value, + version = excluded.version, + update_count = excluded.update_count, + payload = excluded.payload, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes, + updated_at = excluded.updated_at`, + row.item_key, + row.value, + row.version, + row.update_count, + payload2, + row.payload_checksum, + row.payload_bytes, + Date.now() + ); +} +async function applyItemOperation(database, opts) { + let current; + try { + current = await queryOne( + database, + "SELECT item_key, value, version, update_count, payload, payload_checksum, payload_bytes FROM fuzz_items WHERE item_key = ?", + opts.itemKey + ); + } catch (error) { + throw new Error( + `item operation select failed for kind ${opts.kind} key ${JSON.stringify(opts.itemKey)} payloadBytes ${opts.payloadBytes}`, + { cause: error } + ); + } + const payload2 = payloadFor( + opts.seed, + opts.phase, + opts.localIndex, + opts.payloadBytes + ); + const nextVersion = (current?.version ?? 0) + 1; + const nextUpdateCount = (current?.update_count ?? 0) + 1; + const nextValue = `${opts.kind}:${opts.phase}:${opts.localIndex}:${nextVersion}`; + if (opts.kind === "delete") { + try { + await recordItemEvent( + database, + opts.phase, + opts.localIndex, + opts.kind, + opts.itemKey, + false, + null, + nextVersion, + nextUpdateCount, + "", + current !== void 0 + ); + } catch (error) { + throw new Error(`item operation event insert failed for delete key ${JSON.stringify(opts.itemKey)}`, { + cause: error + }); + } + try { + await database.execute("DELETE FROM fuzz_items WHERE item_key = ?", opts.itemKey); + } catch (error) { + throw new Error(`item operation delete failed for key ${JSON.stringify(opts.itemKey)}`, { + cause: error + }); + } + return; + } + if (opts.kind === "insert" && current) { + try { + await recordItemEvent( + database, + opts.phase, + opts.localIndex, + opts.kind, + opts.itemKey, + true, + current.value, + current.version, + current.update_count, + current.payload ?? "", + false + ); + } catch (error) { + throw new Error( + `item operation event insert failed for noop insert key ${JSON.stringify(opts.itemKey)}`, + { cause: error } + ); + } + return; + } + const row = { + item_key: opts.itemKey, + value: nextValue, + version: nextVersion, + update_count: nextUpdateCount, + payload_checksum: checksum(payload2), + payload_bytes: payload2.length + }; + try { + await recordItemEvent( + database, + opts.phase, + opts.localIndex, + opts.kind, + opts.itemKey, + true, + row.value, + row.version, + row.update_count, + payload2, + true + ); + } catch (error) { + throw new Error( + `item operation event insert failed for kind ${opts.kind} key ${JSON.stringify(opts.itemKey)} payloadBytes ${opts.payloadBytes}`, + { cause: error } + ); + } + try { + await upsertLiveItem(database, row, payload2); + } catch (error) { + throw new Error( + `item operation live-row upsert failed for kind ${opts.kind} key ${JSON.stringify(opts.itemKey)} payloadBytes ${opts.payloadBytes}`, + { cause: error } + ); + } +} +async function applyHotUpdates(database, opts) { + for (let i = 0; i < opts.updates; i += 1) { + try { + await applyItemOperation(database, { + seed: opts.seed, + phase: opts.phase, + localIndex: opts.localIndex * 1e3 + i, + kind: "update", + itemKey: opts.itemKey, + payloadBytes: opts.payloadBytes + }); + } catch (error) { + throw new Error( + `hot update failed for ${opts.itemKey} at sub-update ${i + 1}/${opts.updates} with payloadBytes ${opts.payloadBytes}`, + { cause: error } + ); + } + } +} +async function applyTransfer(database, opts) { + await transaction(database, async () => { + const before = await queryOne( + database, + "SELECT COALESCE(SUM(balance), 0) AS total FROM fuzz_accounts" + ); + await database.execute( + "UPDATE fuzz_accounts SET balance = balance - ? WHERE id = ?", + opts.amount, + opts.fromAccount + ); + await database.execute( + "UPDATE fuzz_accounts SET balance = balance + ? WHERE id = ?", + opts.amount, + opts.toAccount + ); + const after = await queryOne( + database, + "SELECT COALESCE(SUM(balance), 0) AS total FROM fuzz_accounts" + ); + await database.execute( + `INSERT INTO fuzz_transfer_events ( + phase, local_index, from_account, to_account, amount, + balance_sum_before, balance_sum_after, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + opts.phase, + opts.localIndex, + opts.fromAccount, + opts.toAccount, + opts.amount, + before?.total ?? 0, + after?.total ?? 0, + Date.now() + ); + }); +} +async function applyEdgePayloads(database, opts) { + const writeEdgePayload = async (id, kind, payload2, sizeLabel) => { + const payloadChecksum = checksum(payload2); + const payloadBytes = payload2.length; + try { + await database.execute("BEGIN"); + } catch (error) { + throw new Error(`edge payload begin failed for ${sizeLabel}`, { + cause: error + }); + } + try { + try { + await database.execute( + `INSERT INTO fuzz_edge_payloads ( + id, kind, payload, payload_checksum, payload_bytes, updated_at + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + kind = excluded.kind, + payload = excluded.payload, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes, + updated_at = excluded.updated_at`, + id, + kind, + payload2, + payloadChecksum, + payloadBytes, + Date.now() + ); + } catch (error) { + throw new Error(`edge payload row upsert failed for ${sizeLabel}`, { + cause: error + }); + } + try { + await database.execute( + `INSERT INTO fuzz_edge_expectations ( + id, present, payload_checksum, payload_bytes + ) VALUES (?, 1, ?, ?) + ON CONFLICT(id) DO UPDATE SET + present = excluded.present, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes`, + id, + payloadChecksum, + payloadBytes + ); + } catch (error) { + throw new Error(`edge payload expectation upsert failed for ${sizeLabel}`, { + cause: error + }); + } + try { + await database.execute("COMMIT"); + } catch (error) { + throw new Error(`edge payload commit failed for ${sizeLabel}`, { + cause: error + }); + } + } catch (error) { + await database.execute("ROLLBACK").catch(() => void 0); + throw error; + } + }; + const sizes = PAGE_BOUNDARY_SIZES.filter((size) => size <= opts.maxPayloadBytes); + if (!sizes.includes(opts.maxPayloadBytes)) sizes.push(opts.maxPayloadBytes); + let ops = 0; + for (const size of sizes) { + const id = `edge-${opts.phase}-${size}`; + const payload2 = payloadFor(opts.seed, opts.phase, size, size); + try { + await writeEdgePayload(id, "boundary", payload2, `size ${size}`); + } catch (error) { + throw new Error(`edge payload write failed for size ${size}`, { + cause: error + }); + } + ops += 1; + } + const unicodePayload = `escaped-nul:\\0 unicode:\u2603\uFE0F phase:${opts.phase} seed:${opts.seed}`; + const unicodeId = `edge-${opts.phase}-unicode-nul`; + try { + await writeEdgePayload( + unicodeId, + "unicode-nul", + unicodePayload, + "unicode escaped-nul payload" + ); + } catch (error) { + throw new Error("edge payload write failed for unicode escaped-nul payload", { + cause: error + }); + } + return ops + 1; +} +async function applyActualNulPayload(database, opts) { + const payload2 = `actual-nul:\0 phase:${opts.phase} seed:${opts.seed}`; + const id = `actual-nul-${opts.phase}`; + await transaction(database, async () => { + await database.execute( + `INSERT INTO fuzz_edge_payloads ( + id, kind, payload, payload_checksum, payload_bytes, updated_at + ) VALUES (?, 'actual-nul', ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + payload = excluded.payload, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes, + updated_at = excluded.updated_at`, + id, + payload2, + checksum(payload2), + payload2.length, + Date.now() + ); + await database.execute( + `INSERT INTO fuzz_edge_expectations ( + id, present, payload_checksum, payload_bytes + ) VALUES (?, 1, ?, ?) + ON CONFLICT(id) DO UPDATE SET + present = 1, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes`, + id, + checksum(payload2), + payload2.length + ); + }); + return 1; +} +async function applyFragmentationChurn(database, opts) { + const rows = Math.max(12, Math.floor(opts.iterations / 2)); + let ops = 0; + for (let i = 0; i < rows; i += 1) { + const size = intBetween(opts.rng, 32, Math.max(32, opts.maxPayloadBytes)); + const id = `frag-${opts.phase}-${i}`; + const payload2 = payloadFor(opts.seed, opts.phase, 1e4 + i, size); + try { + await database.execute( + `INSERT INTO fuzz_edge_payloads ( + id, kind, payload, payload_checksum, payload_bytes, updated_at + ) VALUES (?, 'fragment', ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + payload = excluded.payload, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes, + updated_at = excluded.updated_at`, + id, + payload2, + checksum(payload2), + payload2.length, + Date.now() + ); + } catch (error) { + throw new Error(`fragmentation payload upsert failed for ${id} at size ${size}`, { + cause: error + }); + } + try { + await database.execute( + `INSERT INTO fuzz_edge_expectations ( + id, present, payload_checksum, payload_bytes + ) VALUES (?, 1, ?, ?) + ON CONFLICT(id) DO UPDATE SET + present = 1, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes`, + id, + checksum(payload2), + payload2.length + ); + } catch (error) { + throw new Error(`fragmentation expectation upsert failed for ${id} at size ${size}`, { + cause: error + }); + } + ops += 1; + } + for (let i = 0; i < rows; i += 3) { + const id = `frag-${opts.phase}-${i}`; + try { + await database.execute("DELETE FROM fuzz_edge_payloads WHERE id = ?", id); + } catch (error) { + throw new Error(`fragmentation delete failed for ${id}`, { + cause: error + }); + } + try { + await database.execute( + `INSERT INTO fuzz_edge_expectations ( + id, present, payload_checksum, payload_bytes + ) VALUES (?, 0, 0, 0) + ON CONFLICT(id) DO UPDATE SET + present = 0, + payload_checksum = 0, + payload_bytes = 0`, + id + ); + } catch (error) { + throw new Error(`fragmentation tombstone expectation failed for ${id}`, { + cause: error + }); + } + ops += 1; + } + for (let i = 1; i < rows; i += 4) { + const size = intBetween(opts.rng, 1, Math.max(1, opts.maxPayloadBytes)); + const id = `frag-${opts.phase}-${i}`; + const payload2 = payloadFor(opts.seed, opts.phase, 2e4 + i, size); + try { + await database.execute( + `UPDATE fuzz_edge_payloads + SET payload = ?, payload_checksum = ?, payload_bytes = ?, updated_at = ? + WHERE id = ?`, + payload2, + checksum(payload2), + payload2.length, + Date.now(), + id + ); + } catch (error) { + throw new Error(`fragmentation payload rewrite failed for ${id} at size ${size}`, { + cause: error + }); + } + try { + await database.execute( + `UPDATE fuzz_edge_expectations + SET payload_checksum = ?, payload_bytes = ? + WHERE id = ? AND present = 1`, + checksum(payload2), + payload2.length, + id + ); + } catch (error) { + throw new Error(`fragmentation expectation rewrite failed for ${id} at size ${size}`, { + cause: error + }); + } + ops += 1; + } + if (opts.phase % 2 === 1) { + try { + await database.execute("VACUUM"); + } catch (error) { + throw new Error(`fragmentation vacuum failed for phase ${opts.phase}`, { + cause: error + }); + } + ops += 1; + } + return ops; +} +async function applySchemaChurn(database, phase) { + const table = `fuzz_schema_phase_${phase}`; + const index = `idx_fuzz_schema_phase_${phase}_name`; + const view = `view_fuzz_schema_phase_${phase}`; + const dropIndex = `idx_fuzz_schema_drop_probe_${phase}`; + await database.execute( + `CREATE TABLE IF NOT EXISTS ${table} ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + value INTEGER NOT NULL DEFAULT 0, + extra TEXT + )` + ); + await database.execute(`CREATE INDEX IF NOT EXISTS ${index} ON ${table}(name, value)`); + try { + await database.execute(`ALTER TABLE ${table} ADD COLUMN altered_${phase} TEXT DEFAULT 'altered'`); + } catch { + const column = await queryOne( + database, + `SELECT COUNT(*) AS count FROM pragma_table_info('${table}') WHERE name = ?`, + `altered_${phase}` + ); + if ((column?.count ?? 0) !== 1) throw new Error(`failed to add altered_${phase}`); + } + await database.execute(`CREATE VIEW IF NOT EXISTS ${view} AS SELECT id, name, value FROM ${table}`); + await database.execute( + `INSERT INTO ${table} (name, value, extra) + VALUES (?, ?, ?) + ON CONFLICT(name) DO UPDATE SET + value = excluded.value, + extra = excluded.extra`, + `schema-${phase}`, + phase, + `extra-${phase}` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS fuzz_without_rowid ( + id TEXT PRIMARY KEY, + value INTEGER NOT NULL + ) WITHOUT ROWID` + ); + await database.execute(` + CREATE TRIGGER IF NOT EXISTS trg_fuzz_edge_payload_update + AFTER UPDATE ON fuzz_edge_payloads + BEGIN + INSERT INTO fuzz_trigger_audit ( + payload_id, old_checksum, new_checksum, created_at + ) VALUES ( + new.id, old.payload_checksum, new.payload_checksum, strftime('%s', 'now') * 1000 + ); + END + `); + await database.execute( + `INSERT INTO fuzz_without_rowid (id, value) + VALUES (?, ?) + ON CONFLICT(id) DO UPDATE SET value = excluded.value`, + `phase-${phase}`, + phase + ); + for (const [name, type] of [ + [table, "table"], + [index, "index"], + [view, "view"], + ["trg_fuzz_edge_payload_update", "trigger"], + ["fuzz_without_rowid", "table"] + ]) { + await database.execute( + `INSERT INTO fuzz_schema_registry (name, type) + VALUES (?, ?) + ON CONFLICT(name) DO UPDATE SET type = excluded.type`, + name, + type + ); + } + await database.execute("CREATE TEMP TABLE IF NOT EXISTS fuzz_temp_probe (id INTEGER PRIMARY KEY, value TEXT)"); + await database.execute("INSERT INTO fuzz_temp_probe (value) VALUES (?)", `temp-${phase}`); + await database.execute("DROP TABLE fuzz_temp_probe"); + await database.execute(`CREATE INDEX IF NOT EXISTS ${dropIndex} ON fuzz_schema_registry(type)`); + await database.execute(`DROP INDEX IF EXISTS ${dropIndex}`); + const dropped = await queryOne( + database, + "SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'index' AND name = ?", + dropIndex + ); + await recordProbe( + database, + phase, + "schema", + "drop-index", + 0, + dropped?.count ?? -1, + (dropped?.count ?? -1) !== 0 + ); + return 13; +} +async function applyIndexProbe(database, opts) { + const rows = Math.max(20, opts.iterations); + await transaction(database, async () => { + for (let i = 0; i < rows; i += 1) { + const tenant = `tenant-${intBetween(opts.rng, 0, 5)}`; + const bucket = intBetween(opts.rng, 0, 12); + const score = intBetween(opts.rng, -500, 500); + const label = `${opts.seed}:${opts.phase}:${i}`; + await database.execute( + `INSERT INTO fuzz_indexed (tenant, bucket, score, label, payload) + VALUES (?, ?, ?, ?, ?)`, + tenant, + bucket, + score, + label, + payloadFor(opts.seed, opts.phase, 3e4 + i, intBetween(opts.rng, 8, 256)) + ); + } + }); + return rows; +} +async function applyPreparedChurn(database, opts) { + const rows = Math.max(32, opts.iterations); + for (let i = 0; i < rows; i += 1) { + const id = `prep-${opts.phase}-${i}`; + const payload2 = payloadFor( + opts.seed, + opts.phase, + 7e4 + i, + Math.min(opts.maxPayloadBytes, 64 + i % 257) + ); + await database.execute( + `INSERT INTO fuzz_prepared_churn (id, value, payload, payload_checksum) + VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + value = excluded.value, + payload = excluded.payload, + payload_checksum = excluded.payload_checksum + /* unique-prepared-${opts.phase}-${i} */`, + id, + i, + payload2, + checksum(payload2) + ); + await database.execute( + `INSERT INTO fuzz_prepared_expectations (id, value, payload_checksum) + VALUES (?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + value = excluded.value, + payload_checksum = excluded.payload_checksum`, + id, + i, + checksum(payload2) + ); + } + const repeatedId = `prep-repeat-${opts.phase}`; + await database.execute( + `INSERT INTO fuzz_prepared_churn (id, value, payload, payload_checksum) + VALUES (?, 0, '', 0) + ON CONFLICT(id) DO UPDATE SET value = 0, payload = '', payload_checksum = 0`, + repeatedId + ); + for (let i = 0; i < rows; i += 1) { + const payload2 = payloadFor(opts.seed, opts.phase, 8e4 + i, Math.min(512, opts.maxPayloadBytes)); + await database.execute( + `UPDATE fuzz_prepared_churn + SET value = value + ?, payload = ?, payload_checksum = ? + WHERE id = ?`, + 1, + payload2, + checksum(payload2), + repeatedId + ); + } + const finalPayload = payloadFor(opts.seed, opts.phase, 8e4 + rows - 1, Math.min(512, opts.maxPayloadBytes)); + await database.execute( + `INSERT INTO fuzz_prepared_expectations (id, value, payload_checksum) + VALUES (?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + value = excluded.value, + payload_checksum = excluded.payload_checksum`, + repeatedId, + rows, + checksum(finalPayload) + ); + return rows * 2 + 1; +} +async function applyReadWriteProbe(database, opts) { + if ((await queryOne(database, "SELECT COUNT(*) AS count FROM fuzz_indexed"))?.count === 0) { + await applyIndexProbe(database, opts); + } + const read = database.execute( + `SELECT + COUNT(*) AS joined_rows, + COALESCE(SUM(a.score + b.score), 0) AS score_sum + FROM fuzz_indexed a + JOIN fuzz_indexed b ON b.bucket = a.bucket + WHERE a.tenant <= 'tenant-3'` + ); + const write = applyIndexProbe(database, { + seed: opts.seed, + phase: opts.phase, + rng: opts.rng, + iterations: Math.max(10, Math.floor(opts.iterations / 2)) + }); + const [readRows, writeOps] = await Promise.all([read, write]); + const row = readRows[0]; + const joinedRows = Number(row?.joined_rows ?? -1); + const scoreSum = Number(row?.score_sum ?? Number.NaN); + await recordProbe( + database, + opts.phase, + "readwrite", + "long-read-while-write", + "nonnegative-finite", + `${joinedRows}:${scoreSum}`, + joinedRows < 0 || !Number.isFinite(scoreSum) + ); + return writeOps + 1; +} +async function applyBoundaryKeys(database, opts) { + const keys = [ + "", + " ", + `long-${"k".repeat(2048)}`, + "slash/key", + "comma,key", + "percent%key", + "CaseKey", + "casekey" + ]; + let ops = 0; + for (const [index, key] of keys.entries()) { + try { + await applyItemOperation(database, { + seed: opts.seed, + phase: opts.phase, + localIndex: 9e4 + index, + kind: "upsert", + itemKey: key, + payloadBytes: Math.min(opts.maxPayloadBytes, 128 + index) + }); + } catch (error) { + throw new Error( + `boundary key write failed for literal key ${JSON.stringify(key)} at index ${index}`, + { cause: error } + ); + } + ops += 1; + } + for (let i = 0; i < 128; i += 1) { + const itemKey = `seq-${opts.phase}-${i.toString().padStart(4, "0")}`; + try { + await applyItemOperation(database, { + seed: opts.seed, + phase: opts.phase, + localIndex: 91e3 + i, + kind: i % 4 === 0 ? "delete" : "upsert", + itemKey, + payloadBytes: Math.min(opts.maxPayloadBytes, 32 + i % 97) + }); + } catch (error) { + throw new Error( + `boundary key write failed for sequential key ${JSON.stringify(itemKey)} at index ${i}`, + { cause: error } + ); + } + ops += 1; + } + await recordProbe(database, opts.phase, "boundary-keys", "keys-written", 136, ops, ops !== 136); + return ops; +} +async function applyGrowthProbe(database, opts) { + const chunkBytes = Math.max(1, Math.min(LARGE_WRITE_CHUNK_BYTES, opts.maxPayloadBytes)); + const rows = Math.max(1, Math.ceil(opts.growthTargetBytes / chunkBytes)); + let written = 0; + for (let i = 0; i < rows; i += 1) { + const size = Math.min(chunkBytes, opts.growthTargetBytes - written); + const id = `growth-${opts.phase}-${opts.growthTargetBytes}-${i}`; + const payload2 = payloadFor(opts.seed, opts.phase, 1e5 + i, size); + await database.execute( + `INSERT INTO fuzz_edge_payloads ( + id, kind, payload, payload_checksum, payload_bytes, updated_at + ) VALUES (?, 'growth', ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + payload = excluded.payload, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes, + updated_at = excluded.updated_at`, + id, + payload2, + checksum(payload2), + payload2.length, + Date.now() + ); + await database.execute( + `INSERT INTO fuzz_edge_expectations ( + id, present, payload_checksum, payload_bytes + ) VALUES (?, 1, ?, ?) + ON CONFLICT(id) DO UPDATE SET + present = 1, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes`, + id, + checksum(payload2), + payload2.length + ); + written += size; + } + await recordProbe( + database, + opts.phase, + "growth", + "target-bytes-written", + opts.growthTargetBytes, + written, + written !== opts.growthTargetBytes + ); + return rows; +} +async function applyTruncateRecreateProbe(database, opts) { + const id = `truncate-${opts.phase}`; + const largeSize = Math.max(1, Math.min(opts.maxPayloadBytes, 131072)); + const largePayload = payloadFor(opts.seed, opts.phase, 11e4, largeSize); + const tinyPayload = payloadFor(opts.seed, opts.phase, 110001, 1); + const recreatedPayload = payloadFor(opts.seed, opts.phase, 110002, Math.min(4096, largeSize)); + await database.execute( + `INSERT INTO fuzz_edge_payloads ( + id, kind, payload, payload_checksum, payload_bytes, updated_at + ) VALUES (?, 'truncate', ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + payload = excluded.payload, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes, + updated_at = excluded.updated_at`, + id, + largePayload, + checksum(largePayload), + largePayload.length, + Date.now() + ); + await database.execute( + "UPDATE fuzz_edge_payloads SET payload = ?, payload_checksum = ?, payload_bytes = ?, updated_at = ? WHERE id = ?", + tinyPayload, + checksum(tinyPayload), + tinyPayload.length, + Date.now(), + id + ); + await database.execute("DELETE FROM fuzz_edge_payloads WHERE id = ?", id); + await database.execute("VACUUM"); + await database.execute( + `INSERT INTO fuzz_edge_payloads ( + id, kind, payload, payload_checksum, payload_bytes, updated_at + ) VALUES (?, 'truncate', ?, ?, ?, ?)`, + id, + recreatedPayload, + checksum(recreatedPayload), + recreatedPayload.length, + Date.now() + ); + await database.execute( + `INSERT INTO fuzz_edge_expectations ( + id, present, payload_checksum, payload_bytes + ) VALUES (?, 1, ?, ?) + ON CONFLICT(id) DO UPDATE SET + present = 1, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes`, + id, + checksum(recreatedPayload), + recreatedPayload.length + ); + return 5; +} +async function updateShadowChecksums(database, phase) { + const item = await queryOne( + database, + `SELECT COUNT(*) AS rows, COALESCE(SUM(payload_checksum + version + update_count), 0) AS value + FROM fuzz_items` + ); + const edge = await queryOne( + database, + `SELECT COUNT(*) AS rows, COALESCE(SUM(payload_checksum + payload_bytes), 0) AS value + FROM fuzz_edge_payloads` + ); + await transaction(database, async () => { + for (const [name, rows, value] of [ + ["items", item?.rows ?? 0, item?.value ?? 0], + ["edge", edge?.rows ?? 0, edge?.value ?? 0] + ]) { + await database.execute( + `INSERT INTO fuzz_shadow_checksums (name, value, row_count) + VALUES (?, ?, ?) + ON CONFLICT(name) DO UPDATE SET + value = excluded.value, + row_count = excluded.row_count`, + name, + value, + rows + ); + } + }); + await recordProbe(database, phase, "shadow", "shadow-updated", 2, 2, false); + return 2; +} +async function applyConstraintChaos(database, phase) { + await database.execute("PRAGMA foreign_keys = ON"); + const validPrefix = `valid-${phase}`; + const existingValidRows = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_constraints WHERE id LIKE ?", + `${validPrefix}-%` + ); + const runSeq = existingValidRows?.count ?? 0; + const validId = `${validPrefix}-${runSeq}`; + const uniqValue = `uniq-${phase}-${runSeq}`; + const before = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_constraints" + ); + await database.execute( + `INSERT INTO fuzz_constraints (id, must_not_null, qty, uniq) + VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + must_not_null = excluded.must_not_null, + qty = excluded.qty, + uniq = excluded.uniq`, + validId, + "ok", + phase, + uniqValue + ); + const attempts = [ + { + name: `not-null-${phase}-${runSeq}`, + sql: "INSERT INTO fuzz_constraints (id, must_not_null, qty, uniq) VALUES (?, ?, ?, ?)", + args: [`bad-null-${phase}-${runSeq}`, null, 1, `bad-null-${phase}-${runSeq}`] + }, + { + name: `check-${phase}-${runSeq}`, + sql: "INSERT INTO fuzz_constraints (id, must_not_null, qty, uniq) VALUES (?, ?, ?, ?)", + args: [`bad-check-${phase}-${runSeq}`, "ok", -1, `bad-check-${phase}-${runSeq}`] + }, + { + name: `unique-${phase}-${runSeq}`, + sql: "INSERT INTO fuzz_constraints (id, must_not_null, qty, uniq) VALUES (?, ?, ?, ?)", + args: [`bad-unique-${phase}-${runSeq}`, "ok", 1, uniqValue] + } + ]; + for (const attempt of attempts) { + const attemptBefore = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_constraints" + ); + let failed = false; + try { + await database.execute(attempt.sql, ...attempt.args); + } catch { + failed = true; + } + const attemptAfter = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_constraints" + ); + await database.execute( + `INSERT INTO fuzz_constraint_attempts ( + name, expected_failed, actually_failed, before_count, after_count + ) VALUES (?, 1, ?, ?, ?)`, + attempt.name, + failed ? 1 : 0, + attemptBefore?.count ?? 0, + attemptAfter?.count ?? 0 + ); + } + const after = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_constraints" + ); + if ((after?.count ?? 0) !== (before?.count ?? 0) + 1) { + await database.execute( + `INSERT INTO fuzz_constraint_attempts ( + name, expected_failed, actually_failed, before_count, after_count + ) VALUES (?, 0, 0, ?, ?)`, + `valid-count-${phase}-${runSeq}`, + before?.count ?? 0, + after?.count ?? 0 + ); + } + const parentId = `fk-parent-${phase}-${runSeq}`; + const childId = `fk-child-${phase}-${runSeq}`; + await database.execute( + "INSERT INTO fuzz_fk_parent (id) VALUES (?) ON CONFLICT(id) DO NOTHING", + parentId + ); + await database.execute( + `INSERT INTO fuzz_fk_child (id, parent_id) + VALUES (?, ?) + ON CONFLICT(id) DO UPDATE SET parent_id = excluded.parent_id`, + childId, + parentId + ); + const childBeforeDelete = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_fk_child WHERE parent_id = ?", + parentId + ); + await database.execute("DELETE FROM fuzz_fk_parent WHERE id = ?", parentId); + const childAfterDelete = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_fk_child WHERE parent_id = ?", + parentId + ); + await recordProbe( + database, + phase, + "constraints", + "fk-cascade-delete", + 0, + childAfterDelete?.count ?? -1, + (childBeforeDelete?.count ?? 0) !== 1 || (childAfterDelete?.count ?? -1) !== 0 + ); + const fkBefore = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_fk_child" + ); + let fkFailed = false; + try { + await database.execute( + "INSERT INTO fuzz_fk_child (id, parent_id) VALUES (?, ?)", + `fk-orphan-${phase}-${runSeq}`, + `missing-parent-${phase}-${runSeq}` + ); + } catch { + fkFailed = true; + } + const fkAfter = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_fk_child" + ); + await recordProbe( + database, + phase, + "constraints", + "fk-failure-isolation", + `${fkBefore?.count ?? 0}:failed`, + `${fkAfter?.count ?? 0}:${fkFailed ? "failed" : "inserted"}`, + !fkFailed || (fkAfter?.count ?? 0) !== (fkBefore?.count ?? 0) + ); + return attempts.length + 3; +} +async function applyPragmaProbe(database, phase) { + let ops = 0; + for (const [name, setupSql, checkSql, expected] of [ + ["journal_mode", "PRAGMA journal_mode = DELETE", "PRAGMA journal_mode", "nonempty"], + ["synchronous", "PRAGMA synchronous = NORMAL", "PRAGMA synchronous", "nonempty"], + ["cache_size", "PRAGMA cache_size = -2000", "PRAGMA cache_size", "-2000"], + ["foreign_keys", "PRAGMA foreign_keys = ON", "PRAGMA foreign_keys", "1"], + ["auto_vacuum", "PRAGMA auto_vacuum", "PRAGMA auto_vacuum", "nonempty"] + ]) { + try { + await database.execute(setupSql); + const rows = await database.execute(checkSql); + const actual = String(firstColumn(rows[0]) ?? ""); + await recordProbe( + database, + phase, + "pragma", + name, + expected, + actual, + expected === "nonempty" ? actual.length === 0 : actual !== expected + ); + } catch (err) { + await recordProbe( + database, + phase, + "pragma", + name, + expected, + err instanceof Error ? err.message : "unknown error", + true + ); + } + ops += 1; + } + return ops; +} +async function applySavepointScenario(database, phase) { + const keepId = `save-keep-${phase}`; + const rolledBackId = `save-rolled-back-${phase}`; + await database.execute("BEGIN"); + try { + await database.execute( + "INSERT INTO fuzz_savepoints (id, value) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET value = excluded.value", + keepId, + phase + ); + await database.execute("SAVEPOINT sp_rollback_probe"); + await database.execute( + "INSERT INTO fuzz_savepoints (id, value) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET value = excluded.value", + rolledBackId, + 999e3 + phase + ); + await database.execute( + "UPDATE fuzz_savepoints SET value = value + 1000 WHERE id = ?", + keepId + ); + await database.execute("ROLLBACK TO sp_rollback_probe"); + await database.execute("RELEASE sp_rollback_probe"); + await database.execute("COMMIT"); + } catch (err) { + await database.execute("ROLLBACK").catch(() => void 0); + throw err; + } + await database.execute( + `INSERT INTO fuzz_savepoint_expectations (id, present, value) + VALUES (?, 1, ?) + ON CONFLICT(id) DO UPDATE SET present = 1, value = excluded.value`, + keepId, + phase + ); + await database.execute( + `INSERT INTO fuzz_savepoint_expectations (id, present, value) + VALUES (?, 0, 0) + ON CONFLICT(id) DO UPDATE SET present = 0, value = 0`, + rolledBackId + ); + return 5; +} +async function applyIdempotentReplay(database, phase) { + const targetId = `idem-target-${phase % 3}`; + await database.execute( + "INSERT OR IGNORE INTO fuzz_idempotent_targets (id, value) VALUES (?, 0)", + targetId + ); + for (let i = 0; i < 8; i += 1) { + const opId = `idem-${phase}-${i}`; + const amount = phase + i + 1; + for (let attempt = 0; attempt < 3; attempt += 1) { + await transaction(database, async () => { + const existing = await queryOne( + database, + "SELECT op_id FROM fuzz_idempotent_ops WHERE op_id = ?", + opId + ); + if (!existing) { + await database.execute( + "INSERT INTO fuzz_idempotent_ops (op_id, target_id, amount) VALUES (?, ?, ?)", + opId, + targetId, + amount + ); + await database.execute( + "UPDATE fuzz_idempotent_targets SET value = value + ? WHERE id = ?", + amount, + targetId + ); + } + }); + } + } + return 24; +} +async function ensureRelationalSeed(database) { + await transaction(database, async () => { + for (let i = 0; i < 8; i += 1) { + await database.execute( + "INSERT OR IGNORE INTO fuzz_rel_users (id, name) VALUES (?, ?)", + `user-${i}`, + `User ${i}` + ); + } + for (let i = 0; i < 12; i += 1) { + const productId = `product-${i}`; + const initialQty = 1e4; + await database.execute( + "INSERT OR IGNORE INTO fuzz_rel_products (id, price) VALUES (?, ?)", + productId, + (i + 1) * 7 + ); + await database.execute( + `INSERT OR IGNORE INTO fuzz_inventory ( + product_id, initial_qty, sold_qty, stock_qty + ) VALUES (?, ?, 0, ?)`, + productId, + initialQty, + initialQty + ); + } + }); +} +async function applyRelationalOrder(database, opts) { + await ensureRelationalSeed(database); + const orderPrefix = `order-${opts.phase}-${opts.localIndex}`; + const existingOrders = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_orders WHERE id LIKE ?", + `${orderPrefix}-%` + ); + const orderId = `${orderPrefix}-${existingOrders?.count ?? 0}`; + const userId = `user-${intBetween(opts.rng, 0, 7)}`; + const itemCount = intBetween(opts.rng, 1, 4); + let total = 0; + await transaction(database, async () => { + await database.execute( + "INSERT INTO fuzz_orders (id, user_id, total, status) VALUES (?, ?, 0, 'open')", + orderId, + userId + ); + for (let i = 0; i < itemCount; i += 1) { + const productId = `product-${intBetween(opts.rng, 0, 11)}`; + const product = await queryOne( + database, + "SELECT price FROM fuzz_rel_products WHERE id = ?", + productId + ); + const quantity = intBetween(opts.rng, 1, 5); + const price = product?.price ?? 0; + total += price * quantity; + await database.execute( + `INSERT INTO fuzz_order_items ( + order_id, product_id, quantity, price + ) VALUES (?, ?, ?, ?)`, + orderId, + productId, + quantity, + price + ); + await database.execute( + `UPDATE fuzz_inventory + SET sold_qty = sold_qty + ?, stock_qty = stock_qty - ? + WHERE product_id = ?`, + quantity, + quantity, + productId + ); + } + await database.execute( + "UPDATE fuzz_orders SET total = ?, status = 'paid' WHERE id = ?", + total, + orderId + ); + await database.execute( + "INSERT INTO fuzz_payments (order_id, amount, status) VALUES (?, ?, 'captured')", + orderId, + total + ); + }); + return itemCount + 4; +} +async function applyRollbackProbe(database, phase, rowCount = 20) { + const before = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_items WHERE item_key LIKE ?", + `rollback-${phase}-%` + ); + await database.execute("BEGIN"); + try { + for (let i = 0; i < rowCount; i += 1) { + await database.execute( + `INSERT INTO fuzz_items ( + item_key, value, version, update_count, payload, payload_checksum, + payload_bytes, updated_at + ) VALUES (?, ?, 1, 1, ?, ?, ?, ?)`, + `rollback-${phase}-${i}`, + "should-not-survive", + "rollback-payload", + checksum("rollback-payload"), + "rollback-payload".length, + Date.now() + ); + } + throw new Error("intentional rollback probe"); + } catch { + await database.execute("ROLLBACK"); + } + const after = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_items WHERE item_key LIKE ?", + `rollback-${phase}-%` + ); + await database.execute( + `INSERT INTO fuzz_constraint_attempts ( + name, expected_failed, actually_failed, before_count, after_count + ) VALUES (?, 1, 1, ?, ?)`, + `rollback-probe-${phase}`, + before?.count ?? 0, + after?.count ?? 0 + ); + return rowCount; +} +async function applyNastyScript(database, opts) { + const growKey = `nasty-grow-${opts.phase}`; + let ops = 0; + const growMax = Math.min(opts.maxPayloadBytes, 131072); + const growSizes = PAGE_BOUNDARY_SIZES.filter((size) => size <= growMax); + if (!growSizes.includes(1)) growSizes.unshift(1); + if (!growSizes.includes(growMax)) growSizes.push(growMax); + for (const size of growSizes) { + await applyItemOperation(database, { + seed: opts.seed, + phase: opts.phase, + localIndex: 5e4 + size, + kind: "upsert", + itemKey: growKey, + payloadBytes: Math.min(size, opts.maxPayloadBytes) + }); + ops += 1; + } + const hotUpdates = opts.intense ? 1e4 : 250; + await applyHotUpdates(database, { + seed: opts.seed, + phase: opts.phase, + localIndex: 6e4, + itemKey: `nasty-hot-${opts.phase}`, + updates: hotUpdates, + payloadBytes: Math.min(1024, opts.maxPayloadBytes) + }); + ops += hotUpdates; + if (opts.intense) { + await database.execute("CREATE INDEX IF NOT EXISTS idx_nasty_heavy_write ON fuzz_items(value, version)"); + for (let i = 0; i < 1e4; i += 1) { + await applyItemOperation(database, { + seed: opts.seed, + phase: opts.phase, + localIndex: 12e4 + i, + kind: "upsert", + itemKey: `nasty-bulk-${opts.phase}-${i}`, + payloadBytes: Math.min(256, opts.maxPayloadBytes) + }); + ops += 1; + } + for (let i = 0; i < 1e4; i += 2) { + await applyItemOperation(database, { + seed: opts.seed, + phase: opts.phase, + localIndex: 14e4 + i, + kind: "delete", + itemKey: `nasty-bulk-${opts.phase}-${i}`, + payloadBytes: 1 + }); + ops += 1; + } + await database.execute("DROP INDEX IF EXISTS idx_nasty_heavy_write"); + } + const rollbackRows = opts.intense ? 1e3 : 20; + await applyRollbackProbe(database, opts.phase, rollbackRows); + ops += rollbackRows; + return ops; +} +async function applyDeterministicNastyScript(database, opts) { + let ops = 0; + const growId = `nasty-script-grow-${opts.phase}`; + const maxGrowBytes = Math.min(opts.maxPayloadBytes, 131072); + let finalGrowPayload = ""; + await transaction(database, async () => { + for (let i = 0; i < 256; i += 1) { + const size = Math.max(1, Math.floor(1 + (maxGrowBytes - 1) * i / 255)); + const payload2 = payloadFor(opts.seed, opts.phase, 16e4 + i, size); + finalGrowPayload = payload2; + await database.execute( + `INSERT INTO fuzz_edge_payloads ( + id, kind, payload, payload_checksum, payload_bytes, updated_at + ) VALUES (?, 'nasty-grow', ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + payload = excluded.payload, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes, + updated_at = excluded.updated_at`, + growId, + payload2, + checksum(payload2), + payload2.length, + Date.now() + ); + ops += 1; + } + await database.execute( + `INSERT INTO fuzz_edge_expectations ( + id, present, payload_checksum, payload_bytes + ) VALUES (?, 1, ?, ?) + ON CONFLICT(id) DO UPDATE SET + present = 1, + payload_checksum = excluded.payload_checksum, + payload_bytes = excluded.payload_bytes`, + growId, + checksum(finalGrowPayload), + finalGrowPayload.length + ); + }); + const counterId = `nasty-counter-${opts.phase}`; + await database.execute( + "INSERT INTO fuzz_nasty_counter (id, value) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET value = 0", + counterId + ); + await transaction(database, async () => { + for (let i = 0; i < 1e4; i += 1) { + await database.execute( + "UPDATE fuzz_nasty_counter SET value = value + 1 WHERE id = ?", + counterId + ); + ops += 1; + } + }); + const counter2 = await queryOne( + database, + "SELECT value FROM fuzz_nasty_counter WHERE id = ?", + counterId + ); + await recordProbe( + database, + opts.phase, + "nasty-script", + "same-row-10k-updates", + 1e4, + counter2?.value ?? -1, + (counter2?.value ?? -1) !== 1e4 + ); + const groupId = `nasty-bulk-${opts.phase}`; + await database.execute("CREATE INDEX IF NOT EXISTS idx_fuzz_nasty_rows_group_n ON fuzz_nasty_rows(group_id, n)"); + await transaction(database, async () => { + for (let i = 0; i < 1e4; i += 1) { + await database.execute( + `INSERT INTO fuzz_nasty_rows (group_id, n, payload) + VALUES (?, ?, ?) + ON CONFLICT(group_id, n) DO UPDATE SET payload = excluded.payload`, + groupId, + i, + payloadFor(opts.seed, opts.phase, 17e4 + i, 64) + ); + ops += 1; + } + await database.execute("DELETE FROM fuzz_nasty_rows WHERE group_id = ? AND n % 2 = 0", groupId); + ops += 1; + }); + await database.execute("DROP INDEX IF EXISTS idx_fuzz_nasty_rows_group_n"); + const remaining = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_nasty_rows WHERE group_id = ?", + groupId + ); + await recordProbe( + database, + opts.phase, + "nasty-script", + "insert-10k-delete-every-other", + 5e3, + remaining?.count ?? -1, + (remaining?.count ?? -1) !== 5e3 + ); + const indexLeft = await queryOne( + database, + "SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'index' AND name = 'idx_fuzz_nasty_rows_group_n'" + ); + await recordProbe( + database, + opts.phase, + "nasty-script", + "create-drop-index-around-heavy-writes", + 0, + indexLeft?.count ?? -1, + (indexLeft?.count ?? -1) !== 0 + ); + const rollbackGroupId = `nasty-rollback-${opts.phase}`; + const beforeRollback = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_nasty_rows WHERE group_id = ?", + rollbackGroupId + ); + await database.execute("BEGIN"); + try { + for (let i = 0; i < 1e3; i += 1) { + await database.execute( + "INSERT INTO fuzz_nasty_rows (group_id, n, payload) VALUES (?, ?, ?)", + rollbackGroupId, + i, + "rollback" + ); + ops += 1; + } + await database.execute("ROLLBACK"); + } catch (err) { + await database.execute("ROLLBACK").catch(() => void 0); + throw err; + } + const afterRollback = await queryOne( + database, + "SELECT COUNT(*) AS count FROM fuzz_nasty_rows WHERE group_id = ?", + rollbackGroupId + ); + await recordProbe( + database, + opts.phase, + "nasty-script", + "rollback-1k-inserts", + beforeRollback?.count ?? 0, + afterRollback?.count ?? -1, + (afterRollback?.count ?? -1) !== (beforeRollback?.count ?? 0) + ); + return ops; +} +function shouldRunDeepScenario(mode2, scenario) { + return mode2 === scenario || mode2 === "kitchen-sink" || mode2 === "nasty"; +} +async function applyDeepScenarios(database, opts) { + const runScenario = async (name, fn) => { + try { + return await fn(); + } catch (error) { + const detail = error instanceof Error ? error.message : typeof error === "string" ? error : String(error); + throw new Error( + `deep scenario ${name} failed in mode ${opts.mode} during phase ${opts.phase}: ${detail}`, + { cause: error } + ); + } + }; + if (shouldRunDeepScenario(opts.mode, "edge") || opts.mode === "payloads") { + opts.ops.edgePayload = (opts.ops.edgePayload ?? 0) + await runScenario("edge", () => applyEdgePayloads(database, opts)); + } + if (opts.mode === "actual-nul") { + opts.ops.actualNul = (opts.ops.actualNul ?? 0) + await runScenario("actual-nul", () => applyActualNulPayload(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "fragmentation")) { + opts.ops.fragmentation = (opts.ops.fragmentation ?? 0) + await runScenario("fragmentation", () => applyFragmentationChurn(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "schema")) { + opts.ops.schema = (opts.ops.schema ?? 0) + await runScenario("schema", () => applySchemaChurn(database, opts.phase)); + } + if (shouldRunDeepScenario(opts.mode, "index")) { + opts.ops.index = (opts.ops.index ?? 0) + await runScenario("index", () => applyIndexProbe(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "constraints")) { + opts.ops.constraints = (opts.ops.constraints ?? 0) + await runScenario("constraints", () => applyConstraintChaos(database, opts.phase)); + } + if (shouldRunDeepScenario(opts.mode, "savepoints")) { + opts.ops.savepoints = (opts.ops.savepoints ?? 0) + await runScenario("savepoints", () => applySavepointScenario(database, opts.phase)); + } + if (shouldRunDeepScenario(opts.mode, "pragma")) { + opts.ops.pragma = (opts.ops.pragma ?? 0) + await runScenario("pragma", () => applyPragmaProbe(database, opts.phase)); + } + if (shouldRunDeepScenario(opts.mode, "prepared")) { + opts.ops.prepared = (opts.ops.prepared ?? 0) + await runScenario("prepared", () => applyPreparedChurn(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "growth")) { + opts.ops.growth = (opts.ops.growth ?? 0) + await runScenario("growth", () => applyGrowthProbe(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "readwrite")) { + opts.ops.readwrite = (opts.ops.readwrite ?? 0) + await runScenario("readwrite", () => applyReadWriteProbe(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "truncate")) { + opts.ops.truncate = (opts.ops.truncate ?? 0) + await runScenario("truncate", () => applyTruncateRecreateProbe(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "boundary-keys")) { + opts.ops.boundaryKeys = (opts.ops.boundaryKeys ?? 0) + await runScenario("boundary-keys", () => applyBoundaryKeys(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "relational")) { + const orders = Math.max(1, Math.floor(opts.iterations / 20)); + for (let i = 0; i < orders; i += 1) { + opts.ops.relational = (opts.ops.relational ?? 0) + await runScenario( + "relational", + () => applyRelationalOrder(database, { + phase: opts.phase, + localIndex: i, + rng: opts.rng + }) + ); + } + } + if (opts.mode === "kitchen-sink" || opts.mode === "nasty") { + opts.ops.idempotent = (opts.ops.idempotent ?? 0) + await runScenario("idempotent", () => applyIdempotentReplay(database, opts.phase)); + opts.ops.nasty = (opts.ops.nasty ?? 0) + await runScenario( + "nasty", + () => applyNastyScript(database, { ...opts, intense: opts.mode === "nasty" }) + ); + } + if (opts.mode === "nasty-script") { + opts.ops.nasty = (opts.ops.nasty ?? 0) + await runScenario("nasty-script", () => applyDeterministicNastyScript(database, opts)); + } + if (shouldRunDeepScenario(opts.mode, "shadow")) { + opts.ops.shadow = (opts.ops.shadow ?? 0) + await runScenario("shadow", () => updateShadowChecksums(database, opts.phase)); + } +} +function chooseKind(mode2, rng) { + const roll = rng(); + if (mode2 === "transactions") { + if (roll < 0.55) return "transfer"; + if (roll < 0.75) return "upsert"; + if (roll < 0.9) return "update"; + return "delete"; + } + if (mode2 === "hot") { + if (roll < 0.6) return "hot"; + if (roll < 0.75) return "upsert"; + if (roll < 0.9) return "update"; + return "delete"; + } + if (mode2 === "payloads") { + if (roll < 0.4) return "upsert"; + if (roll < 0.7) return "insert"; + if (roll < 0.9) return "update"; + return "delete"; + } + if (roll < 0.2) return "insert"; + if (roll < 0.45) return "update"; + if (roll < 0.65) return "delete"; + if (roll < 0.85) return "upsert"; + if (roll < 0.95) return "hot"; + return "transfer"; +} +async function validate(database) { + const integrity = await queryOne( + database, + "PRAGMA integrity_check" + ); + const quick = await queryOne( + database, + "PRAGMA quick_check" + ); + const totals = await queryOne( + database, + `WITH latest AS ( + SELECT e.* + FROM fuzz_item_events e + JOIN ( + SELECT item_key, MAX(seq) AS seq + FROM fuzz_item_events + GROUP BY item_key + ) m ON m.item_key = e.item_key AND m.seq = e.seq + ) + SELECT + (SELECT COUNT(*) FROM fuzz_item_events) AS total_events, + (SELECT COUNT(*) FROM fuzz_items) AS active_rows, + (SELECT COUNT(*) FROM latest WHERE present = 1) AS expected_rows, + (SELECT COALESCE(SUM(version), 0) FROM fuzz_items) AS actual_version_sum, + (SELECT COALESCE(SUM(version), 0) FROM latest WHERE present = 1) AS expected_version_sum, + (SELECT COALESCE(SUM(payload_checksum), 0) FROM fuzz_items) AS actual_payload_checksum_sum, + (SELECT COALESCE(SUM(payload_checksum), 0) FROM latest WHERE present = 1) AS expected_payload_checksum_sum` + ); + const mismatches = await queryOne( + database, + `WITH latest AS ( + SELECT e.* + FROM fuzz_item_events e + JOIN ( + SELECT item_key, MAX(seq) AS seq + FROM fuzz_item_events + GROUP BY item_key + ) m ON m.item_key = e.item_key AND m.seq = e.seq + ) + SELECT + ( + SELECT COUNT(*) + FROM latest l + LEFT JOIN fuzz_items i ON i.item_key = l.item_key + WHERE l.present = 1 AND i.item_key IS NULL + ) AS missing_rows, + ( + SELECT COUNT(*) + FROM fuzz_items i + LEFT JOIN latest l ON l.item_key = i.item_key + WHERE l.item_key IS NULL OR l.present = 0 + ) AS extra_rows, + ( + SELECT COUNT(*) + FROM latest l + JOIN fuzz_items i ON i.item_key = l.item_key + WHERE l.present = 1 + AND ( + i.value != l.value OR + i.version != l.version OR + i.update_count != l.update_count OR + i.payload_checksum != l.payload_checksum OR + i.payload_bytes != l.payload_bytes + ) + ) AS mismatched_rows, + ( + SELECT COUNT(*) + FROM ( + SELECT item_key + FROM fuzz_items + GROUP BY item_key + HAVING COUNT(*) > 1 + ) + ) AS duplicate_keys` + ); + const accounts = await queryOne( + database, + `SELECT + COUNT(*) AS account_count, + COALESCE(SUM(balance), 0) AS account_balance_sum, + ( + SELECT COUNT(*) + FROM fuzz_transfer_events + WHERE balance_sum_before != ? OR balance_sum_after != ? + ) AS account_balance_mismatch + FROM fuzz_accounts`, + ACCOUNT_COUNT * ACCOUNT_INITIAL_BALANCE, + ACCOUNT_COUNT * ACCOUNT_INITIAL_BALANCE + ); + const edge = await queryOne( + database, + `SELECT + (SELECT COUNT(*) FROM fuzz_edge_payloads) AS edge_rows, + (SELECT COUNT(*) FROM fuzz_edge_expectations WHERE present = 1) AS edge_expected_rows, + ( + SELECT COUNT(*) + FROM fuzz_edge_expectations e + LEFT JOIN fuzz_edge_payloads p ON p.id = e.id + WHERE + (e.present = 1 AND p.id IS NULL) OR + (e.present = 0 AND p.id IS NOT NULL) OR + (e.present = 1 AND ( + p.payload_checksum != e.payload_checksum OR + p.payload_bytes != e.payload_bytes + )) + ) AS edge_mismatches` + ); + const indexProbe = await queryOne( + database, + `SELECT + (SELECT COUNT(*) FROM fuzz_indexed) AS index_rows, + ( + SELECT COUNT(*) + FROM ( + SELECT id FROM fuzz_indexed + WHERE tenant = 'tenant-1' AND bucket BETWEEN 2 AND 8 AND score BETWEEN -250 AND 250 + EXCEPT + SELECT id FROM fuzz_indexed NOT INDEXED + WHERE tenant = 'tenant-1' AND bucket BETWEEN 2 AND 8 AND score BETWEEN -250 AND 250 + ) + ) + ( + SELECT COUNT(*) + FROM ( + SELECT id FROM fuzz_indexed NOT INDEXED + WHERE tenant = 'tenant-1' AND bucket BETWEEN 2 AND 8 AND score BETWEEN -250 AND 250 + EXCEPT + SELECT id FROM fuzz_indexed + WHERE tenant = 'tenant-1' AND bucket BETWEEN 2 AND 8 AND score BETWEEN -250 AND 250 + ) + ) AS index_mismatches` + ); + const relational = await queryOne( + database, + `SELECT + (SELECT COUNT(*) FROM fuzz_orders) AS relational_orders, + ( + SELECT COUNT(*) + FROM fuzz_orders o + LEFT JOIN ( + SELECT order_id, COALESCE(SUM(quantity * price), 0) AS item_total + FROM fuzz_order_items + GROUP BY order_id + ) i ON i.order_id = o.id + WHERE o.total != COALESCE(i.item_total, 0) + ) + ( + SELECT COUNT(*) + FROM fuzz_orders o + LEFT JOIN ( + SELECT order_id, COALESCE(SUM(amount), 0) AS payment_total + FROM fuzz_payments + WHERE status = 'captured' + GROUP BY order_id + ) p ON p.order_id = o.id + WHERE o.status = 'paid' AND o.total != COALESCE(p.payment_total, 0) + ) + ( + SELECT COUNT(*) + FROM fuzz_inventory + WHERE initial_qty != sold_qty + stock_qty OR stock_qty < 0 + ) AS relational_mismatches` + ); + const constraints = await queryOne( + database, + `SELECT + COUNT(*) AS constraint_attempts, + COALESCE(SUM( + CASE + WHEN expected_failed = 1 AND (actually_failed != 1 OR before_count != after_count) THEN 1 + WHEN expected_failed = 0 AND after_count != before_count + 1 THEN 1 + ELSE 0 + END + ), 0) AS constraint_leaks + FROM fuzz_constraint_attempts` + ); + const savepoints = await queryOne( + database, + `SELECT + (SELECT COUNT(*) FROM fuzz_savepoints) AS savepoint_rows, + ( + SELECT COUNT(*) + FROM fuzz_savepoint_expectations e + LEFT JOIN fuzz_savepoints s ON s.id = e.id + WHERE + (e.present = 1 AND s.id IS NULL) OR + (e.present = 0 AND s.id IS NOT NULL) OR + (e.present = 1 AND s.value != e.value) + ) AS savepoint_mismatches` + ); + const idempotency = await queryOne( + database, + `SELECT + (SELECT COUNT(*) FROM fuzz_idempotent_ops) AS idempotent_ops, + ( + SELECT COUNT(*) + FROM fuzz_idempotent_targets t + LEFT JOIN ( + SELECT target_id, COALESCE(SUM(amount), 0) AS expected + FROM fuzz_idempotent_ops + GROUP BY target_id + ) o ON o.target_id = t.id + WHERE t.value != COALESCE(o.expected, 0) + ) AS idempotent_mismatches` + ); + const schema = await queryOne( + database, + `SELECT + COUNT(*) AS schema_objects, + COALESCE(SUM(CASE WHEN m.name IS NULL THEN 1 ELSE 0 END), 0) AS schema_missing_objects + FROM fuzz_schema_registry r + LEFT JOIN sqlite_master m ON m.name = r.name AND m.type = r.type` + ); + const probes = await queryOne( + database, + `SELECT + COUNT(*) AS probe_rows, + COALESCE(SUM(mismatch), 0) AS probe_mismatches + FROM fuzz_probe_results` + ); + const prepared = await queryOne( + database, + `SELECT + (SELECT COUNT(*) FROM fuzz_prepared_churn) AS prepared_rows, + ( + SELECT COUNT(*) + FROM fuzz_prepared_expectations e + LEFT JOIN fuzz_prepared_churn p ON p.id = e.id + WHERE + p.id IS NULL OR + p.value != e.value OR + p.payload_checksum != e.payload_checksum + ) AS prepared_mismatches` + ); + const shadow = await queryOne( + database, + `WITH recomputed AS ( + SELECT 'items' AS name, + COUNT(*) AS row_count, + COALESCE(SUM(payload_checksum + version + update_count), 0) AS value + FROM fuzz_items + UNION ALL + SELECT 'edge' AS name, + COUNT(*) AS row_count, + COALESCE(SUM(payload_checksum + payload_bytes), 0) AS value + FROM fuzz_edge_payloads + ) + SELECT + (SELECT COUNT(*) FROM fuzz_shadow_checksums) AS shadow_rows, + ( + SELECT COUNT(*) + FROM fuzz_shadow_checksums s + JOIN recomputed r ON r.name = s.name + WHERE s.value != r.value OR s.row_count != r.row_count + ) AS shadow_mismatches` + ); + const summary = { + totalEvents: totals?.total_events ?? 0, + activeRows: totals?.active_rows ?? 0, + expectedRows: totals?.expected_rows ?? 0, + missingRows: mismatches?.missing_rows ?? 0, + extraRows: mismatches?.extra_rows ?? 0, + mismatchedRows: mismatches?.mismatched_rows ?? 0, + duplicateKeys: mismatches?.duplicate_keys ?? 0, + actualVersionSum: totals?.actual_version_sum ?? 0, + expectedVersionSum: totals?.expected_version_sum ?? 0, + actualPayloadChecksumSum: totals?.actual_payload_checksum_sum ?? 0, + expectedPayloadChecksumSum: totals?.expected_payload_checksum_sum ?? 0, + accountCount: accounts?.account_count ?? 0, + accountBalanceSum: accounts?.account_balance_sum ?? 0, + expectedAccountBalanceSum: ACCOUNT_COUNT * ACCOUNT_INITIAL_BALANCE, + accountBalanceMismatch: accounts?.account_balance_mismatch ?? 0, + integrityCheck: integrity?.integrity_check ?? "missing", + quickCheck: quick?.quick_check ?? "missing", + edgeRows: edge?.edge_rows ?? 0, + edgeExpectedRows: edge?.edge_expected_rows ?? 0, + edgeMismatches: edge?.edge_mismatches ?? 0, + indexRows: indexProbe?.index_rows ?? 0, + indexMismatches: indexProbe?.index_mismatches ?? 0, + relationalOrders: relational?.relational_orders ?? 0, + relationalMismatches: relational?.relational_mismatches ?? 0, + constraintAttempts: constraints?.constraint_attempts ?? 0, + constraintLeaks: constraints?.constraint_leaks ?? 0, + savepointRows: savepoints?.savepoint_rows ?? 0, + savepointMismatches: savepoints?.savepoint_mismatches ?? 0, + idempotentOps: idempotency?.idempotent_ops ?? 0, + idempotentMismatches: idempotency?.idempotent_mismatches ?? 0, + schemaObjects: schema?.schema_objects ?? 0, + schemaMissingObjects: schema?.schema_missing_objects ?? 0, + probeRows: probes?.probe_rows ?? 0, + probeMismatches: probes?.probe_mismatches ?? 0, + preparedRows: prepared?.prepared_rows ?? 0, + preparedMismatches: prepared?.prepared_mismatches ?? 0, + shadowRows: shadow?.shadow_rows ?? 0, + shadowMismatches: shadow?.shadow_mismatches ?? 0 + }; + return summary; +} +async function debugItemMismatches(database, limit = 5) { + const rows = await database.execute( + `WITH latest AS ( + SELECT e.* + FROM fuzz_item_events e + JOIN ( + SELECT item_key, MAX(seq) AS seq + FROM fuzz_item_events + GROUP BY item_key + ) m ON m.item_key = e.item_key AND m.seq = e.seq + ) + SELECT + COALESCE(l.item_key, i.item_key) AS item_key, + i.value AS actual_value, + l.value AS expected_value, + i.version AS actual_version, + l.version AS expected_version, + i.update_count AS actual_update_count, + l.update_count AS expected_update_count, + i.payload_checksum AS actual_payload_checksum, + l.payload_checksum AS expected_payload_checksum, + i.payload_bytes AS actual_payload_bytes, + l.payload_bytes AS expected_payload_bytes + FROM latest l + FULL OUTER JOIN fuzz_items i ON i.item_key = l.item_key + WHERE + (l.present = 1 AND i.item_key IS NULL) OR + ((l.item_key IS NULL OR l.present = 0) AND i.item_key IS NOT NULL) OR + ( + l.present = 1 AND i.item_key IS NOT NULL AND ( + i.value != l.value OR + i.version != l.version OR + i.update_count != l.update_count OR + i.payload_checksum != l.payload_checksum OR + i.payload_bytes != l.payload_bytes + ) + ) + ORDER BY COALESCE(l.item_key, i.item_key) + LIMIT ?`, + limit + ); + const itemMismatches = rows.map((row) => ({ + itemKey: row.item_key, + actualValue: row.actual_value, + expectedValue: row.expected_value, + actualVersion: row.actual_version, + expectedVersion: row.expected_version, + actualUpdateCount: row.actual_update_count, + expectedUpdateCount: row.expected_update_count, + actualPayloadChecksum: row.actual_payload_checksum, + expectedPayloadChecksum: row.expected_payload_checksum, + actualPayloadBytes: row.actual_payload_bytes, + expectedPayloadBytes: row.expected_payload_bytes + })); + const recentEventsByKey = {}; + for (const row of itemMismatches) { + const events = await database.execute( + `SELECT + seq, + phase, + local_index, + kind, + present, + value, + version, + update_count, + payload_checksum, + payload_bytes, + applied + FROM fuzz_item_events + WHERE item_key = ? + ORDER BY seq DESC + LIMIT 10`, + row.itemKey + ); + recentEventsByKey[row.itemKey] = events.map((event21) => ({ + seq: event21.seq, + phase: event21.phase, + localIndex: event21.local_index, + kind: event21.kind, + present: event21.present, + value: event21.value, + version: event21.version, + updateCount: event21.update_count, + payloadChecksum: event21.payload_checksum, + payloadBytes: event21.payload_bytes, + applied: event21.applied + })); + } + return { itemMismatches, recentEventsByKey }; +} +var rawSqliteFuzzer = actor52({ + options: { + actionTimeout: 3e5 + }, + db: db9({ + onMigrate: async (database) => { + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_items ( + item_key TEXT PRIMARY KEY, + value TEXT NOT NULL, + version INTEGER NOT NULL, + update_count INTEGER NOT NULL, + payload TEXT NOT NULL, + payload_checksum INTEGER NOT NULL, + payload_bytes INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_item_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + phase INTEGER NOT NULL, + local_index INTEGER NOT NULL, + kind TEXT NOT NULL, + item_key TEXT NOT NULL, + present INTEGER NOT NULL, + value TEXT, + version INTEGER NOT NULL, + update_count INTEGER NOT NULL, + payload_checksum INTEGER NOT NULL, + payload_bytes INTEGER NOT NULL, + applied INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_fuzz_item_events_key_seq ON fuzz_item_events(item_key, seq)" + ); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_accounts ( + id TEXT PRIMARY KEY, + balance INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_transfer_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + phase INTEGER NOT NULL, + local_index INTEGER NOT NULL, + from_account TEXT NOT NULL, + to_account TEXT NOT NULL, + amount INTEGER NOT NULL, + balance_sum_before INTEGER NOT NULL, + balance_sum_after INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_edge_payloads ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + payload_checksum INTEGER NOT NULL, + payload_bytes INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_edge_expectations ( + id TEXT PRIMARY KEY, + present INTEGER NOT NULL, + payload_checksum INTEGER NOT NULL, + payload_bytes INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_trigger_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload_id TEXT NOT NULL, + old_checksum INTEGER NOT NULL, + new_checksum INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_fuzz_edge_kind_size ON fuzz_edge_payloads(kind, payload_bytes)" + ); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_indexed ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant TEXT NOT NULL, + bucket INTEGER NOT NULL, + score INTEGER NOT NULL, + label TEXT NOT NULL, + payload TEXT NOT NULL + ) + `); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_fuzz_indexed_tenant_bucket_score ON fuzz_indexed(tenant, bucket, score)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_fuzz_indexed_score_label ON fuzz_indexed(score, label)" + ); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_rel_users ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_rel_products ( + id TEXT PRIMARY KEY, + price INTEGER NOT NULL CHECK (price >= 0) + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_orders ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + total INTEGER NOT NULL, + status TEXT NOT NULL, + FOREIGN KEY (user_id) REFERENCES fuzz_rel_users(id) + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_order_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id TEXT NOT NULL, + product_id TEXT NOT NULL, + quantity INTEGER NOT NULL CHECK (quantity > 0), + price INTEGER NOT NULL CHECK (price >= 0), + FOREIGN KEY (order_id) REFERENCES fuzz_orders(id), + FOREIGN KEY (product_id) REFERENCES fuzz_rel_products(id) + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_payments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id TEXT NOT NULL, + amount INTEGER NOT NULL, + status TEXT NOT NULL, + FOREIGN KEY (order_id) REFERENCES fuzz_orders(id) + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_inventory ( + product_id TEXT PRIMARY KEY, + initial_qty INTEGER NOT NULL, + sold_qty INTEGER NOT NULL, + stock_qty INTEGER NOT NULL, + FOREIGN KEY (product_id) REFERENCES fuzz_rel_products(id) + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_constraints ( + id TEXT PRIMARY KEY, + must_not_null TEXT NOT NULL, + qty INTEGER NOT NULL CHECK (qty >= 0), + uniq TEXT NOT NULL UNIQUE + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_constraint_attempts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + expected_failed INTEGER NOT NULL, + actually_failed INTEGER NOT NULL, + before_count INTEGER NOT NULL, + after_count INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_fk_parent ( + id TEXT PRIMARY KEY + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_fk_child ( + id TEXT PRIMARY KEY, + parent_id TEXT NOT NULL, + FOREIGN KEY (parent_id) REFERENCES fuzz_fk_parent(id) ON DELETE CASCADE + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_savepoints ( + id TEXT PRIMARY KEY, + value INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_savepoint_expectations ( + id TEXT PRIMARY KEY, + present INTEGER NOT NULL, + value INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_idempotent_ops ( + op_id TEXT PRIMARY KEY, + target_id TEXT NOT NULL, + amount INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_idempotent_targets ( + id TEXT PRIMARY KEY, + value INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_schema_registry ( + name TEXT PRIMARY KEY, + type TEXT NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_probe_results ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + phase INTEGER NOT NULL, + scenario TEXT NOT NULL, + name TEXT NOT NULL, + expected TEXT NOT NULL, + actual TEXT NOT NULL, + mismatch INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_prepared_churn ( + id TEXT PRIMARY KEY, + value INTEGER NOT NULL, + payload TEXT NOT NULL, + payload_checksum INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_prepared_expectations ( + id TEXT PRIMARY KEY, + value INTEGER NOT NULL, + payload_checksum INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_shadow_checksums ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL, + row_count INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_nasty_counter ( + id TEXT PRIMARY KEY, + value INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS fuzz_nasty_rows ( + group_id TEXT NOT NULL, + n INTEGER NOT NULL, + payload TEXT NOT NULL, + PRIMARY KEY (group_id, n) + ) + `); + } + }), + actions: { + reset: async (c) => { + await c.db.execute("DELETE FROM fuzz_nasty_rows"); + await c.db.execute("DELETE FROM fuzz_nasty_counter"); + await c.db.execute("DELETE FROM fuzz_shadow_checksums"); + await c.db.execute("DELETE FROM fuzz_prepared_expectations"); + await c.db.execute("DELETE FROM fuzz_prepared_churn"); + await c.db.execute("DELETE FROM fuzz_probe_results"); + await c.db.execute("DELETE FROM fuzz_schema_registry"); + await c.db.execute("DELETE FROM fuzz_idempotent_targets"); + await c.db.execute("DELETE FROM fuzz_idempotent_ops"); + await c.db.execute("DELETE FROM fuzz_savepoint_expectations"); + await c.db.execute("DELETE FROM fuzz_savepoints"); + await c.db.execute("DELETE FROM fuzz_fk_child"); + await c.db.execute("DELETE FROM fuzz_fk_parent"); + await c.db.execute("DELETE FROM fuzz_constraint_attempts"); + await c.db.execute("DELETE FROM fuzz_constraints"); + await c.db.execute("DELETE FROM fuzz_payments"); + await c.db.execute("DELETE FROM fuzz_order_items"); + await c.db.execute("DELETE FROM fuzz_orders"); + await c.db.execute("DELETE FROM fuzz_inventory"); + await c.db.execute("DELETE FROM fuzz_rel_products"); + await c.db.execute("DELETE FROM fuzz_rel_users"); + await c.db.execute("DELETE FROM fuzz_indexed"); + await c.db.execute("DELETE FROM fuzz_trigger_audit"); + await c.db.execute("DELETE FROM fuzz_edge_expectations"); + await c.db.execute("DELETE FROM fuzz_edge_payloads"); + await c.db.execute("DELETE FROM fuzz_transfer_events"); + await c.db.execute("DELETE FROM fuzz_accounts"); + await c.db.execute("DELETE FROM fuzz_item_events"); + await c.db.execute("DELETE FROM fuzz_items"); + await ensureAccounts(c.db); + return await validate(c.db); + }, + runPhase: async (c, input) => { + const mode2 = input.mode ?? "balanced"; + const iterations = Math.max(1, Math.floor(input.iterations)); + const keySpace = Math.max(1, Math.floor(input.keySpace ?? DEFAULT_KEY_SPACE)); + const maxPayloadBytes = Math.max( + 1, + Math.floor(input.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES) + ); + const growthTargetBytes = Math.max( + 1, + Math.floor(input.growthTargetBytes ?? DEFAULT_GROWTH_TARGET_BYTES) + ); + const rng = makeRng(`${input.seed}:${input.phase}:${mode2}`); + const ops = {}; + let stage = "ensureAccounts"; + try { + await ensureAccounts(c.db); + for (let i = 0; i < iterations; i += 1) { + const kind = chooseKind(mode2, rng); + ops[kind] = (ops[kind] ?? 0) + 1; + stage = `base:${kind}:iteration:${i}`; + if (kind === "transfer") { + const fromIndex = intBetween(rng, 0, ACCOUNT_COUNT - 1); + let toIndex = intBetween(rng, 0, ACCOUNT_COUNT - 1); + if (toIndex === fromIndex) toIndex = (toIndex + 1) % ACCOUNT_COUNT; + const fromAccount = `acct-${fromIndex}`; + const toAccount = `acct-${toIndex}`; + try { + await applyTransfer(c.db, { + phase: input.phase, + localIndex: i, + fromAccount, + toAccount, + amount: intBetween(rng, 1, 500) + }); + } catch (error) { + throw new Error( + `base operation transfer failed at iteration ${i} from ${fromAccount} to ${toAccount}`, + { cause: error } + ); + } + } else if (kind === "hot") { + const itemKey = `hot-${intBetween(rng, 0, 3)}`; + const updates = intBetween(rng, 2, mode2 === "hot" ? 12 : 5); + try { + await applyHotUpdates(c.db, { + seed: input.seed, + phase: input.phase, + localIndex: i, + itemKey, + updates, + payloadBytes: intBetween(rng, 1, maxPayloadBytes) + }); + } catch (error) { + throw new Error( + `base operation hot failed at iteration ${i} for ${itemKey} with ${updates} updates`, + { cause: error } + ); + } + } else { + const itemKey = mode2 === "hot" && rng() < 0.6 ? `hot-${intBetween(rng, 0, 3)}` : `item-${intBetween(rng, 0, keySpace - 1)}`; + const payloadBytes = mode2 === "payloads" ? intBetween(rng, Math.min(256, maxPayloadBytes), maxPayloadBytes) : intBetween(rng, 1, maxPayloadBytes); + try { + await applyItemOperation(c.db, { + seed: input.seed, + phase: input.phase, + localIndex: i, + kind, + itemKey, + payloadBytes + }); + } catch (error) { + throw new Error( + `base operation ${kind} failed at iteration ${i} for ${JSON.stringify(itemKey)} with payloadBytes ${payloadBytes}`, + { cause: error } + ); + } + } + } + stage = "deep-scenarios"; + await applyDeepScenarios(c.db, { + seed: input.seed, + phase: input.phase, + mode: mode2, + iterations, + rng, + maxPayloadBytes, + growthTargetBytes, + ops + }); + stage = "validate"; + return { + seed: input.seed, + phase: input.phase, + mode: mode2, + iterations, + ops, + validation: await validate(c.db) + }; + } catch (error) { + const detail = error instanceof Error ? error.message : typeof error === "string" ? error : String(error); + throw new Error( + `runPhase failed during ${stage} for mode ${mode2} phase ${input.phase} seed ${input.seed}: ${detail}`, + { cause: error } + ); + } + }, + validate: async (c) => { + await ensureAccounts(c.db); + return await validate(c.db); + }, + debugItemMismatches: async (c, limit) => { + await ensureAccounts(c.db); + return await debugItemMismatches(c.db, limit ?? 5); + }, + goToSleep: (c) => { + c.sleep(); + return { ok: true }; + } + } +}); + +// src/actors/testing/sqlite-memory-pressure.ts +import { actor as actor53 } from "rivetkit"; +import { db as db10 } from "rivetkit/db"; +var DEFAULT_INSERT_ROWS = 128; +var DEFAULT_ROW_BYTES3 = 16 * 1024; +var DEFAULT_SCAN_ROWS = 512; +var INSERT_BATCH_ROWS = 32; +function finiteInt(value, fallback) { + if (value === void 0) return fallback; + if (!Number.isFinite(value) || value < 0) { + throw new Error(`expected a non-negative finite number, got ${value}`); + } + return Math.floor(value); +} +function copyNativeMetrics(metrics) { + if (!metrics) return null; + const raw = metrics; + const numberField3 = (camel, snake) => Number(raw[camel] ?? raw[snake] ?? 0); + return { + requestBuildNs: numberField3("requestBuildNs", "request_build_ns"), + serializeNs: numberField3("serializeNs", "serialize_ns"), + transportNs: numberField3("transportNs", "transport_ns"), + stateUpdateNs: numberField3("stateUpdateNs", "state_update_ns"), + totalNs: numberField3("totalNs", "total_ns"), + commitCount: numberField3("commitCount", "commit_count"), + pageCacheEntries: numberField3("pageCacheEntries", "page_cache_entries"), + pageCacheWeightedSize: numberField3( + "pageCacheWeightedSize", + "page_cache_weighted_size" + ), + pageCacheCapacityPages: numberField3( + "pageCacheCapacityPages", + "page_cache_capacity_pages" + ), + writeBufferDirtyPages: numberField3( + "writeBufferDirtyPages", + "write_buffer_dirty_pages" + ), + dbSizePages: numberField3("dbSizePages", "db_size_pages") + }; +} +async function queryOne2(database, sql, ...args) { + const rows = await database.execute(sql, ...args); + if (!rows[0]) throw new Error(`query returned no rows: ${sql}`); + return rows[0]; +} +async function storageStats(database) { + const [pageCount, freelistCount, pageSize] = await Promise.all([ + queryOne2(database, "PRAGMA page_count"), + queryOne2(database, "PRAGMA freelist_count"), + queryOne2(database, "PRAGMA page_size") + ]); + const nativeMetrics = await database.nativeMetrics?.(); + const copiedMetrics = copyNativeMetrics(nativeMetrics); + return { + page_count: pageCount.page_count, + freelist_count: freelistCount.freelist_count, + page_size: pageSize.page_size, + vfs: copiedMetrics + }; +} +var sqliteMemoryPressure = actor53({ + options: { + actionTimeout: 3e5 + }, + state: { + sleepCount: 0 + }, + db: db10({ + onMigrate: async (database) => { + await database.execute(` + CREATE TABLE IF NOT EXISTS pressure_rows ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + seed TEXT NOT NULL, + cycle INTEGER NOT NULL, + bucket INTEGER NOT NULL, + payload BLOB NOT NULL, + touched_count INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL + ) + `); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_pressure_rows_seed_cycle ON pressure_rows(seed, cycle)" + ); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_pressure_rows_bucket ON pressure_rows(bucket)" + ); + await database.execute(` + CREATE TABLE IF NOT EXISTS pressure_cycles ( + cycle INTEGER PRIMARY KEY, + seed TEXT NOT NULL, + inserted_rows INTEGER NOT NULL, + deleted_rows INTEGER NOT NULL, + active_rows INTEGER NOT NULL, + active_bytes INTEGER NOT NULL, + duration_ms REAL NOT NULL, + created_at INTEGER NOT NULL + ) + `); + } + }), + onSleep: (c) => { + c.state.sleepCount += 1; + console.log( + JSON.stringify({ + kind: "sqlite_memory_pressure_on_sleep", + actorId: c.actorId, + sleepCount: c.state.sleepCount, + timestamp: (/* @__PURE__ */ new Date()).toISOString() + }) + ); + }, + actions: { + reset: async (c) => { + await c.db.execute("DELETE FROM pressure_cycles"); + await c.db.execute("DELETE FROM pressure_rows"); + await c.db.execute("VACUUM"); + return { + ok: true, + storage: await storageStats(c.db) + }; + }, + goToSleep: (c) => { + c.sleep(); + return { ok: true }; + }, + releaseStorage: async (c) => { + const before = await storageStats(c.db); + return { + ok: true, + before, + after: await storageStats(c.db) + }; + }, + stats: async (c) => { + const rowStats = await queryOne2( + c.db, + "SELECT COUNT(*) AS active_rows, COALESCE(SUM(length(payload)), 0) AS active_bytes, COALESCE(SUM(touched_count), 0) AS touched_sum FROM pressure_rows" + ); + const cycles = await queryOne2( + c.db, + "SELECT COUNT(*) AS count FROM pressure_cycles" + ); + const integrity = await queryOne2( + c.db, + "PRAGMA integrity_check" + ); + return { + activeRows: rowStats.active_rows, + activeBytes: rowStats.active_bytes ?? 0, + touchedCount: rowStats.touched_sum ?? 0, + cycles: cycles.count, + integrityCheck: integrity.integrity_check, + storage: await storageStats(c.db) + }; + }, + runCycle: async (c, input) => { + const startedAt = performance.now(); + const insertRows = finiteInt(input.insertRows, DEFAULT_INSERT_ROWS); + const rowBytes = finiteInt(input.rowBytes, DEFAULT_ROW_BYTES3); + const scanRows = Math.max(1, finiteInt(input.scanRows, DEFAULT_SCAN_ROWS)); + const now = Date.now(); + let insertedRows = 0; + const logStage = (stage, phase, fields = {}) => { + console.log( + JSON.stringify({ + kind: "sqlite_memory_pressure_run_cycle_stage", + actorId: c.actorId, + seed: input.seed, + cycle: input.cycle, + stage, + phase, + elapsedMs: performance.now() - startedAt, + timestamp: (/* @__PURE__ */ new Date()).toISOString(), + ...fields + }) + ); + }; + const executeTimed = async (stage, sql, ...args) => { + const stageStartedAt = performance.now(); + logStage(stage, "start", { argCount: args.length }); + try { + const rows = await c.db.execute(sql, ...args); + logStage(stage, "end", { + durationMs: performance.now() - stageStartedAt, + rowCount: rows.length + }); + return rows; + } catch (err) { + logStage(stage, "error", { + durationMs: performance.now() - stageStartedAt, + error: err instanceof Error ? err.message : String(err) + }); + throw err; + } + }; + logStage("run_cycle", "start", { + insertRows, + rowBytes, + scanRows + }); + await executeTimed("begin", "BEGIN"); + try { + while (insertedRows < insertRows) { + const batchRows = Math.min( + INSERT_BATCH_ROWS, + insertRows - insertedRows + ); + const placeholders = []; + const args = []; + for (let i = 0; i < batchRows; i += 1) { + const rowIndex = insertedRows + i; + placeholders.push("(?, ?, ?, randomblob(?), 0, ?)"); + args.push( + input.seed, + input.cycle, + (input.cycle + rowIndex) % 32, + rowBytes, + now + rowIndex + ); + } + await executeTimed( + "insert_batch", + `INSERT INTO pressure_rows (seed, cycle, bucket, payload, touched_count, created_at) VALUES ${placeholders.join(", ")}`, + ...args + ); + insertedRows += batchRows; + logStage("insert_batch_progress", "end", { + insertedRows, + batchRows + }); + } + await executeTimed("commit", "COMMIT"); + } catch (err) { + await executeTimed("rollback", "ROLLBACK").catch(() => void 0); + throw err; + } + const scan = await executeTimed( + "scan_recent", + "SELECT id, length(payload) AS payload_bytes FROM pressure_rows ORDER BY id DESC LIMIT ?", + scanRows + ); + const bucketAgg = await executeTimed( + "bucket_agg", + "SELECT bucket, COUNT(*) AS rows, SUM(length(payload)) AS bytes FROM pressure_rows WHERE bucket BETWEEN ? AND ? GROUP BY bucket ORDER BY bucket", + input.cycle % 16, + input.cycle % 16 + 15 + ); + await executeTimed( + "touch_recent", + "UPDATE pressure_rows SET touched_count = touched_count + 1 WHERE id IN (SELECT id FROM pressure_rows ORDER BY id DESC LIMIT ?)", + Math.min(scanRows, insertRows) + ); + let deletedRows = 0; + const rowStatsRows = await executeTimed( + "row_stats", + "SELECT COUNT(*) AS active_rows, COALESCE(SUM(length(payload)), 0) AS active_bytes FROM pressure_rows" + ); + const rowStats = rowStatsRows[0]; + if (!rowStats) throw new Error("query returned no rows: row_stats"); + const integrityRows = await executeTimed( + "integrity_check", + "PRAGMA integrity_check" + ); + const integrity = integrityRows[0]; + if (!integrity) { + throw new Error("query returned no rows: integrity_check"); + } + const durationMs = performance.now() - startedAt; + await executeTimed( + "record_cycle", + "INSERT OR REPLACE INTO pressure_cycles (cycle, seed, inserted_rows, deleted_rows, active_rows, active_bytes, duration_ms, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + input.cycle, + input.seed, + insertedRows, + deletedRows, + rowStats.active_rows, + rowStats.active_bytes ?? 0, + durationMs, + now + ); + const storageStartedAt = performance.now(); + logStage("storage_stats", "start"); + const storage = await storageStats(c.db); + logStage("storage_stats", "end", { + durationMs: performance.now() - storageStartedAt, + pageCount: storage.page_count, + dbSizePages: storage.vfs?.dbSizePages ?? null, + pageCacheEntries: storage.vfs?.pageCacheEntries ?? null + }); + logStage("run_cycle", "end", { + durationMs, + activeRows: rowStats.active_rows, + activeBytes: rowStats.active_bytes ?? 0, + pageCount: storage.page_count + }); + return { + seed: input.seed, + cycle: input.cycle, + insertedRows, + deletedRows, + activeRows: rowStats.active_rows, + activeBytes: rowStats.active_bytes ?? 0, + scannedRows: scan.length, + bucketsRead: bucketAgg.length, + integrityCheck: integrity.integrity_check, + storage, + durationMs + }; + } + } +}); + +// src/actors/testing/mock-agentic-loop.ts +import { + actor as actor54 +} from "rivetkit"; +import { db as db11 } from "rivetkit/db"; +var DEFAULT_SLEEP_GRACE_PERIOD_MS = 12e4; +var DEFAULT_ON_SLEEP_DELAY_MS = 0; +var debugSocketsByActorId = /* @__PURE__ */ new Map(); +function sleep3(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +function positiveInteger3(value, name) { + if (!Number.isInteger(value) || value < 1) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} +function stringValue(value, name) { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`${name} must be a non-empty string`); + } + return value; +} +function typedRows2(rows) { + return rows; +} +function numberFromEnv(name, fallback) { + const value = process.env[name]; + if (value === void 0 || value === "") return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 0) { + throw new Error(`${name} must be a finite non-negative number`); + } + return parsed; +} +function send(websocket, payload2) { + if (websocket.readyState !== 1) return; + websocket.send(JSON.stringify(payload2)); +} +function debugPayload(row, replayed) { + return { + type: "debugEvent", + eventId: row.event_id, + name: row.name, + actorId: row.actor_id, + connectionId: row.connection_id, + requestId: row.request_id, + details: JSON.parse(row.details_json), + createdAt: row.created_at, + replayed + }; +} +function publishDebugEvent(row) { + const sockets = debugSocketsByActorId.get(row.actor_id); + if (!sockets) return; + for (const socket of sockets) { + send(socket, debugPayload(row, false)); + } +} +function addDebugSocket(actorId, websocket) { + const sockets = debugSocketsByActorId.get(actorId) ?? /* @__PURE__ */ new Set(); + sockets.add(websocket); + debugSocketsByActorId.set(actorId, sockets); + return () => { + sockets.delete(websocket); + if (sockets.size === 0) { + debugSocketsByActorId.delete(actorId); + } + }; +} +async function recordDebugEvent(c, input) { + const row = { + event_id: crypto.randomUUID(), + name: input.name, + actor_id: c.actorId, + connection_id: input.connectionId ?? null, + request_id: input.requestId ?? null, + details_json: JSON.stringify(input.details ?? {}), + created_at: input.createdAt ?? Date.now() + }; + try { + await c.db.execute( + "INSERT INTO mock_agentic_debug_events (event_id, name, actor_id, connection_id, request_id, details_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + row.event_id, + row.name, + row.actor_id, + row.connection_id, + row.request_id, + row.details_json, + row.created_at + ); + publishDebugEvent(row); + } catch (error) { + c.log.warn({ + msg: "mock agentic debug event failed", + name: input.name, + err: error instanceof Error ? error.message : String(error) + }); + } +} +async function replayDebugEvents(database, websocket) { + const rows = typedRows2( + await database.execute(` + SELECT event_id, name, actor_id, connection_id, request_id, details_json, created_at + FROM ( + SELECT event_id, name, actor_id, connection_id, request_id, details_json, created_at + FROM mock_agentic_debug_events + ORDER BY created_at DESC + LIMIT 200 + ) + ORDER BY created_at ASC + `) + ); + for (const row of rows) { + send(websocket, debugPayload(row, true)); + } +} +function verifyEntryRows(rows, expectedSeconds) { + const seen = /* @__PURE__ */ new Set(); + const indexes = rows.map((row) => row.idx).sort((a, b) => a - b); + for (const idx of indexes) seen.add(idx); + const missing = []; + for (let idx = 1; idx <= expectedSeconds; idx += 1) { + if (!seen.has(idx)) missing.push(idx); + } + const contiguous = rows.length === expectedSeconds && missing.length === 0 && indexes.every((idx, offset) => idx === offset + 1); + return { + expectedSeconds, + count: rows.length, + contiguous, + missing, + indexes, + ok: contiguous + }; +} +function verifyAllRows(rows, expectedRequests) { + const expectedByRequest = new Map( + expectedRequests.map((request) => [request.requestId, request.seconds]) + ); + const rowsByRequest = /* @__PURE__ */ new Map(); + for (const row of rows) { + const requestRows = rowsByRequest.get(row.request_id) ?? []; + requestRows.push(row); + rowsByRequest.set(row.request_id, requestRows); + } + const requests = expectedRequests.map((request) => { + const result = verifyEntryRows( + rowsByRequest.get(request.requestId) ?? [], + request.seconds + ); + return { + requestId: request.requestId, + ...result + }; + }); + const unexpectedRequestIds = [...rowsByRequest.keys()].filter((requestId) => !expectedByRequest.has(requestId)).sort(); + const expectedTotalRows = expectedRequests.reduce( + (total, request) => total + request.seconds, + 0 + ); + const ok = unexpectedRequestIds.length === 0 && rows.length === expectedTotalRows && requests.every((request) => request.ok); + return { + type: "verifiedAll", + expectedRequests: expectedRequests.length, + expectedTotalRows, + totalRows: rows.length, + rows, + unexpectedRequestIds, + requests, + ok + }; +} +var mockAgenticLoop = actor54({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: DEFAULT_SLEEP_GRACE_PERIOD_MS + }, + db: db11({ + onMigrate: async (database) => { + await database.execute(` + CREATE TABLE IF NOT EXISTS mock_agentic_entries ( + request_id TEXT NOT NULL, + idx INTEGER NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (request_id, idx) + ) + `); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_mock_agentic_entries_created_at ON mock_agentic_entries(created_at)" + ); + await database.execute(` + CREATE TABLE IF NOT EXISTS mock_agentic_sleep_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + sleep_started_at INTEGER NOT NULL + ) + `); + await database.execute(` + CREATE TABLE IF NOT EXISTS mock_agentic_debug_events ( + event_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + actor_id TEXT NOT NULL, + connection_id TEXT, + request_id TEXT, + details_json TEXT NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await database.execute( + "CREATE INDEX IF NOT EXISTS idx_mock_agentic_debug_events_created_at ON mock_agentic_debug_events(created_at)" + ); + } + }), + async onWake(c) { + await recordDebugEvent(c, { + name: "onWake", + details: { + key: c.key, + name: c.name + } + }); + }, + async onSleep(c) { + const delayMs = numberFromEnv( + "MOCK_AGENTIC_ON_SLEEP_DELAY_MS", + DEFAULT_ON_SLEEP_DELAY_MS + ); + const sleepStartedAt = Date.now(); + await recordDebugEvent(c, { + name: "onSleepStart", + createdAt: sleepStartedAt, + details: { + delayMs + } + }); + await c.db.execute( + "INSERT OR REPLACE INTO mock_agentic_sleep_state (id, sleep_started_at) VALUES (1, ?)", + sleepStartedAt + ); + c.log.info({ + msg: "mock agentic loop onSleep delay", + delayMs, + sleepStartedAt + }); + await sleep3(delayMs); + await recordDebugEvent(c, { + name: "onSleepEnd", + details: { + delayMs, + sleepStartedAt, + elapsedMs: Date.now() - sleepStartedAt + } + }); + }, + async onRequest(c, request) { + const url = new URL(request.url); + if (url.pathname === "/bypass" || url.pathname === "/request/bypass") { + const [sleepState] = typedRows2( + await c.db.execute( + "SELECT sleep_started_at FROM mock_agentic_sleep_state WHERE id = 1" + ) + ); + return new Response(JSON.stringify({ + type: "bypass", + transport: "http", + sleepStarted: sleepState !== void 0, + sleepStartedAt: sleepState?.sleep_started_at ?? null, + timestamp: Date.now() + }), { + headers: { + "content-type": "application/json" + } + }); + } + return new Response("not found", { status: 404 }); + }, + onWebSocket(c, websocket) { + const connectionId = crypto.randomUUID(); + let activeInference; + const removeDebugSocket = addDebugSocket(c.actorId, websocket); + send(websocket, { + type: "hello", + connectionId, + timestamp: Date.now() + }); + void (async () => { + try { + await replayDebugEvents(c.db, websocket); + } catch (error) { + c.log.warn({ + msg: "mock agentic debug replay failed", + err: error instanceof Error ? error.message : String(error) + }); + } + await recordDebugEvent(c, { + name: "webSocketOpen", + connectionId + }); + })(); + const verify = async (requestId, expectedSeconds) => { + const rows = typedRows2( + await c.db.execute( + "SELECT request_id, idx, created_at FROM mock_agentic_entries WHERE request_id = ? ORDER BY idx ASC", + requestId + ) + ); + return { + type: "verified", + requestId, + ...verifyEntryRows(rows, expectedSeconds) + }; + }; + const sleepStatus = async () => { + const [sleepState] = typedRows2( + await c.db.execute( + "SELECT sleep_started_at FROM mock_agentic_sleep_state WHERE id = 1" + ) + ); + return { + sleepStarted: sleepState !== void 0, + sleepStartedAt: sleepState?.sleep_started_at ?? null + }; + }; + const runInference = async (requestId, seconds) => { + send(websocket, { + type: "started", + requestId, + seconds, + timestamp: Date.now() + }); + await c.db.execute( + "DELETE FROM mock_agentic_entries WHERE request_id = ?", + requestId + ); + for (let idx = 1; idx <= seconds; idx += 1) { + await sleep3(1e3); + const createdAt = Date.now(); + await c.db.execute( + "INSERT INTO mock_agentic_entries (request_id, idx, created_at) VALUES (?, ?, ?)", + requestId, + idx, + createdAt + ); + send(websocket, { + type: "progress", + requestId, + idx, + seconds, + createdAt + }); + } + const verification = await verify(requestId, seconds); + send(websocket, { + type: "done", + requestId, + seconds, + timestamp: Date.now(), + verification + }); + }; + websocket.addEventListener("message", async (event21) => { + try { + if (typeof event21.data !== "string") { + throw new Error("message data must be a JSON string"); + } + const message = JSON.parse(event21.data); + const type = stringValue(message.type, "type"); + if (type === "history") { + const rows = typedRows2( + await c.db.execute( + "SELECT request_id, idx, created_at FROM mock_agentic_entries ORDER BY created_at ASC, request_id ASC, idx ASC" + ) + ); + const [count] = typedRows2( + await c.db.execute( + "SELECT COUNT(*) AS count FROM mock_agentic_entries" + ) + ); + send(websocket, { + type: "history", + totalRows: count?.count ?? rows.length, + entries: rows, + timestamp: Date.now() + }); + return; + } + if (type === "ping") { + send(websocket, { + type: "pong", + probeId: stringValue(message.probeId, "probeId"), + ...await sleepStatus(), + timestamp: Date.now() + }); + return; + } + if (type === "verify") { + const requestId = stringValue(message.requestId, "requestId"); + const expectedSeconds = positiveInteger3( + message.expectedSeconds, + "expectedSeconds" + ); + send(websocket, await verify(requestId, expectedSeconds)); + return; + } + if (type === "infer") { + const requestId = stringValue(message.requestId, "requestId"); + const seconds = positiveInteger3(message.seconds, "seconds"); + await recordDebugEvent(c, { + name: "inferenceRequested", + connectionId, + requestId, + details: { + seconds + } + }); + const previousInference = activeInference; + const inference = (async () => { + await previousInference?.catch(() => void 0); + await runInference(requestId, seconds); + })(); + activeInference = inference; + await c.keepAwake(inference); + if (activeInference === inference) { + activeInference = void 0; + } + return; + } + throw new Error(`unknown message type: ${type}`); + } catch (error) { + send(websocket, { + type: "error", + message: error instanceof Error ? error.message : "unknown websocket error", + timestamp: Date.now() + }); + } + }); + websocket.addEventListener("close", async () => { + removeDebugSocket(); + await recordDebugEvent(c, { + name: "webSocketClose", + connectionId + }); + }); + }, + actions: { + verify: async (c, requestId, expectedSeconds) => { + const rows = typedRows2( + await c.db.execute( + "SELECT request_id, idx, created_at FROM mock_agentic_entries WHERE request_id = ? ORDER BY idx ASC", + requestId + ) + ); + return { + requestId, + expectedSeconds, + count: rows.length, + indexes: rows.map((row) => row.idx) + }; + }, + verifyAll: async (c, expectedRequests) => { + if (!Array.isArray(expectedRequests)) { + throw new Error("expectedRequests must be an array"); + } + for (const request of expectedRequests) { + stringValue(request.requestId, "requestId"); + positiveInteger3(request.seconds, "seconds"); + } + const rows = typedRows2( + await c.db.execute( + "SELECT request_id, idx, created_at FROM mock_agentic_entries ORDER BY request_id ASC, idx ASC" + ) + ); + return verifyAllRows(rows, expectedRequests); + } + } +}); + +// src/actors/testing/sleep-close-fuzz.ts +import { actor as actor55 } from "rivetkit"; +var sleepCloseFuzz = actor55({ + options: { + canHibernateWebSocket: false + }, + state: { + connectionCount: 0, + messageCount: 0 + }, + onWebSocket(c, websocket) { + c.state.connectionCount += 1; + const connectionId = crypto.randomUUID(); + websocket.send( + JSON.stringify({ + type: "welcome", + connectionId, + connectionCount: c.state.connectionCount + }) + ); + const interval = setInterval(() => { + if (websocket.readyState !== 1) return; + websocket.send( + JSON.stringify({ + type: "tick", + connectionId, + timestamp: Date.now() + }) + ); + }, 500); + websocket.addEventListener("message", (event21) => { + c.state.messageCount += 1; + websocket.send( + JSON.stringify({ + type: "echo", + connectionId, + received: event21.data + }) + ); + }); + websocket.addEventListener("close", () => { + clearInterval(interval); + c.state.connectionCount -= 1; + }); + }, + actions: { + getStats(c) { + return { + connectionCount: c.state.connectionCount, + messageCount: c.state.messageCount + }; + } + } +}); + +// src/actors/testing/load-test-agent.ts +import { actor as actor56 } from "rivetkit"; +import { db as db12 } from "rivetkit/db"; +var DEFAULT_TOKENS_PER_SECOND = 20; +var DEFAULT_DURATION_MS = 5e3; +function send2(websocket, payload2) { + if (websocket.readyState !== 1) return; + websocket.send(JSON.stringify(payload2)); +} +function parsePositiveNumber(value, name, fallback) { + if (value === void 0 || value === null) return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive number`); + } + return parsed; +} +function sleep4(ms, signal) { + if (signal.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const timeout = setTimeout(resolve, ms); + signal.addEventListener( + "abort", + () => { + clearTimeout(timeout); + resolve(); + }, + { once: true } + ); + }); +} +var loadTestAgent = actor56({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 3e4 + }, + db: db12({ + onMigrate: async (db16) => { + await db16.execute(` + CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + connection_id TEXT NOT NULL, + request_id TEXT NOT NULL, + token_index INTEGER NOT NULL, + token TEXT NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await db16.execute(` + CREATE INDEX IF NOT EXISTS messages_request_idx + ON messages (request_id, token_index) + `); + } + }), + state: { + connectionCount: 0, + inferenceCount: 0, + tokenCount: 0 + }, + onWebSocket(c, websocket) { + c.state.connectionCount += 1; + const connectionId = crypto.randomUUID(); + send2(websocket, { + type: "connected", + connectionId, + connectionCount: c.state.connectionCount, + timestamp: Date.now() + }); + websocket.addEventListener("message", async (event21) => { + try { + const message = typeof event21.data === "string" ? JSON.parse(event21.data) : void 0; + if (message && message.type === "ping") { + send2(websocket, { + type: "pong", + connectionId, + id: message.id, + timestamp: Date.now() + }); + return; + } + if (!message || message.type !== "inference") { + throw new Error("expected inference message"); + } + const requestId = typeof message.requestId === "string" && message.requestId ? message.requestId : crypto.randomUUID(); + const tokensPerSecond = parsePositiveNumber( + message.tokensPerSecond, + "tokensPerSecond", + DEFAULT_TOKENS_PER_SECOND + ); + const durationMs = parsePositiveNumber( + message.durationMs, + "durationMs", + DEFAULT_DURATION_MS + ); + const intervalMs = 1e3 / tokensPerSecond; + const targetTokens = Math.max( + 1, + Math.floor(durationMs / 1e3 * tokensPerSecond) + ); + const inference = (async () => { + c.state.inferenceCount += 1; + send2(websocket, { + type: "inference-start", + connectionId, + requestId, + tokensPerSecond, + durationMs, + targetTokens, + timestamp: Date.now() + }); + const startedAt = performance.now(); + for (let i = 0; i < targetTokens; i++) { + if (c.abortSignal.aborted || websocket.readyState !== 1) { + break; + } + const tokenIndex = i + 1; + const token = `token-${tokenIndex}`; + const createdAt = Date.now(); + await c.db.execute( + "INSERT INTO messages (connection_id, request_id, token_index, token, created_at) VALUES (?, ?, ?, ?, ?)", + connectionId, + requestId, + tokenIndex, + token, + createdAt + ); + c.state.tokenCount += 1; + send2(websocket, { + type: "token", + connectionId, + requestId, + tokenIndex, + token, + timestamp: createdAt + }); + const nextAt = startedAt + tokenIndex * intervalMs; + const delayMs = Math.max(0, nextAt - performance.now()); + if (delayMs > 0) { + await sleep4(delayMs, c.abortSignal); + } + } + send2(websocket, { + type: "inference-complete", + connectionId, + requestId, + tokenCount: targetTokens, + timestamp: Date.now() + }); + })(); + await c.keepAwake(inference); + } catch (error) { + send2(websocket, { + type: "error", + message: error instanceof Error ? error.message : "unknown websocket error", + timestamp: Date.now() + }); + } + }); + websocket.addEventListener("close", () => { + c.state.connectionCount -= 1; + }); + }, + actions: { + getStats(c) { + return { + connectionCount: c.state.connectionCount, + inferenceCount: c.state.inferenceCount, + tokenCount: c.state.tokenCount + }; + } + } +}); + +// src/actors/testing/load-test-agent-2.ts +import { actor as actor57 } from "rivetkit"; +import { db as db13 } from "rivetkit/db"; +var AsyncMutex = class { + locked = false; + waiters = []; + async acquire() { + if (!this.locked) { + this.locked = true; + return; + } + await new Promise((resolve) => this.waiters.push(resolve)); + this.locked = true; + } + release() { + const next = this.waiters.shift(); + if (next) { + next(); + return; + } + this.locked = false; + } +}; +function createSerializedDb(execute) { + const mutex = new AsyncMutex(); + let activeTransaction = null; + const createTransactionDb = () => { + const tx = Object.assign( + (query, ...values) => execute(query, ...values), + { + withTransaction: async (_stats, fn) => fn(tx) + } + ); + return tx; + }; + const queryWithMutex = async (query, ...values) => { + await mutex.acquire(); + try { + return await execute(query, ...values); + } finally { + mutex.release(); + } + }; + return Object.assign(queryWithMutex, { + withTransaction: async (stats, fn) => { + if (activeTransaction) { + return fn(activeTransaction); + } + await mutex.acquire(); + const tx = createTransactionDb(); + try { + await executeTrackedQuery(execute, stats, "transaction-begin", "BEGIN"); + activeTransaction = tx; + try { + const result = await fn(tx); + activeTransaction = null; + await executeTrackedQuery(execute, stats, "transaction-commit", "COMMIT"); + return result; + } catch (error) { + activeTransaction = null; + await executeTrackedQuery( + execute, + stats, + "transaction-rollback", + "ROLLBACK" + ); + throw error; + } + } finally { + activeTransaction = null; + mutex.release(); + } + } + }); +} +var MESSAGE_COUNT = 84; +var MESSAGE_TOOL_REF_COUNT = 122; +var TOOL_CALL_COUNT = 61; +var EXECUTOR_TOOL_COUNT = 42; +var THREAD_EVENT_COUNT = 233; +var MESSAGE_CONTENT_BYTES = 2600; +var THREAD_EVENT_PAYLOAD_BYTES = 1e3; +var TOOL_CALL_RESULT_BYTES = 2700; +var EXECUTOR_TOOL_SCHEMA_BYTES = 550; +var SLOW_QUERY_MS = 1e3; +function send3(websocket, message) { + if (websocket.readyState === 1) { + websocket.send(JSON.stringify(message)); + } +} +var loadTestAgent2 = actor57({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 1e3 + }, + state: { + runCount: 0, + wakeCount: 0, + queryStats: createAgentConcurrent2QueryStats() + }, + db: db13({ + onMigrate: async (database) => { + await createAgentConcurrent2Schema(database); + await seedAgentConcurrent2Data(database); + } + }), + vars: { + sql: null, + wakeStats: null, + wakeStartedAt: null, + wakeIteration: 0 + }, + onWebSocket: (c, websocket) => { + send3(websocket, { + type: "connected", + timestamp: Date.now() + }); + websocket.addEventListener("message", (event21) => { + const promise = handleAgentConcurrent2Message(c, websocket, event21.data); + void c.keepAwake(promise); + }); + }, + actions: { + run: async (c, clientId) => { + const runtime = ensureAgentConcurrent2Runtime(c); + c.state.runCount++; + runtime.vars.wakeIteration++; + const cycleStats = createAgentConcurrent2QueryStats(); + const stats = createAgentConcurrent2StatsSet( + cycleStats, + runtime.wakeStats, + c.state.queryStats + ); + const result = await runAgentConcurrent2Workload( + runtime.sql, + clientId ?? `agent2-action-${c.state.runCount}`, + 0, + stats + ); + return { + ...result, + stats: snapshotAgentConcurrent2Stats(c, cycleStats) + }; + }, + getRunCount: (c) => c.state.runCount, + sleep: (c) => { + c.sleep(); + return true; + } + } +}); +async function handleAgentConcurrent2Message(c, websocket, data) { + let trigger = "unknown"; + let cycleStats = null; + try { + const request = parseAgentConcurrent2Request(data); + trigger = request.type; + if (request.type === "ping") { + send3(websocket, { + type: "pong", + id: request.id, + timestamp: Date.now() + }); + return; + } + if (request.type === "force_sleep") { + send3(websocket, { type: "sleeping", timestamp: Date.now() }); + c.sleep(); + return; + } + const runtime = ensureAgentConcurrent2Runtime(c); + c.state.runCount++; + runtime.vars.wakeIteration++; + cycleStats = createAgentConcurrent2QueryStats(); + const stats = createAgentConcurrent2StatsSet( + cycleStats, + runtime.wakeStats, + c.state.queryStats + ); + if (request.type === "agent2_resume") { + const startedAt = performance.now(); + const result2 = await runCatchupSnapshot( + runtime.sql, + request.version, + stats + ); + send3(websocket, { + type: "agent2_result", + trigger: request.type, + totalMs: Math.round(performance.now() - startedAt), + results: [result2], + stats: snapshotAgentConcurrent2Stats(c, cycleStats) + }); + return; + } + const result = await runAgentConcurrent2Workload( + runtime.sql, + request.clientId, + request.staggerHandleMs ?? 0, + stats + ); + send3(websocket, { + type: "agent2_result", + trigger: request.type, + ...result, + stats: snapshotAgentConcurrent2Stats(c, cycleStats) + }); + } catch (error) { + send3(websocket, { + type: "agent2_error", + trigger, + error: error instanceof Error ? error.message : String(error), + ...cycleStats ? { stats: snapshotAgentConcurrent2Stats(c, cycleStats) } : {} + }); + } +} +function parseAgentConcurrent2Request(data) { + if (typeof data !== "string") { + throw new Error("agent concurrent 2 request must be a string"); + } + const parsed = JSON.parse(data); + if (!parsed || typeof parsed !== "object") { + throw new Error("agent concurrent 2 request must be an object"); + } + const request = parsed; + if (request.type === "ping") { + return { + type: "ping", + ...typeof request.id === "number" ? { id: request.id } : {} + }; + } + if (request.type === "force_sleep") { + return { type: "force_sleep" }; + } + if (request.type === "agent2_resume") { + return { type: "agent2_resume", version: numberField(request, "version") }; + } + if (request.type === "agent2_connect") { + return { + type: "agent2_connect", + clientId: stringField(request, "clientId"), + ...typeof request.staggerHandleMs === "number" ? { staggerHandleMs: request.staggerHandleMs } : {} + }; + } + throw new Error(`unknown agent concurrent 2 request type: ${String(request.type)}`); +} +function stringField(record, field) { + const value = record[field]; + if (typeof value !== "string" || value.length === 0) { + throw new Error(`agent concurrent 2 request ${field} must be a string`); + } + return value; +} +function numberField(record, field) { + const value = record[field]; + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`agent concurrent 2 request ${field} must be a finite number`); + } + return value; +} +function createAgentConcurrent2Db(db16) { + return createSerializedDb(async (query, ...values) => { + const converted = values.map( + (value) => typeof value === "boolean" ? value ? 1 : 0 : value + ); + return await db16.execute(query, ...converted); + }); +} +function ensureAgentConcurrent2Runtime(c) { + c.vars.sql ??= createAgentConcurrent2Db(c.db); + c.state.queryStats ??= createAgentConcurrent2QueryStats(); + c.state.wakeCount ??= 0; + if (!c.vars.wakeStats) { + c.vars.wakeStats = createAgentConcurrent2QueryStats(); + c.vars.wakeStartedAt = Date.now(); + c.vars.wakeIteration = 0; + c.state.wakeCount++; + } + return { + sql: c.vars.sql, + wakeStats: c.vars.wakeStats, + vars: c.vars + }; +} +function createAgentConcurrent2QueryStats() { + return { + total: 0, + reads: 0, + mutations: 0, + tx: 0, + other: 0, + rows: 0, + errors: 0, + slow: 0, + maxMs: 0, + maxStep: "", + byOperation: {}, + byTable: {} + }; +} +function createAgentConcurrent2StatsSet(cycle, wake, actor61) { + return { cycle, wake, actor: actor61 }; +} +function snapshotAgentConcurrent2Stats(c, cycle) { + return { + wakeIndex: c.state.wakeCount, + actorIteration: c.state.runCount, + wakeIteration: c.vars.wakeIteration, + cycle: cloneAgentConcurrent2QueryStats(cycle), + wake: cloneAgentConcurrent2QueryStats( + c.vars.wakeStats ?? createAgentConcurrent2QueryStats() + ), + actor: cloneAgentConcurrent2QueryStats(c.state.queryStats) + }; +} +function cloneAgentConcurrent2QueryStats(stats) { + return { + total: stats.total, + reads: stats.reads, + mutations: stats.mutations, + tx: stats.tx, + other: stats.other, + rows: stats.rows, + errors: stats.errors, + slow: stats.slow, + maxMs: stats.maxMs, + maxStep: stats.maxStep, + byOperation: { ...stats.byOperation }, + byTable: { ...stats.byTable } + }; +} +async function runAgentConcurrent2Workload(sql, clientId, staggerHandleMs, stats) { + const startedAt = performance.now(); + const buildToolPlanContext = runBuildToolPlanContext(sql, stats); + const catchupSnapshot = runCatchupSnapshot(sql, 0, stats); + const recoverToolCalls = runRecoverToolCalls(sql, stats); + const mutationMix = runMutationMix(sql, clientId, stats); + const handleExecutorConnect = delay2(staggerHandleMs).then( + () => runHandleClientConnect(sql, clientId, stats) + ); + const results = await Promise.all([ + handleExecutorConnect, + buildToolPlanContext, + catchupSnapshot, + recoverToolCalls, + mutationMix + ]); + return { + totalMs: Math.round(performance.now() - startedAt), + results + }; +} +async function runHandleClientConnect(sql, clientId, stats) { + const startedAt = performance.now(); + const steps = []; + const nextSeq = await sql.withTransaction(stats, async (tx) => { + const latestExecutor = await timedQuery( + tx, + stats, + steps, + "load-latest-executor-id", + `SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1` + ); + const latestExecutorId = String( + latestExecutor[0]?.executor_id ?? "seed-executor" + ); + await timedQuery( + tx, + stats, + steps, + "select-cached-executor-tools", + `SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`, + latestExecutorId + ); + const executorType = await timedQuery( + tx, + stats, + steps, + "select-executor-type", + `SELECT value FROM thread_meta_kv WHERE key = 'executor_type'` + ); + if (!executorType[0]?.value) { + await timedQuery( + tx, + stats, + steps, + "set-executor-type", + `INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES ('executor_type', ?, ?)`, + "local-client", + (/* @__PURE__ */ new Date()).toISOString() + ); + } + const sandboxIntent = await timedQuery( + tx, + stats, + steps, + "select-workspace-intent", + `SELECT value FROM thread_meta_kv WHERE key = 'workspace_intent'` + ); + if (hasPendingLaunch(sandboxIntent[0]?.value)) { + await timedQuery( + tx, + stats, + steps, + "clear-pending-launch", + `UPDATE thread_meta_kv SET value = ?, updated_at = ? WHERE key = 'workspace_intent'`, + JSON.stringify({ spec: null, pendingLaunch: null }), + (/* @__PURE__ */ new Date()).toISOString() + ); + } + const seqRows = await timedQuery( + tx, + stats, + steps, + "select-next-thread-event-seq", + `SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events` + ); + const seq = Number(seqRows[0]?.seq ?? 1); + await timedQuery( + tx, + stats, + steps, + "insert-client-connected-event", + `INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`, + seq, + "client_connected", + JSON.stringify({ type: "client_connected", clientId }), + (/* @__PURE__ */ new Date()).toISOString() + ); + return seq; + }); + steps.push({ + name: "transaction-total", + durationMs: Math.round(performance.now() - startedAt), + rowCount: nextSeq + }); + return { + name: "handle-client-connect", + totalMs: Math.round(performance.now() - startedAt), + steps + }; +} +async function runBuildToolPlanContext(sql, stats) { + const startedAt = performance.now(); + const steps = []; + const latestExecutor = await timedQuery( + sql, + stats, + steps, + "load-latest-executor-id", + `SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1` + ); + const latestExecutorId = String(latestExecutor[0]?.executor_id ?? "seed-executor"); + await timedQuery( + sql, + stats, + steps, + "select-executor-tools", + `SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`, + latestExecutorId + ); + await timedQuery( + sql, + stats, + steps, + "count-uncancelled-top-level", + `SELECT COUNT(*) as count FROM messages WHERE cancelled = 0 AND parent_tool_use_id IS NULL` + ); + const unresolvedRows = await timedQuery( + sql, + stats, + steps, + "find-unresolved-assistant-message", + `SELECT m.* + FROM message_tool_refs AS tool_use + JOIN messages AS m + ON m.message_id = tool_use.assistant_message_id + WHERE tool_use.block_type = 'tool_use' + AND tool_use.cancelled = 0 + AND m.cancelled = 0 + AND m.role = 'assistant' + AND m.parent_tool_use_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM message_tool_refs AS tool_result + JOIN messages AS tool_result_message + ON tool_result_message.message_id = tool_result.source_message_id + WHERE tool_result.assistant_message_id = tool_use.assistant_message_id + AND tool_result.block_type = 'tool_result' + AND tool_result.cancelled = 0 + AND tool_result.tool_use_id = tool_use.tool_use_id + AND tool_result_message.parent_tool_use_id IS NULL + ) + GROUP BY m.message_id + ORDER BY m.created_at DESC + LIMIT 1` + ); + const unresolvedMessageId = unresolvedRows[0]?.message_id; + if (typeof unresolvedMessageId === "string") { + await timedQuery( + sql, + stats, + steps, + "get-persisted-tool-result-ids", + `SELECT tool_result.tool_use_id + FROM message_tool_refs AS tool_result + JOIN messages AS tool_result_message + ON tool_result_message.message_id = tool_result.source_message_id + WHERE tool_result.assistant_message_id = ? + AND tool_result.block_type = 'tool_result' + AND tool_result.cancelled = 0 + AND tool_result_message.parent_tool_use_id IS NULL`, + unresolvedMessageId + ); + await timedQuery( + sql, + stats, + steps, + "get-tool-calls-by-message-id", + `SELECT * FROM tool_calls WHERE message_id = ?`, + unresolvedMessageId + ); + } + await timedQuery( + sql, + stats, + steps, + "is-last-message-cancelled-assistant", + `SELECT role, cancelled FROM messages + WHERE parent_tool_use_id IS NULL + ORDER BY created_at DESC + LIMIT 1` + ); + await timedQuery( + sql, + stats, + steps, + "get-last-uncancelled", + `SELECT m.* FROM messages m + WHERE m.cancelled = 0 AND m.parent_tool_use_id IS NULL + ORDER BY m.created_at DESC + LIMIT 1` + ); + return { + name: "build-tool-plan-context", + totalMs: Math.round(performance.now() - startedAt), + steps + }; +} +async function runCatchupSnapshot(sql, version, stats) { + const startedAt = performance.now(); + const steps = []; + await Promise.all([ + timedQuery( + sql, + stats, + steps, + "thread-events-list-since-version", + `SELECT seq, event_type, payload, created_at FROM thread_events WHERE seq > ? ORDER BY seq ASC`, + version + ), + timedQuery( + sql, + stats, + steps, + "environment-snapshot", + `SELECT snapshot FROM environment_snapshot WHERE id = 1` + ), + timedQuery( + sql, + stats, + steps, + "thread-settings-snapshot", + `SELECT settings FROM thread_settings_snapshot WHERE id = 1` + ), + timedQuery( + sql, + stats, + steps, + "retry-state", + `SELECT * FROM retry_state WHERE id = 1` + ), + timedQuery( + sql, + stats, + steps, + "queued-messages", + `SELECT * FROM queued_messages ORDER BY created_at ASC` + ), + timedQuery( + sql, + stats, + steps, + "executor-artifacts", + `SELECT artifact_key, data_type, length(content_base64) AS bytes, tool_call_id, updated_at FROM executor_artifacts ORDER BY updated_at ASC` + ), + timedQuery( + sql, + stats, + steps, + "tool-approvals", + `SELECT * FROM tool_approvals ORDER BY timestamp ASC` + ), + timedQuery( + sql, + stats, + steps, + "compaction-summaries", + `SELECT cut_message_id, created_at FROM compaction_summaries ORDER BY created_at ASC` + ), + timedQuery( + sql, + stats, + steps, + "executor-status", + `SELECT value FROM thread_meta_kv WHERE key = 'executor_status'` + ) + ]); + steps.sort((a, b) => b.durationMs - a.durationMs); + return { + name: "catchup-snapshot", + totalMs: Math.round(performance.now() - startedAt), + steps + }; +} +async function runRecoverToolCalls(sql, stats) { + const startedAt = performance.now(); + const steps = []; + await timedQuery( + sql, + stats, + steps, + "hydrate-tool-progress", + `SELECT id, progress + FROM tool_calls + WHERE progress IS NOT NULL + AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')` + ); + await timedQuery( + sql, + stats, + steps, + "get-pending-tool-calls", + `SELECT * FROM tool_calls + WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running') + ORDER BY issued_at ASC` + ); + await timedQuery( + sql, + stats, + steps, + "get-next-tool-expiry", + `SELECT MIN(expires_at) AS expires_at + FROM tool_calls + WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')` + ); + return { + name: "recover-tool-calls", + totalMs: Math.round(performance.now() - startedAt), + steps + }; +} +async function runMutationMix(sql, clientId, stats) { + const startedAt = performance.now(); + const steps = []; + const writeCount = await sql.withTransaction(stats, async (tx) => { + const now = (/* @__PURE__ */ new Date()).toISOString(); + const suffix = safeId(clientId); + const seqRows = await timedQuery( + tx, + stats, + steps, + "select-max-thread-event-seq", + `SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events` + ); + const seq = Number(seqRows[0]?.seq ?? 1); + const lastMessageRows = await timedQuery( + tx, + stats, + steps, + "select-last-message-created-at", + `SELECT MAX(created_at) AS created_at FROM messages` + ); + const latestToolRows = await timedQuery( + tx, + stats, + steps, + "select-existing-tool-call", + `SELECT id FROM tool_calls ORDER BY issued_at DESC LIMIT 1` + ); + await timedQuery( + tx, + stats, + steps, + "select-sandbox-row", + `SELECT sandbox_id, restart_attempts, traffic_access_token, project_id, repository_url, additional_repositories, setup + FROM e2b_sandbox + WHERE id = 1` + ); + const messageIdValue = `agent2-message-${suffix}-${seq}`; + const toolUseIdValue = `agent2-tool-${suffix}-${seq}`; + const toolCallIdValue = `agent2-call-${suffix}-${seq}`; + const latestToolCallId = String(latestToolRows[0]?.id ?? toolUseID(1)); + const lastCreatedAt = String(lastMessageRows[0]?.created_at ?? now); + await timedQuery( + tx, + stats, + steps, + "upsert-agent-state", + `INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES (?, ?, ?)`, + "last_agent_state", + JSON.stringify({ status: "working", clientId, lastCreatedAt }), + now + ); + await timedQuery( + tx, + stats, + steps, + "insert-work-event", + `INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`, + seq, + "message_added", + JSON.stringify({ type: "message_added", messageId: messageIdValue }), + now + ); + await timedQuery( + tx, + stats, + steps, + "insert-message", + `INSERT INTO messages (role, content, meta, user_state, message_id, created_at, cancelled, parent_tool_use_id, tool_result_for_message_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "assistant", + "agent concurrent 2 mutation payload", + JSON.stringify({ clientId, seq }), + null, + messageIdValue, + now, + 0, + null, + null + ); + await timedQuery( + tx, + stats, + steps, + "delete-message-tool-refs", + `DELETE FROM message_tool_refs WHERE source_message_id = ?`, + messageIdValue + ); + await timedQuery( + tx, + stats, + steps, + "insert-message-added-event", + `INSERT OR IGNORE INTO message_added_events (message_id, seq) VALUES (?, ?)`, + messageIdValue, + seq + ); + await timedQuery( + tx, + stats, + steps, + "insert-message-tool-ref", + `INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled) + VALUES (?, ?, ?, ?, ?)`, + messageIdValue, + messageIdValue, + toolUseIdValue, + "tool_use", + 0 + ); + await timedQuery( + tx, + stats, + steps, + "insert-tool-call", + `INSERT OR IGNORE INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + toolCallIdValue, + `provider-${toolCallIdValue}`, + "tool_1", + JSON.stringify({ path: `/tmp/${toolCallIdValue}` }), + "seed-executor", + messageIdValue, + now, + null, + "running", + null, + JSON.stringify({ pct: 0.5, clientId }), + null + ); + await timedQuery( + tx, + stats, + steps, + "update-tool-call-progress", + `UPDATE tool_calls SET progress = ? WHERE id = ? AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + JSON.stringify({ pct: 0.75, clientId, updatedAt: now }), + toolCallIdValue + ); + await timedQuery( + tx, + stats, + steps, + "update-existing-tool-call-progress", + `UPDATE tool_calls SET progress = ? WHERE id = ? AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + JSON.stringify({ pct: 0.25, clientId, updatedAt: now }), + latestToolCallId + ); + return seq; + }); + steps.push({ + name: "transaction-total", + durationMs: Math.round(performance.now() - startedAt), + rowCount: writeCount + }); + return { + name: "mutation-mix", + totalMs: Math.round(performance.now() - startedAt), + steps + }; +} +async function timedQuery(sql, stats, steps, name, query, ...values) { + const startedAt = performance.now(); + try { + const rows = await sql(query, ...values); + const durationMs = Math.round(performance.now() - startedAt); + recordAgentConcurrent2Query(stats, name, query, durationMs, rows.length, false); + steps.push({ + name, + durationMs, + rowCount: rows.length + }); + return rows; + } catch (error) { + const durationMs = Math.round(performance.now() - startedAt); + recordAgentConcurrent2Query(stats, name, query, durationMs, 0, true); + throw error; + } +} +async function executeTrackedQuery(execute, stats, name, query, ...values) { + const startedAt = performance.now(); + try { + const rows = await execute(query, ...values); + recordAgentConcurrent2Query( + stats, + name, + query, + Math.round(performance.now() - startedAt), + rows.length, + false + ); + return rows; + } catch (error) { + recordAgentConcurrent2Query( + stats, + name, + query, + Math.round(performance.now() - startedAt), + 0, + true + ); + throw error; + } +} +function recordAgentConcurrent2Query(stats, name, query, durationMs, rowCount, failed) { + const classification = classifyAgentConcurrent2Query(query); + for (const target of [stats.cycle, stats.wake, stats.actor]) { + target.total++; + target.rows += rowCount; + if (failed) target.errors++; + if (durationMs >= SLOW_QUERY_MS) target.slow++; + if (durationMs > target.maxMs) { + target.maxMs = durationMs; + target.maxStep = `${name}:${classification.table}`; + } + target.byOperation[classification.operation] = (target.byOperation[classification.operation] ?? 0) + 1; + target.byTable[classification.table] = (target.byTable[classification.table] ?? 0) + 1; + if (classification.kind === "read") { + target.reads++; + } else if (classification.kind === "mutation") { + target.mutations++; + } else if (classification.kind === "tx") { + target.tx++; + } else { + target.other++; + } + } +} +function classifyAgentConcurrent2Query(query) { + const normalized = query.trim().replace(/\s+/g, " "); + const operation = normalized.match(/^([a-z]+)/i)?.[1]?.toLowerCase() ?? "other"; + const table = extractAgentConcurrent2Table(normalized, operation); + if (operation === "select") { + return { operation, kind: "read", table }; + } + if (operation === "insert" || operation === "update" || operation === "delete" || operation === "replace") { + return { operation, kind: "mutation", table }; + } + if (operation === "begin" || operation === "commit" || operation === "rollback") { + return { operation, kind: "tx", table }; + } + return { operation, kind: "other", table }; +} +function extractAgentConcurrent2Table(query, operation) { + const lower = query.toLowerCase(); + if (operation === "select") { + return firstMatch(lower, /\bfrom\s+([a-z0-9_]+)/) ?? "unknown"; + } + if (operation === "insert" || operation === "replace") { + return firstMatch(lower, /\binto\s+([a-z0-9_]+)/) ?? "unknown"; + } + if (operation === "update") { + return firstMatch(lower, /\bupdate\s+([a-z0-9_]+)/) ?? "unknown"; + } + if (operation === "delete") { + return firstMatch(lower, /\bfrom\s+([a-z0-9_]+)/) ?? "unknown"; + } + if (operation === "begin" || operation === "commit" || operation === "rollback") { + return "transaction"; + } + return "unknown"; +} +function firstMatch(value, pattern) { + return pattern.exec(value)?.[1] ?? null; +} +function hasPendingLaunch(value) { + if (typeof value !== "string" || value.length === 0) { + return false; + } + try { + const parsed = JSON.parse(value); + return parsed.pendingLaunch !== null && parsed.pendingLaunch !== void 0; + } catch { + return false; + } +} +function delay2(ms) { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms))); +} +async function createAgentConcurrent2Schema(database) { + await database.execute(`CREATE TABLE IF NOT EXISTS executor_tools ( + executor_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + schema TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (executor_id, tool_name) + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_executor_tools_executor ON executor_tools(executor_id)` + ); + await database.execute(`CREATE TABLE IF NOT EXISTS thread_meta_kv ( + key TEXT PRIMARY KEY, + value TEXT, + updated_at TEXT NOT NULL + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS thread_events ( + seq INTEGER PRIMARY KEY, + event_type TEXT NOT NULL, + payload TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_thread_events_seq ON thread_events(seq)` + ); + await database.execute(`CREATE TABLE IF NOT EXISTS message_added_events ( + message_id TEXT PRIMARY KEY, + seq INTEGER NOT NULL UNIQUE + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS messages ( + message_id TEXT PRIMARY KEY, + role TEXT NOT NULL CHECK(role IN ('user', 'assistant', 'info')), + content TEXT NOT NULL, + meta TEXT, + user_state TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + cancelled INTEGER NOT NULL DEFAULT 0, + read_at TEXT, + parent_tool_use_id TEXT, + tool_result_for_message_id TEXT + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_role_cancelled_created_at ON messages(parent_tool_use_id, role, cancelled, created_at DESC)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_cancelled_created_at ON messages(parent_tool_use_id, cancelled, created_at DESC)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_created_at ON messages(parent_tool_use_id, created_at DESC)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_role_created_at ON messages(role, created_at)` + ); + await database.execute(`CREATE TABLE IF NOT EXISTS message_tool_refs ( + source_message_id TEXT NOT NULL, + assistant_message_id TEXT NOT NULL, + tool_use_id TEXT NOT NULL, + block_type TEXT NOT NULL CHECK(block_type IN ('tool_use', 'tool_result')), + cancelled INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (source_message_id, block_type, tool_use_id) + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_assistant_lookup ON message_tool_refs(assistant_message_id, block_type, cancelled, tool_use_id)` + ); + await database.execute( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_message_tool_refs_live_tool_result ON message_tool_refs(assistant_message_id, tool_use_id) WHERE block_type = 'tool_result' AND cancelled = 0` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_source_message ON message_tool_refs(source_message_id)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_tool_use_lookup ON message_tool_refs(tool_use_id, assistant_message_id) WHERE block_type = 'tool_use' AND cancelled = 0` + ); + await database.execute(`CREATE TABLE IF NOT EXISTS tool_calls ( + id TEXT PRIMARY KEY, + provider_tool_use_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + args TEXT NOT NULL, + executor_id TEXT, + message_id TEXT NOT NULL, + issued_at TEXT NOT NULL, + expires_at TEXT, + state TEXT NOT NULL CHECK(state IN ('queued', 'pending_reconnect', 'pending_ack', 'running', 'completed', 'expired', 'revoked')), + result TEXT, + progress TEXT, + completed_at TEXT + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_message_id ON tool_calls(message_id)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_state ON tool_calls(state)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_expires_at ON tool_calls(expires_at) WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS environment_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), snapshot TEXT NOT NULL, updated_at TEXT NOT NULL)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS thread_settings_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), settings TEXT NOT NULL, updated_at TEXT NOT NULL)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS retry_state (id INTEGER PRIMARY KEY CHECK (id = 1), attempt INTEGER NOT NULL DEFAULT 0, scheduled_at INTEGER NOT NULL, reason TEXT NOT NULL)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS queued_messages (message_id TEXT PRIMARY KEY, content TEXT NOT NULL, user_state TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), steer INTEGER NOT NULL DEFAULT 0, user_meta TEXT)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS executor_artifacts (artifact_key TEXT PRIMARY KEY, data_type TEXT NOT NULL, content_base64 TEXT NOT NULL, tool_call_id TEXT, updated_at TEXT NOT NULL)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS e2b_sandbox ( + id INTEGER PRIMARY KEY CHECK (id = 1), + sandbox_id TEXT, + restart_attempts INTEGER NOT NULL DEFAULT 0, + traffic_access_token TEXT, + project_id TEXT, + repository_url TEXT, + additional_repositories TEXT, + setup TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS tool_approvals (id TEXT PRIMARY KEY, tool_call_id TEXT NOT NULL UNIQUE, tool_name TEXT NOT NULL, args TEXT NOT NULL, reason TEXT, to_allow TEXT, context TEXT NOT NULL CHECK(context IN ('thread', 'subagent')), subagent_tool_name TEXT, parent_tool_call_id TEXT, timestamp INTEGER NOT NULL, matched_rule TEXT, rule_source TEXT CHECK(rule_source IN ('user', 'built-in')))` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_approvals_timestamp ON tool_approvals(timestamp)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS compaction_summaries (summary_id TEXT PRIMARY KEY, summary_text TEXT NOT NULL, cut_message_id TEXT NOT NULL, created_at TEXT NOT NULL)` + ); +} +async function seedAgentConcurrent2Data(database) { + const existing = await database.execute(`SELECT COUNT(*) AS count FROM messages`); + if (Number(existing[0]?.count ?? 0) > 0) { + return; + } + const now = (/* @__PURE__ */ new Date("2026-05-16T03:58:18.661Z")).getTime(); + const text2 = (size) => "x".repeat(size); + const isoAt = (index) => new Date(now + index * 1e3).toISOString(); + await batchInsert2(database, `INSERT INTO thread_meta_kv (key, value, updated_at)`, [ + ["executor_type", "local-client", isoAt(0)], + ["workspace_intent", JSON.stringify({ spec: null, pendingLaunch: null }), isoAt(0)], + ["executor_status", JSON.stringify({ available: true, message: "ready" }), isoAt(0)] + ]); + const messageRows = []; + for (let index = 1; index <= MESSAGE_COUNT; index++) { + const role = index % 2 === 0 ? "assistant" : "user"; + messageRows.push([ + messageId(index), + role, + text2(MESSAGE_CONTENT_BYTES), + null, + null, + isoAt(index), + 0, + null, + null, + null + ]); + } + await batchInsert2( + database, + `INSERT INTO messages (message_id, role, content, meta, user_state, created_at, cancelled, read_at, parent_tool_use_id, tool_result_for_message_id)`, + messageRows, + 20 + ); + const messageToolRefRows = []; + for (let index = 0; index < MESSAGE_TOOL_REF_COUNT / 2; index++) { + const assistantIndex = 2 + index % 42 * 2; + const sourceIndex = Math.max(1, assistantIndex - 1); + const resultIndex = Math.min(MESSAGE_COUNT, assistantIndex + 1); + const toolUseId = toolUseID(index + 1); + messageToolRefRows.push([ + messageId(sourceIndex), + messageId(assistantIndex), + toolUseId, + "tool_use", + 0 + ]); + messageToolRefRows.push([ + messageId(resultIndex), + messageId(assistantIndex), + toolUseId, + "tool_result", + 0 + ]); + } + await batchInsert2( + database, + `INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled)`, + messageToolRefRows, + 50 + ); + const toolCallRows = []; + for (let index = 1; index <= TOOL_CALL_COUNT; index++) { + const assistantIndex = 2 + (index - 1) % 42 * 2; + toolCallRows.push([ + toolUseID(index), + `provider-${index}`, + `tool_${index % 21}`, + JSON.stringify({ path: `/tmp/file-${index}` }), + "seed-executor", + messageId(assistantIndex), + isoAt(index), + null, + "completed", + JSON.stringify({ + ok: true, + run: { status: "done", result: text2(TOOL_CALL_RESULT_BYTES) } + }), + null, + isoAt(index + 100) + ]); + } + await batchInsert2( + database, + `INSERT INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at)`, + toolCallRows, + 20 + ); + const executorToolRows = []; + for (let index = 1; index <= EXECUTOR_TOOL_COUNT; index++) { + const schema = JSON.stringify({ + name: `tool_${index}`, + description: text2(EXECUTOR_TOOL_SCHEMA_BYTES), + input_schema: { type: "object", properties: {} } + }); + executorToolRows.push(["seed-executor", `tool_${index}`, schema, isoAt(index)]); + } + await batchInsert2( + database, + `INSERT INTO executor_tools (executor_id, tool_name, schema, updated_at)`, + executorToolRows, + 42 + ); + const threadEventRows = []; + for (let index = 1; index <= THREAD_EVENT_COUNT; index++) { + threadEventRows.push([ + index, + index % 3 === 0 ? "message_added" : "agent_state_changed", + JSON.stringify({ type: "seed_event", body: text2(THREAD_EVENT_PAYLOAD_BYTES) }), + isoAt(index) + ]); + } + await batchInsert2( + database, + `INSERT INTO thread_events (seq, event_type, payload, created_at)`, + threadEventRows, + 25 + ); + const messageAddedRows = []; + for (let index = 1; index <= MESSAGE_COUNT; index++) { + messageAddedRows.push([messageId(index), index]); + } + await batchInsert2( + database, + `INSERT INTO message_added_events (message_id, seq)`, + messageAddedRows, + 50 + ); + await database.execute( + `INSERT INTO environment_snapshot (id, snapshot, updated_at) VALUES (1, ?, ?)`, + JSON.stringify({ cwd: "/workspace", body: text2(3620) }), + isoAt(0) + ); + await database.execute( + `INSERT INTO thread_settings_snapshot (id, settings, updated_at) VALUES (1, ?, ?)`, + JSON.stringify({ maxTokens: 2e4, body: text2(55) }), + isoAt(0) + ); + await database.execute( + `INSERT INTO e2b_sandbox (id, sandbox_id, restart_attempts, traffic_access_token, project_id, repository_url, additional_repositories, setup, created_at, updated_at) + VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "sandbox-seed", + 0, + "token-seed", + "project-seed", + "https://example.invalid/repo.git", + JSON.stringify([]), + JSON.stringify({ commands: [] }), + isoAt(0), + isoAt(0) + ); +} +async function batchInsert2(database, insertPrefix, rows, batchSize = 100) { + if (rows.length === 0) { + return; + } + const columnCount = rows[0]?.length ?? 0; + if (columnCount === 0) { + return; + } + const rowPlaceholder = `(${"?,".repeat(columnCount).slice(0, -1)})`; + for (let index = 0; index < rows.length; index += batchSize) { + const chunk = rows.slice(index, index + batchSize); + const values = chunk.map(() => rowPlaceholder).join(","); + const bindings = chunk.flat(); + await database.execute(`${insertPrefix} VALUES ${values}`, ...bindings); + } +} +function messageId(index) { + return `M-${String(index).padStart(22, "0")}`; +} +function toolUseID(index) { + return `toolu_${String(index).padStart(22, "0")}`; +} +function safeId(value) { + return value.replace(/[^a-zA-Z0-9_-]/g, "-").slice(0, 80); +} + +// src/actors/testing/sigterm-sleep-probe.ts +import { actor as actor58 } from "rivetkit"; +import { db as db14 } from "rivetkit/db"; +var DEFAULT_ON_SLEEP_DURATION_MS = 5e3; +var DEFAULT_ON_SLEEP_TICK_MS = 1e3; +var SLEEP_TIMEOUT_MS = 10 * 60 * 1e3; +var SLEEP_GRACE_PERIOD_MS = 30 * 60 * 1e3; +var ACTOR_STOPPED_CLOSE_CODE = 1e3; +var ACTOR_STOPPED_CLOSE_REASON = "actor stopped"; +function sleep5(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +function formatError(error) { + if (error instanceof Error) return error.stack ?? error.message; + return String(error); +} +var sigtermSleepProbe = actor58({ + state: { + label: "unprepared", + wakeCount: 0, + sleepCount: 0, + onSleepDurationMs: DEFAULT_ON_SLEEP_DURATION_MS, + onSleepTickMs: DEFAULT_ON_SLEEP_TICK_MS, + connectionCount: 0, + messageCount: 0, + onSleepStartedAt: null, + onSleepAsyncFinishedAt: null, + onSleepFinishedAt: null, + onSleepLastError: null + }, + createVars: () => ({ + websockets: /* @__PURE__ */ new Set() + }), + db: db14({ + onMigrate: async (database) => { + await database.execute(` + CREATE TABLE IF NOT EXISTS sigterm_sleep_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + event TEXT NOT NULL, + sleep_count INTEGER NOT NULL, + detail TEXT, + created_at INTEGER NOT NULL + ) + `); + } + }), + onWebSocket: (c, websocket) => { + c.vars.websockets.add(websocket); + c.state.connectionCount += 1; + const connectionId = crypto.randomUUID(); + c.log.info({ + msg: "sigterm sleep probe websocket connected", + label: c.state.label, + connectionId, + connectionCount: c.state.connectionCount + }); + websocket.send( + JSON.stringify({ + type: "welcome", + connectionId, + label: c.state.label, + connectionCount: c.state.connectionCount + }) + ); + websocket.addEventListener("message", (event21) => { + c.state.messageCount += 1; + const data = event21.data; + if (typeof data !== "string") return; + try { + const parsed = JSON.parse(data); + if (parsed.type === "ping") { + websocket.send( + JSON.stringify({ + type: "pong", + connectionId, + messageCount: c.state.messageCount, + timestamp: Date.now() + }) + ); + return; + } + } catch { + } + websocket.send( + JSON.stringify({ + type: "echo", + connectionId, + received: data, + messageCount: c.state.messageCount, + timestamp: Date.now() + }) + ); + }); + websocket.addEventListener("close", (event21) => { + c.vars.websockets.delete(websocket); + c.state.connectionCount -= 1; + c.log.info({ + msg: "sigterm sleep probe websocket closed", + label: c.state.label, + connectionId, + connectionCount: c.state.connectionCount, + code: event21.code, + reason: event21.reason + }); + }); + }, + onWake: async (c) => { + c.state.wakeCount += 1; + c.log.info({ + msg: "sigterm sleep probe onWake", + label: c.state.label, + wakeCount: c.state.wakeCount, + sleepCount: c.state.sleepCount + }); + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "wake", + c.state.sleepCount, + `wake-${c.state.wakeCount}`, + Date.now() + ); + }, + onSleep: async (c) => { + const sleepCount = c.state.sleepCount + 1; + const startedAt = Date.now(); + c.state.sleepCount = sleepCount; + c.state.onSleepStartedAt = startedAt; + c.state.onSleepAsyncFinishedAt = null; + c.state.onSleepFinishedAt = null; + c.state.onSleepLastError = null; + c.log.info({ + msg: "sigterm sleep probe onSleep start", + label: c.state.label, + sleepCount, + onSleepDurationMs: c.state.onSleepDurationMs, + onSleepTickMs: c.state.onSleepTickMs + }); + try { + for (const websocket of c.vars.websockets) { + if (websocket.readyState !== 1) continue; + websocket.send( + JSON.stringify({ + type: "onSleepStarted", + sleepCount, + onSleepDurationMs: c.state.onSleepDurationMs, + onSleepTickMs: c.state.onSleepTickMs, + timestamp: startedAt + }) + ); + } + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "on-sleep-start", + sleepCount, + c.state.label, + startedAt + ); + const deadline = startedAt + c.state.onSleepDurationMs; + let tickIndex = 0; + while (Date.now() < deadline) { + const waitMs = Math.min( + c.state.onSleepTickMs, + Math.max(0, deadline - Date.now()) + ); + if (waitMs > 0) await sleep5(waitMs); + tickIndex += 1; + const tickAt = Date.now(); + const detail = `tick=${tickIndex} elapsed-ms=${tickAt - startedAt}`; + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "on-sleep-tick", + sleepCount, + detail, + tickAt + ); + c.log.info({ + msg: "sigterm sleep probe onSleep tick", + label: c.state.label, + sleepCount, + tickIndex, + elapsedMs: tickAt - startedAt + }); + for (const websocket of c.vars.websockets) { + if (websocket.readyState !== 1) continue; + websocket.send( + JSON.stringify({ + type: "onSleepTick", + sleepCount, + tickIndex, + elapsedMs: tickAt - startedAt, + timestamp: tickAt + }) + ); + } + } + const asyncFinishedAt = Date.now(); + c.state.onSleepAsyncFinishedAt = asyncFinishedAt; + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "on-sleep-after-await", + sleepCount, + `delay-ms=${asyncFinishedAt - startedAt}`, + asyncFinishedAt + ); + const finishedAt = Date.now(); + c.state.onSleepFinishedAt = finishedAt; + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "on-sleep-finish", + sleepCount, + c.state.label, + finishedAt + ); + for (const websocket of c.vars.websockets) { + if (websocket.readyState !== 1) continue; + websocket.send( + JSON.stringify({ + type: "onSleepFinished", + sleepCount, + elapsedMs: finishedAt - startedAt, + timestamp: finishedAt + }) + ); + websocket.close( + ACTOR_STOPPED_CLOSE_CODE, + ACTOR_STOPPED_CLOSE_REASON + ); + } + c.log.info({ + msg: "sigterm sleep probe onSleep finish", + label: c.state.label, + sleepCount, + elapsedMs: finishedAt - startedAt + }); + } catch (error) { + const message = formatError(error); + c.state.onSleepLastError = message; + c.log.error({ + msg: "sigterm sleep probe onSleep error", + label: c.state.label, + sleepCount, + error: message + }); + throw error; + } + }, + actions: { + prepare: async (c, label = `sigterm-sleep-probe-${Date.now()}`, onSleepDurationMs = DEFAULT_ON_SLEEP_DURATION_MS, onSleepTickMs = DEFAULT_ON_SLEEP_TICK_MS) => { + if (!Number.isFinite(onSleepDurationMs) || onSleepDurationMs < 0) { + throw new Error("onSleepDurationMs must be a finite non-negative number"); + } + if (!Number.isFinite(onSleepTickMs) || onSleepTickMs <= 0) { + throw new Error("onSleepTickMs must be a finite positive number"); + } + c.state.label = label; + c.state.onSleepDurationMs = onSleepDurationMs; + c.state.onSleepTickMs = onSleepTickMs; + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "prepared", + c.state.sleepCount, + label, + Date.now() + ); + return { + label: c.state.label, + onSleepDurationMs: c.state.onSleepDurationMs, + onSleepTickMs: c.state.onSleepTickMs, + wakeCount: c.state.wakeCount, + sleepCount: c.state.sleepCount, + connectionCount: c.state.connectionCount, + messageCount: c.state.messageCount + }; + }, + getProof: async (c) => { + const rows = await c.db.execute("SELECT * FROM sigterm_sleep_log ORDER BY id"); + return { + state: { + label: c.state.label, + wakeCount: c.state.wakeCount, + sleepCount: c.state.sleepCount, + onSleepDurationMs: c.state.onSleepDurationMs, + onSleepTickMs: c.state.onSleepTickMs, + connectionCount: c.state.connectionCount, + messageCount: c.state.messageCount, + onSleepStartedAt: c.state.onSleepStartedAt, + onSleepAsyncFinishedAt: c.state.onSleepAsyncFinishedAt, + onSleepFinishedAt: c.state.onSleepFinishedAt, + onSleepLastError: c.state.onSleepLastError + }, + rows + }; + } + }, + options: { + canHibernateWebSocket: false, + sleepTimeout: SLEEP_TIMEOUT_MS, + sleepGracePeriod: SLEEP_GRACE_PERIOD_MS + } +}); + +// src/actors/testing/slow-reconnect-actor.ts +import { actor as actor59, setup } from "rivetkit"; +import { db as db15 } from "rivetkit/db"; +var AsyncMutex2 = class { + locked = false; + waiters = []; + async acquire() { + if (!this.locked) { + this.locked = true; + return; + } + await new Promise((resolve) => this.waiters.push(resolve)); + this.locked = true; + } + release() { + const next = this.waiters.shift(); + if (next) { + next(); + return; + } + this.locked = false; + } +}; +function createDb(execute) { + const mutex = new AsyncMutex2(); + let activeTransaction = null; + const createTransactionDb = () => { + const tx = Object.assign( + (query, ...values) => execute(query, ...values), + { + withTransaction: async (fn) => fn(tx) + } + ); + return tx; + }; + const queryWithMutex = async (query, ...values) => { + if (activeTransaction) { + return activeTransaction(query, ...values); + } + await mutex.acquire(); + try { + return await execute(query, ...values); + } finally { + mutex.release(); + } + }; + const sql = Object.assign(queryWithMutex, { + withTransaction: async (fn) => { + if (activeTransaction) { + return fn(activeTransaction); + } + await mutex.acquire(); + const tx = createTransactionDb(); + try { + await execute("BEGIN"); + activeTransaction = tx; + try { + const result = await fn(tx); + activeTransaction = null; + await execute("COMMIT"); + return result; + } catch (error) { + activeTransaction = null; + await execute("ROLLBACK"); + throw error; + } + } finally { + activeTransaction = null; + mutex.release(); + } + } + }); + return sql; +} +var MESSAGE_COUNT2 = 84; +var MESSAGE_TOOL_REF_COUNT2 = 122; +var TOOL_CALL_COUNT2 = 61; +var EXECUTOR_TOOL_COUNT2 = 42; +var THREAD_EVENT_COUNT2 = 233; +var MESSAGE_CONTENT_BYTES2 = 10620; +var THREAD_EVENT_PAYLOAD_BYTES2 = 4036; +var TOOL_CALL_RESULT_BYTES2 = 10975; +var EXECUTOR_TOOL_SCHEMA_BYTES2 = 2235; +var slowReconnectActor = actor59({ + state: { runCount: 0 }, + db: db15({ + onMigrate: async (database) => { + await createSlowReconnectSchema(database); + } + }), + vars: { sql: null }, + onWebSocket: (c, ws) => { + const sock = ws; + if (sock.readyState === WebSocket.OPEN) { + sock.send("pong"); + } + ws.addEventListener("message", (event21) => { + const promise = handleSlowReconnectWebSocketMessage(c, sock, event21.data); + void c.keepAwake(promise); + }); + }, + actions: { + prepare: async (c) => { + await createSlowReconnectSchema(c.db); + return await seedSlowReconnectData(c.db); + }, + reproReconnect: async (c, clientId) => { + c.vars.sql ??= createSlowReconnectDb(c.db); + c.state.runCount++; + return await runReconnectRepro(c.vars.sql, clientId ?? `action-${c.state.runCount}`, 0); + }, + getRunCount: (c) => c.state.runCount, + sleep: (c) => { + c.sleep(); + return true; + } + } +}); +async function handleSlowReconnectWebSocketMessage(c, sock, data) { + if (data === "ping") { + if (sock.readyState === WebSocket.OPEN) { + sock.send("pong"); + } + return; + } + let trigger = "unknown"; + try { + const request = parseSlowReconnectRequest(data); + trigger = request.type; + c.vars.sql ??= createSlowReconnectDb(c.db); + c.state.runCount++; + if (request.type === "client_resume") { + const startedAt = performance.now(); + const result2 = await runCatchupSnapshot2(c.vars.sql, request.version); + sendJSON(sock, { + type: "slow_reconnect_result", + trigger: request.type, + totalMs: Math.round(performance.now() - startedAt), + results: [result2] + }); + return; + } + const clientId = request.type === "executor_connect" ? request.clientId : request.clientId ?? `slow-reconnect-${c.state.runCount}`; + const staggerHandleMs = request.type === "repro_reconnect" ? request.staggerHandleMs ?? 0 : 0; + const result = await runReconnectRepro(c.vars.sql, clientId, staggerHandleMs); + if (request.type === "executor_connect") { + sendJSON(sock, { + type: "executor_connected", + executorId: clientId, + registeredToolCount: EXECUTOR_TOOL_COUNT2, + guidanceInventory: [], + resumeBootstrap: true + }); + } + sendJSON(sock, { + type: "slow_reconnect_result", + trigger: request.type, + ...result + }); + } catch (error) { + sendJSON(sock, { + type: "slow_reconnect_error", + trigger, + error: error instanceof Error ? error.message : String(error) + }); + } +} +function parseSlowReconnectRequest(data) { + if (typeof data !== "string") { + throw new Error("slowReconnectActor request must be a string"); + } + const parsed = JSON.parse(data); + if (!parsed || typeof parsed !== "object") { + throw new Error("slowReconnectActor request must be an object"); + } + const request = parsed; + if (request.type === "client_resume") { + return { type: "client_resume", version: numberField2(request, "version") }; + } + if (request.type === "executor_connect") { + const executorType = request.executorType; + return { + type: "executor_connect", + clientId: stringField2(request, "clientId"), + ...executorType === "local-client" || executorType === "sandbox" || executorType === "virtual" ? { executorType } : {} + }; + } + if (request.type === "repro_reconnect") { + return { + type: "repro_reconnect", + ...typeof request.clientId === "string" ? { clientId: request.clientId } : {}, + ...typeof request.staggerHandleMs === "number" ? { staggerHandleMs: request.staggerHandleMs } : {} + }; + } + throw new Error(`Unknown slowReconnectActor request type: ${String(request.type)}`); +} +function stringField2(record, field) { + const value = record[field]; + if (typeof value !== "string" || value.length === 0) { + throw new Error(`slowReconnectActor request ${field} must be a non-empty string`); + } + return value; +} +function numberField2(record, field) { + const value = record[field]; + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`slowReconnectActor request ${field} must be a finite number`); + } + return value; +} +function sendJSON(sock, message) { + if (sock.readyState === WebSocket.OPEN) { + sock.send(JSON.stringify(message)); + } +} +function createSlowReconnectDb(db16) { + return createDb(async (query, ...values) => { + const converted = values.map( + (value) => typeof value === "boolean" ? value ? 1 : 0 : value + ); + return await db16.execute(query, ...converted); + }); +} +async function runReconnectRepro(sql, clientId, staggerHandleMs) { + const startedAt = performance.now(); + const buildToolPlanContext = runBuildToolPlanContext2(sql); + const catchupSnapshot = runCatchupSnapshot2(sql, 0); + const recoverToolCalls = runRecoverToolCalls2(sql); + const handleExecutorConnect = delay3(staggerHandleMs).then( + () => runHandleExecutorConnect(sql, clientId) + ); + const results = await Promise.all([ + handleExecutorConnect, + buildToolPlanContext, + catchupSnapshot, + recoverToolCalls + ]); + return { + totalMs: Math.round(performance.now() - startedAt), + results + }; +} +async function runHandleExecutorConnect(sql, clientId) { + const startedAt = performance.now(); + const steps = []; + const nextSeq = await sql.withTransaction(async (tx) => { + const latestExecutor = await timedQuery2( + tx, + steps, + "load-latest-executor-id", + `SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1` + ); + const latestExecutorId = String(latestExecutor[0]?.executor_id ?? "seed-executor"); + await timedQuery2( + tx, + steps, + "select-cached-executor-tools", + `SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`, + latestExecutorId + ); + const executorType = await timedQuery2( + tx, + steps, + "select-executor-type", + `SELECT value FROM thread_meta_kv WHERE key = 'executor_type'` + ); + if (!executorType[0]?.value) { + await timedQuery2( + tx, + steps, + "set-executor-type", + `INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES ('executor_type', ?, ?)`, + "local-client", + (/* @__PURE__ */ new Date()).toISOString() + ); + } + const sandboxIntent = await timedQuery2( + tx, + steps, + "select-sandbox-intent", + `SELECT value FROM thread_meta_kv WHERE key = 'sandbox_intent'` + ); + if (hasPendingLaunch2(sandboxIntent[0]?.value)) { + await timedQuery2( + tx, + steps, + "clear-pending-launch", + `UPDATE thread_meta_kv SET value = ?, updated_at = ? WHERE key = 'sandbox_intent'`, + JSON.stringify({ spec: null, pendingLaunch: null }), + (/* @__PURE__ */ new Date()).toISOString() + ); + } + const seqRows = await timedQuery2( + tx, + steps, + "select-next-thread-event-seq", + `SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events` + ); + const seq = Number(seqRows[0]?.seq ?? 1); + await timedQuery2( + tx, + steps, + "insert-executor-connected-event", + `INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`, + seq, + "executor_connected", + JSON.stringify({ type: "executor_connected", executorId: clientId }), + (/* @__PURE__ */ new Date()).toISOString() + ); + return seq; + }); + steps.push({ + name: "transaction-total", + durationMs: Math.round(performance.now() - startedAt), + rowCount: nextSeq + }); + return { + name: "handle-executor-connect", + totalMs: Math.round(performance.now() - startedAt), + steps + }; +} +async function runBuildToolPlanContext2(sql) { + const startedAt = performance.now(); + const steps = []; + const latestExecutor = await timedQuery2( + sql, + steps, + "load-latest-executor-id", + `SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1` + ); + const latestExecutorId = String(latestExecutor[0]?.executor_id ?? "seed-executor"); + await timedQuery2( + sql, + steps, + "select-executor-tools", + `SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`, + latestExecutorId + ); + await timedQuery2( + sql, + steps, + "count-uncancelled-top-level", + `SELECT COUNT(*) as count FROM messages WHERE cancelled = 0 AND parent_tool_use_id IS NULL` + ); + const unresolvedRows = await timedQuery2( + sql, + steps, + "find-unresolved-assistant-message", + `SELECT m.* + FROM message_tool_refs AS tool_use + JOIN messages AS m + ON m.message_id = tool_use.assistant_message_id + WHERE tool_use.block_type = 'tool_use' + AND tool_use.cancelled = 0 + AND m.cancelled = 0 + AND m.role = 'assistant' + AND m.parent_tool_use_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM message_tool_refs AS tool_result + JOIN messages AS tool_result_message + ON tool_result_message.message_id = tool_result.source_message_id + WHERE tool_result.assistant_message_id = tool_use.assistant_message_id + AND tool_result.block_type = 'tool_result' + AND tool_result.cancelled = 0 + AND tool_result.tool_use_id = tool_use.tool_use_id + AND tool_result_message.parent_tool_use_id IS NULL + ) + GROUP BY m.message_id + ORDER BY m.created_at DESC + LIMIT 1` + ); + const unresolvedMessageId = unresolvedRows[0]?.message_id; + if (typeof unresolvedMessageId === "string") { + await timedQuery2( + sql, + steps, + "get-persisted-tool-result-ids", + `SELECT tool_result.tool_use_id + FROM message_tool_refs AS tool_result + JOIN messages AS tool_result_message + ON tool_result_message.message_id = tool_result.source_message_id + WHERE tool_result.assistant_message_id = ? + AND tool_result.block_type = 'tool_result' + AND tool_result.cancelled = 0 + AND tool_result_message.parent_tool_use_id IS NULL`, + unresolvedMessageId + ); + await timedQuery2( + sql, + steps, + "get-tool-calls-by-message-id", + `SELECT * FROM tool_calls WHERE message_id = ?`, + unresolvedMessageId + ); + } + await timedQuery2( + sql, + steps, + "is-last-message-cancelled-assistant", + `SELECT role, cancelled FROM messages + WHERE parent_tool_use_id IS NULL + ORDER BY created_at DESC + LIMIT 1` + ); + await timedQuery2( + sql, + steps, + "get-last-uncancelled", + `SELECT m.* FROM messages m + WHERE m.cancelled = 0 AND m.parent_tool_use_id IS NULL + ORDER BY m.created_at DESC + LIMIT 1` + ); + return { + name: "build-tool-plan-context", + totalMs: Math.round(performance.now() - startedAt), + steps + }; +} +async function runCatchupSnapshot2(sql, version) { + const startedAt = performance.now(); + const steps = []; + await Promise.all([ + timedQuery2( + sql, + steps, + "thread-events-list-since-version", + `SELECT seq, event_type, payload, created_at FROM thread_events WHERE seq > ? ORDER BY seq ASC`, + version + ), + timedQuery2( + sql, + steps, + "environment-snapshot", + `SELECT snapshot FROM environment_snapshot WHERE id = 1` + ), + timedQuery2( + sql, + steps, + "thread-settings-snapshot", + `SELECT settings FROM thread_settings_snapshot WHERE id = 1` + ), + timedQuery2(sql, steps, "retry-state", `SELECT * FROM retry_state WHERE id = 1`), + timedQuery2( + sql, + steps, + "queued-messages", + `SELECT * FROM queued_messages ORDER BY created_at ASC` + ), + timedQuery2( + sql, + steps, + "executor-artifacts", + `SELECT artifact_key, data_type, length(content_base64) AS bytes, tool_call_id, updated_at FROM executor_artifacts ORDER BY updated_at ASC` + ), + timedQuery2(sql, steps, "tool-approvals", `SELECT * FROM tool_approvals ORDER BY timestamp ASC`), + timedQuery2( + sql, + steps, + "compaction-summaries", + `SELECT cut_message_id, created_at FROM compaction_summaries ORDER BY created_at ASC` + ), + timedQuery2( + sql, + steps, + "executor-status", + `SELECT value FROM thread_meta_kv WHERE key = 'executor_status'` + ) + ]); + steps.sort((a, b) => b.durationMs - a.durationMs); + return { name: "catchup-snapshot", totalMs: Math.round(performance.now() - startedAt), steps }; +} +async function runRecoverToolCalls2(sql) { + const startedAt = performance.now(); + const steps = []; + await timedQuery2( + sql, + steps, + "hydrate-tool-progress", + `SELECT id, progress + FROM tool_calls + WHERE progress IS NOT NULL + AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')` + ); + await timedQuery2( + sql, + steps, + "get-pending-tool-calls", + `SELECT * FROM tool_calls + WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running') + ORDER BY issued_at ASC` + ); + await timedQuery2( + sql, + steps, + "get-next-tool-expiry", + `SELECT MIN(expires_at) AS expires_at + FROM tool_calls + WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')` + ); + return { name: "recover-tool-calls", totalMs: Math.round(performance.now() - startedAt), steps }; +} +async function timedQuery2(sql, steps, name, query, ...values) { + const startedAt = performance.now(); + const rows = await sql(query, ...values); + steps.push({ + name, + durationMs: Math.round(performance.now() - startedAt), + rowCount: rows.length + }); + return rows; +} +function hasPendingLaunch2(value) { + if (typeof value !== "string" || value.length === 0) { + return false; + } + try { + const parsed = JSON.parse(value); + return parsed.pendingLaunch !== null && parsed.pendingLaunch !== void 0; + } catch { + return false; + } +} +function delay3(ms) { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms))); +} +async function createSlowReconnectSchema(database) { + await database.execute(`CREATE TABLE IF NOT EXISTS executor_tools ( + executor_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + schema TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (executor_id, tool_name) + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_executor_tools_executor ON executor_tools(executor_id)` + ); + await database.execute(`CREATE TABLE IF NOT EXISTS thread_meta_kv ( + key TEXT PRIMARY KEY, + value TEXT, + updated_at TEXT NOT NULL + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS thread_events ( + seq INTEGER PRIMARY KEY, + event_type TEXT NOT NULL, + payload TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + )`); + await database.execute(`CREATE INDEX IF NOT EXISTS idx_thread_events_seq ON thread_events(seq)`); + await database.execute(`CREATE TABLE IF NOT EXISTS message_added_events ( + message_id TEXT PRIMARY KEY, + seq INTEGER NOT NULL UNIQUE + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS messages ( + message_id TEXT PRIMARY KEY, + role TEXT NOT NULL CHECK(role IN ('user', 'assistant', 'info')), + content TEXT NOT NULL, + meta TEXT, + user_state TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + cancelled INTEGER NOT NULL DEFAULT 0, + read_at TEXT, + parent_tool_use_id TEXT, + tool_result_for_message_id TEXT + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_role_cancelled_created_at ON messages(parent_tool_use_id, role, cancelled, created_at DESC)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_cancelled_created_at ON messages(parent_tool_use_id, cancelled, created_at DESC)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_created_at ON messages(parent_tool_use_id, created_at DESC)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_role_created_at ON messages(role, created_at)` + ); + await database.execute(`CREATE TABLE IF NOT EXISTS message_tool_refs ( + source_message_id TEXT NOT NULL, + assistant_message_id TEXT NOT NULL, + tool_use_id TEXT NOT NULL, + block_type TEXT NOT NULL CHECK(block_type IN ('tool_use', 'tool_result')), + cancelled INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (source_message_id, block_type, tool_use_id) + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_assistant_lookup ON message_tool_refs(assistant_message_id, block_type, cancelled, tool_use_id)` + ); + await database.execute( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_message_tool_refs_live_tool_result ON message_tool_refs(assistant_message_id, tool_use_id) WHERE block_type = 'tool_result' AND cancelled = 0` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_source_message ON message_tool_refs(source_message_id)` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_tool_use_lookup ON message_tool_refs(tool_use_id, assistant_message_id) WHERE block_type = 'tool_use' AND cancelled = 0` + ); + await database.execute(`CREATE TABLE IF NOT EXISTS tool_calls ( + id TEXT PRIMARY KEY, + provider_tool_use_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + args TEXT NOT NULL, + executor_id TEXT, + message_id TEXT NOT NULL, + issued_at TEXT NOT NULL, + expires_at TEXT, + state TEXT NOT NULL CHECK(state IN ('queued', 'pending_reconnect', 'pending_ack', 'running', 'completed', 'expired', 'revoked')), + result TEXT, + progress TEXT, + completed_at TEXT + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_message_id ON tool_calls(message_id)` + ); + await database.execute(`CREATE INDEX IF NOT EXISTS idx_tool_calls_state ON tool_calls(state)`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_expires_at ON tool_calls(expires_at) WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS environment_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), snapshot TEXT NOT NULL, updated_at TEXT NOT NULL)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS thread_settings_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), settings TEXT NOT NULL, updated_at TEXT NOT NULL)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS retry_state (id INTEGER PRIMARY KEY CHECK (id = 1), attempt INTEGER NOT NULL DEFAULT 0, scheduled_at INTEGER NOT NULL, reason TEXT NOT NULL)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS queued_messages (message_id TEXT PRIMARY KEY, content TEXT NOT NULL, user_state TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), steer INTEGER NOT NULL DEFAULT 0, user_meta TEXT)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS executor_artifacts (artifact_key TEXT PRIMARY KEY, data_type TEXT NOT NULL, content_base64 TEXT NOT NULL, tool_call_id TEXT, updated_at TEXT NOT NULL)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS tool_approvals (id TEXT PRIMARY KEY, tool_call_id TEXT NOT NULL UNIQUE, tool_name TEXT NOT NULL, args TEXT NOT NULL, reason TEXT, to_allow TEXT, context TEXT NOT NULL CHECK(context IN ('thread', 'subagent')), subagent_tool_name TEXT, parent_tool_call_id TEXT, timestamp INTEGER NOT NULL, matched_rule TEXT, rule_source TEXT CHECK(rule_source IN ('user', 'built-in')))` + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_approvals_timestamp ON tool_approvals(timestamp)` + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS compaction_summaries (summary_id TEXT PRIMARY KEY, summary_text TEXT NOT NULL, cut_message_id TEXT NOT NULL, created_at TEXT NOT NULL)` + ); +} +async function seedSlowReconnectData(database) { + const existing = await database.execute(`SELECT COUNT(*) AS count FROM messages`); + if (Number(existing[0]?.count ?? 0) > 0) { + return { + seeded: false, + messages: MESSAGE_COUNT2, + toolCalls: TOOL_CALL_COUNT2, + threadEvents: THREAD_EVENT_COUNT2 + }; + } + const now = (/* @__PURE__ */ new Date("2026-05-16T03:58:18.661Z")).getTime(); + const text2 = (size) => "x".repeat(size); + const isoAt = (index) => new Date(now + index * 1e3).toISOString(); + await batchInsert3(database, `INSERT INTO thread_meta_kv (key, value, updated_at)`, [ + ["executor_type", "local-client", isoAt(0)], + ["sandbox_intent", JSON.stringify({ spec: null, pendingLaunch: null }), isoAt(0)], + ["executor_status", JSON.stringify({ available: true, message: "ready" }), isoAt(0)] + ]); + const messageRows = []; + for (let index = 1; index <= MESSAGE_COUNT2; index++) { + const role = index % 2 === 0 ? "assistant" : "user"; + messageRows.push([ + messageId2(index), + role, + text2(MESSAGE_CONTENT_BYTES2), + null, + null, + isoAt(index), + 0, + null, + null, + null + ]); + } + await batchInsert3( + database, + `INSERT INTO messages (message_id, role, content, meta, user_state, created_at, cancelled, read_at, parent_tool_use_id, tool_result_for_message_id)`, + messageRows, + 20 + ); + const messageToolRefRows = []; + for (let index = 0; index < MESSAGE_TOOL_REF_COUNT2 / 2; index++) { + const assistantIndex = 2 + index % 42 * 2; + const sourceIndex = Math.max(1, assistantIndex - 1); + const resultIndex = Math.min(MESSAGE_COUNT2, assistantIndex + 1); + const toolUseId = toolUseID2(index + 1); + messageToolRefRows.push([ + messageId2(sourceIndex), + messageId2(assistantIndex), + toolUseId, + "tool_use", + 0 + ]); + messageToolRefRows.push([ + messageId2(resultIndex), + messageId2(assistantIndex), + toolUseId, + "tool_result", + 0 + ]); + } + await batchInsert3( + database, + `INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled)`, + messageToolRefRows, + 50 + ); + const toolCallRows = []; + for (let index = 1; index <= TOOL_CALL_COUNT2; index++) { + const assistantIndex = 2 + (index - 1) % 42 * 2; + toolCallRows.push([ + toolUseID2(index), + `provider-${index}`, + `tool_${index % 21}`, + JSON.stringify({ path: `/tmp/file-${index}` }), + "seed-executor", + messageId2(assistantIndex), + isoAt(index), + null, + "completed", + JSON.stringify({ + ok: true, + run: { status: "done", result: text2(TOOL_CALL_RESULT_BYTES2) } + }), + null, + isoAt(index + 100) + ]); + } + await batchInsert3( + database, + `INSERT INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at)`, + toolCallRows, + 20 + ); + const executorToolRows = []; + for (let index = 1; index <= EXECUTOR_TOOL_COUNT2; index++) { + const schema = JSON.stringify({ + name: `tool_${index}`, + description: text2(EXECUTOR_TOOL_SCHEMA_BYTES2), + input_schema: { type: "object", properties: {} } + }); + executorToolRows.push(["seed-executor", `tool_${index}`, schema, isoAt(index)]); + } + await batchInsert3( + database, + `INSERT INTO executor_tools (executor_id, tool_name, schema, updated_at)`, + executorToolRows, + 42 + ); + const threadEventRows = []; + for (let index = 1; index <= THREAD_EVENT_COUNT2; index++) { + threadEventRows.push([ + index, + index % 3 === 0 ? "message_added" : "agent_state_changed", + JSON.stringify({ type: "seed_event", body: text2(THREAD_EVENT_PAYLOAD_BYTES2) }), + isoAt(index) + ]); + } + await batchInsert3( + database, + `INSERT INTO thread_events (seq, event_type, payload, created_at)`, + threadEventRows, + 25 + ); + const messageAddedRows = []; + for (let index = 1; index <= MESSAGE_COUNT2; index++) { + messageAddedRows.push([messageId2(index), index]); + } + await batchInsert3( + database, + `INSERT INTO message_added_events (message_id, seq)`, + messageAddedRows, + 50 + ); + await database.execute( + `INSERT INTO environment_snapshot (id, snapshot, updated_at) VALUES (1, ?, ?)`, + JSON.stringify({ cwd: "/workspace", body: text2(3620) }), + isoAt(0) + ); + await database.execute( + `INSERT INTO thread_settings_snapshot (id, settings, updated_at) VALUES (1, ?, ?)`, + JSON.stringify({ maxTokens: 2e4, body: text2(55) }), + isoAt(0) + ); + return { + seeded: true, + messages: MESSAGE_COUNT2, + toolCalls: TOOL_CALL_COUNT2, + threadEvents: THREAD_EVENT_COUNT2 + }; +} +async function batchInsert3(database, insertPrefix, rows, batchSize = 100) { + if (rows.length === 0) { + return; + } + const columnCount = rows[0]?.length ?? 0; + if (columnCount === 0) { + return; + } + const rowPlaceholder = `(${"?,".repeat(columnCount).slice(0, -1)})`; + for (let index = 0; index < rows.length; index += batchSize) { + const chunk = rows.slice(index, index + batchSize); + const values = chunk.map(() => rowPlaceholder).join(","); + const bindings = chunk.flat(); + await database.execute(`${insertPrefix} VALUES ${values}`, ...bindings); + } +} +function messageId2(index) { + return `M-${String(index).padStart(22, "0")}`; +} +function toolUseID2(index) { + return `toolu_${String(index).padStart(22, "0")}`; +} +var registry = setup({ + use: { slowReconnectActor }, + maxIncomingMessageSize: 5 * 1024 * 1024, + maxOutgoingMessageSize: 5 * 1024 * 1024 +}); +if (import.meta.main) { + registry.start(); +} + +// src/actors/ai/ai-agent.ts +import { openai } from "@ai-sdk/openai"; +import { generateText, tool } from "ai"; +import { actor as actor60, event as event20 } from "rivetkit"; +import { z } from "zod"; + +// src/actors/ai/my-tools.ts +async function getWeather(location) { + return { + location, + temperature: Math.floor(Math.random() * 30) + 10, + condition: ["sunny", "cloudy", "rainy", "snowy"][Math.floor(Math.random() * 4)], + humidity: Math.floor(Math.random() * 50) + 30 + }; +} + +// src/actors/ai/ai-agent.ts +var aiAgent = actor60({ + // Persistent state that survives restarts: https://rivet.dev/docs/actors/state + state: { + messages: [] + }, + events: { + messageReceived: event20() + }, + actions: { + // Callable functions from clients: https://rivet.dev/docs/actors/actions + getMessages: (c) => c.state.messages, + sendMessage: async (c, userMessage) => { + const userMsg = { + role: "user", + content: userMessage, + timestamp: Date.now() + }; + c.state.messages.push(userMsg); + const { text: text2 } = await generateText({ + model: openai("gpt-4o-mini"), + prompt: userMessage, + messages: c.state.messages, + tools: { + weather: tool({ + description: "Get the weather in a location", + parameters: z.object({ + location: z.string().describe( + "The location to get the weather for" + ) + }), + execute: async ({ location }) => { + return await getWeather(location); + } + }) + } + }); + const assistantMsg = { + role: "assistant", + content: text2, + timestamp: Date.now() + }; + c.state.messages.push(assistantMsg); + c.broadcast("messageReceived", assistantMsg); + return assistantMsg; + } + } +}); + +// src/index.ts +function numberFromEnv2(name, fallback) { + const value = process.env[name]; + if (value === void 0 || value === "") return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + throw new Error(`${name} must be a finite number`); + } + return parsed; +} +function serverlessPoolConfig() { + if (resolveMode() !== "serverless-local") return void 0; + const url = process.env.RIVET_SERVERLESS_URL ?? process.env.KITCHEN_SINK_SERVERLESS_URL ?? "http://127.0.0.1:3000/api/rivet"; + return { + name: process.env.RIVET_POOL, + url, + requestLifespan: numberFromEnv2( + "RIVET_SERVERLESS_REQUEST_LIFESPAN", + 15 * 60 + ), + drainGracePeriod: numberFromEnv2( + "RIVET_SERVERLESS_DRAIN_GRACE_PERIOD", + 15 * 60 + ), + metadataPollInterval: numberFromEnv2( + "RIVET_SERVERLESS_METADATA_POLL_INTERVAL_MS", + 1e3 + ), + metadata: { + source: "kitchen-sink", + smoke: "raw-websocket-serverless" + } + }; +} +var registry2 = setup2({ + configurePool: serverlessPoolConfig(), + serverless: { + publicToken: process.env.RIVET_PUBLIC_TOKEN ?? process.env.RIVET_TOKEN ?? "dev", + maxStartPayloadBytes: numberFromEnv2( + "RIVET_SERVERLESS_MAX_START_PAYLOAD_BYTES", + 16 * 1024 * 1024 + ) + }, + use: { + // Overview + state basics + counter, + counterConn, + counterWithParams, + counterWithLifecycle, + pingPongCounter, + // Core API + inputActor, + syncActionActor, + asyncActionActor, + promiseActor, + shortTimeoutActor, + longTimeoutActor, + defaultTimeoutActor, + syncTimeoutActor, + customTimeoutActor, + errorHandlingActor, + // State and storage + onStateChangeActor, + metadataActor, + staticVarActor, + nestedVarActor, + dynamicVarActor, + uniqueVarActor, + driverCtxActor, + kvActor, + largePayloadActor, + largePayloadConnActor, + sqliteRawActor, + sqliteDrizzleActor, + parallelismTest, + // Realtime and connections + connStateActor, + rejectConnectionActor, + requestAccessActor, + // HTTP and WebSocket + rawHttpActor, + rawHttpNoHandlerActor, + rawHttpVoidReturnActor, + rawHttpHonoActor, + rawHttpRequestPropertiesActor, + rawWebSocketActor, + rawWebSocketBinaryActor, + rawFetchCounter, + rawWebSocketChatRoom, + rawWebSocketServerlessSmoke, + tunnelStress, + // Lifecycle and scheduling + runWithTicks, + runWithQueueConsumer, + runWithEarlyExit, + runWithError, + runWithoutHandler, + sleep: sleep2, + sleepWithLongRpc, + sleepWithNoSleepOption, + sleepWithRawHttp, + sleepWithRawWebSocket, + scheduled, + destroyActor, + destroyObserver, + hibernationActor, + // Queues + worker, + workerTimeout, + // Workflows + timer, + order, + batch, + approval, + dashboard, + race, + payment, + workflowHistorySimple, + workflowHistoryLoop, + workflowHistoryJoin, + workflowHistoryRace, + workflowHistoryFull, + workflowHistoryInProgress, + workflowHistoryRetrying, + workflowHistoryFailed, + workflowCounterActor, + workflowQueueActor, + workflowSleepActor, + workflowQueueTimeoutActor, + // Inter-actor + inventory, + checkout, + // Testing fixtures + inlineClientActor, + testCounter, + testCounterSqlite, + testSqliteLoad, + testSqliteBench, + sqliteColdStartBench, + sqliteRealworldBench, + rawSqliteFuzzer, + sqliteMemoryPressure, + mockAgenticLoop, + sleepCloseFuzz, + loadTestAgent, + loadTestAgent2, + sigtermSleepProbe, + slowReconnectActor, + // AI + aiAgent + } +}); + +// src/server.ts +import { serve } from "@hono/node-server"; +import { Hono as Hono3 } from "hono"; +import * as v8 from "v8"; +var app = new Hono3(); +var port = Number.parseInt(process.env.PORT ?? "3000", 10); +var mode = resolveMode(); +process.on("exit", (code) => { + console.log(JSON.stringify({ kind: "process_exit", code, pid: process.pid })); +}); +if (process.env.SQLITE_MEMORY_SOAK_DIAGNOSTICS === "1") { + for (const signal of ["SIGINT", "SIGTERM"]) { + process.on(signal, () => { + console.log( + JSON.stringify({ + kind: "process_signal", + signal, + pid: process.pid, + ppid: process.ppid, + timestamp: (/* @__PURE__ */ new Date()).toISOString() + }) + ); + process.exit(signal === "SIGINT" ? 130 : 143); + }); + } +} +process.on("beforeExit", (code) => { + console.log(JSON.stringify({ kind: "process_before_exit", code, pid: process.pid })); +}); +process.on("uncaughtException", (error) => { + console.error( + JSON.stringify({ + kind: "uncaught_exception", + error: error.stack ?? error.message + }) + ); +}); +process.on("unhandledRejection", (reason) => { + console.error( + JSON.stringify({ + kind: "unhandled_rejection", + error: reason instanceof Error ? reason.stack ?? reason.message : String(reason) + }) + ); +}); +async function memoryBreakdown(forceGc) { + const gc = globalThis.gc; + if (forceGc && typeof gc === "function") gc(); + const memory = process.memoryUsage(); + const heap = v8.getHeapStatistics(); + const spaces = v8.getHeapSpaceStatistics(); + const nativeNonV8Estimate = Math.max(0, memory.rss - heap.total_heap_size); + return { + pid: process.pid, + timestamp: (/* @__PURE__ */ new Date()).toISOString(), + uptimeSeconds: process.uptime(), + gcRequested: forceGc, + gcAvailable: typeof gc === "function", + process: { + rssBytes: memory.rss, + heapTotalBytes: memory.heapTotal, + heapUsedBytes: memory.heapUsed, + externalBytes: memory.external, + arrayBuffersBytes: memory.arrayBuffers + }, + v8: { + totalHeapSizeBytes: heap.total_heap_size, + usedHeapSizeBytes: heap.used_heap_size, + heapSizeLimitBytes: heap.heap_size_limit, + mallocedMemoryBytes: heap.malloced_memory, + externalMemoryBytes: heap.external_memory, + peakMallocedMemoryBytes: heap.peak_malloced_memory, + spaces: spaces.map((space) => ({ + name: space.space_name, + sizeBytes: space.space_size, + usedBytes: space.space_used_size, + availableBytes: space.space_available_size, + physicalSizeBytes: space.physical_space_size + })) + }, + estimates: { + jsHeapResidentBytes: memory.heapTotal, + jsHeapUsedBytes: memory.heapUsed, + v8ExternalBytes: memory.external, + nativeNonV8ResidentEstimateBytes: nativeNonV8Estimate + }, + resourceUsage: process.resourceUsage() + }; +} +app.get("/debug/memory", async (c) => { + const forceGc = c.req.query("gc") === "1"; + return c.json(await memoryBreakdown(forceGc)); +}); +app.get("/health", () => registry2.routes.health()); +app.get("/metadata", () => registry2.routes.metadata()); +app.get("/metrics", (c) => registry2.routes.prometheusMetrics(c.req.raw)); +app.post("/debug/heap-snapshot", (c) => { + if (process.env.SQLITE_MEMORY_SOAK_DIAGNOSTICS !== "1") { + return c.json({ error: "disabled" }, 404); + } + const path = c.req.query("path"); + if (!path) { + return c.json({ error: "missing path" }, 400); + } + const writtenPath = v8.writeHeapSnapshot(path); + return c.json({ path: writtenPath }); +}); +app.use("*", async (c, next) => { + const startedAt = Date.now(); + await next(); +}); +if (mode === "serverful") { + registry2.start(); +} else { + app.all("/api/rivet/*", (c) => registry2.handler(c.req.raw)); + app.all("/api/rivet", (c) => registry2.handler(c.req.raw)); +} +var server = serve({ fetch: app.fetch, port }, () => { + if (mode === "serverful") { + console.log( + `kitchen sink (serverful) listening on http://127.0.0.1:${port}` + ); + } else { + console.log( + `kitchen sink (${mode}) listening on http://127.0.0.1:${port}/api/rivet` + ); + } +}); +var httpServer = server; +httpServer.requestTimeout = 0; +httpServer.headersTimeout = 0; +httpServer.keepAliveTimeout = 0; +httpServer.timeout = 0; +//# sourceMappingURL=server.mjs.map \ No newline at end of file diff --git a/examples/kitchen-sink/dist-server/server.mjs.map b/examples/kitchen-sink/dist-server/server.mjs.map new file mode 100644 index 0000000000..1430ac9547 --- /dev/null +++ b/examples/kitchen-sink/dist-server/server.mjs.map @@ -0,0 +1 @@ +{"version":3,"sources":["../src/index.ts","../src/mode.ts","../src/actors/counter/counter.ts","../src/actors/counter/counter-conn.ts","../src/actors/counter/conn-params.ts","../src/actors/counter/lifecycle.ts","../src/actors/counter/ping-pong-counter.ts","../src/actors/actions/action-inputs.ts","../src/actors/actions/action-types.ts","../src/actors/actions/action-timeout.ts","../src/actors/actions/error-handling.ts","../src/actors/state/actor-onstatechange.ts","../src/actors/state/metadata.ts","../src/actors/state/vars.ts","../src/actors/state/kv.ts","../src/actors/state/large-payloads.ts","../src/actors/state/sqlite-raw.ts","../src/actors/state/sqlite-drizzle/mod.ts","../src/actors/state/sqlite-drizzle/schema.ts","../src/actors/state/sqlite-drizzle/drizzle/meta/_journal.json","../src/actors/state/sqlite-drizzle/drizzle/0000_left_wrecking_crew.sql","../src/actors/state/sqlite-drizzle/drizzle/migrations.js","../src/actors/state/parallelism-test.ts","../src/actors/connections/conn-state.ts","../src/actors/connections/reject-connection.ts","../src/actors/connections/request-access.ts","../src/actors/http/raw-http.ts","../src/actors/http/raw-http-request-properties.ts","../src/actors/http/raw-websocket.ts","../src/actors/http/raw-fetch-counter.ts","../src/actors/http/raw-websocket-chat-room.ts","../src/actors/http/raw-websocket-serverless-smoke.ts","../src/actors/http/tunnel-stress.ts","../src/actors/lifecycle/run.ts","../src/actors/lifecycle/sleep.ts","../src/actors/lifecycle/scheduled.ts","../src/actors/lifecycle/destroy.ts","../src/actors/lifecycle/hibernation.ts","../src/actors/queue/worker.ts","../src/actors/queue/worker-timeout.ts","../src/actors/workflow/workflow-fixtures.ts","../src/actors/workflow/timer.ts","../src/actors/workflow/_helpers.ts","../src/actors/workflow/order.ts","../src/actors/workflow/batch.ts","../src/actors/workflow/approval.ts","../src/actors/workflow/dashboard.ts","../src/actors/workflow/race.ts","../src/actors/workflow/payment.ts","../src/actors/workflow/history-examples.ts","../src/actors/inter-actor/cross-actor-actions.ts","../src/actors/testing/inline-client.ts","../src/actors/testing/test-counter.ts","../src/actors/testing/test-counter-sqlite.ts","../src/actors/testing/test-sqlite-load.ts","../src/actors/testing/test-sqlite-bench.ts","../src/actors/testing/sqlite-cold-start-bench.ts","../src/actors/testing/sqlite-realworld-bench.ts","../src/actors/testing/raw-sqlite-fuzzer.ts","../src/actors/testing/sqlite-memory-pressure.ts","../src/actors/testing/mock-agentic-loop.ts","../src/actors/testing/sleep-close-fuzz.ts","../src/actors/testing/load-test-agent.ts","../src/actors/testing/load-test-agent-2.ts","../src/actors/testing/sigterm-sleep-probe.ts","../src/actors/testing/slow-reconnect-actor.ts","../src/actors/ai/ai-agent.ts","../src/actors/ai/my-tools.ts","../src/server.ts"],"sourcesContent":["import { setup } from \"rivetkit\";\nimport { resolveMode } from \"./mode.ts\";\n// Counter\nimport { counter } from \"./actors/counter/counter.ts\";\nimport { counterConn } from \"./actors/counter/counter-conn.ts\";\nimport { counterWithParams } from \"./actors/counter/conn-params.ts\";\nimport { counterWithLifecycle } from \"./actors/counter/lifecycle.ts\";\nimport { pingPongCounter } from \"./actors/counter/ping-pong-counter.ts\";\n// Actions\nimport { inputActor } from \"./actors/actions/action-inputs.ts\";\nimport {\n\tsyncActionActor,\n\tasyncActionActor,\n\tpromiseActor,\n} from \"./actors/actions/action-types.ts\";\nimport {\n\tshortTimeoutActor,\n\tlongTimeoutActor,\n\tdefaultTimeoutActor,\n\tsyncTimeoutActor,\n} from \"./actors/actions/action-timeout.ts\";\nimport {\n\terrorHandlingActor,\n\tcustomTimeoutActor,\n} from \"./actors/actions/error-handling.ts\";\n// State\nimport { onStateChangeActor } from \"./actors/state/actor-onstatechange.ts\";\nimport { metadataActor } from \"./actors/state/metadata.ts\";\nimport {\n\tstaticVarActor,\n\tnestedVarActor,\n\tdynamicVarActor,\n\tuniqueVarActor,\n\tdriverCtxActor,\n} from \"./actors/state/vars.ts\";\nimport { kvActor } from \"./actors/state/kv.ts\";\nimport {\n\tlargePayloadActor,\n\tlargePayloadConnActor,\n} from \"./actors/state/large-payloads.ts\";\nimport { sqliteRawActor } from \"./actors/state/sqlite-raw.ts\";\nimport { sqliteDrizzleActor } from \"./actors/state/sqlite-drizzle/mod.ts\";\nimport { parallelismTest } from \"./actors/state/parallelism-test.ts\";\n// Connections\nimport { connStateActor } from \"./actors/connections/conn-state.ts\";\nimport { rejectConnectionActor } from \"./actors/connections/reject-connection.ts\";\nimport { requestAccessActor } from \"./actors/connections/request-access.ts\";\n// HTTP\nimport {\n\trawHttpActor,\n\trawHttpNoHandlerActor,\n\trawHttpVoidReturnActor,\n\trawHttpHonoActor,\n} from \"./actors/http/raw-http.ts\";\nimport { rawHttpRequestPropertiesActor } from \"./actors/http/raw-http-request-properties.ts\";\nimport {\n\trawWebSocketActor,\n\trawWebSocketBinaryActor,\n} from \"./actors/http/raw-websocket.ts\";\nimport { rawFetchCounter } from \"./actors/http/raw-fetch-counter.ts\";\nimport { rawWebSocketChatRoom } from \"./actors/http/raw-websocket-chat-room.ts\";\nimport { rawWebSocketServerlessSmoke } from \"./actors/http/raw-websocket-serverless-smoke.ts\";\nimport { tunnelStress } from \"./actors/http/tunnel-stress.ts\";\n// Lifecycle\nimport {\n\trunWithTicks,\n\trunWithQueueConsumer,\n\trunWithEarlyExit,\n\trunWithError,\n\trunWithoutHandler,\n} from \"./actors/lifecycle/run.ts\";\nimport {\n\tsleep,\n\tsleepWithLongRpc,\n\tsleepWithNoSleepOption,\n\tsleepWithRawHttp,\n\tsleepWithRawWebSocket,\n} from \"./actors/lifecycle/sleep.ts\";\nimport { scheduled } from \"./actors/lifecycle/scheduled.ts\";\nimport {\n\tdestroyActor,\n\tdestroyObserver,\n} from \"./actors/lifecycle/destroy.ts\";\nimport { hibernationActor } from \"./actors/lifecycle/hibernation.ts\";\n// Queues\nimport { worker } from \"./actors/queue/worker.ts\";\nimport { workerTimeout } from \"./actors/queue/worker-timeout.ts\";\n// Workflows\nimport {\n\tworkflowCounterActor,\n\tworkflowQueueActor,\n\tworkflowSleepActor,\n\tworkflowQueueTimeoutActor,\n} from \"./actors/workflow/workflow-fixtures.ts\";\nimport { timer } from \"./actors/workflow/timer.ts\";\nimport { order } from \"./actors/workflow/order.ts\";\nimport { batch } from \"./actors/workflow/batch.ts\";\nimport { approval } from \"./actors/workflow/approval.ts\";\nimport { dashboard } from \"./actors/workflow/dashboard.ts\";\nimport { race } from \"./actors/workflow/race.ts\";\nimport { payment } from \"./actors/workflow/payment.ts\";\nimport {\n\tworkflowHistorySimple,\n\tworkflowHistoryLoop,\n\tworkflowHistoryJoin,\n\tworkflowHistoryRace,\n\tworkflowHistoryFull,\n\tworkflowHistoryInProgress,\n\tworkflowHistoryRetrying,\n\tworkflowHistoryFailed,\n} from \"./actors/workflow/history-examples.ts\";\n// Inter-actor\nimport {\n\tinventory,\n\tcheckout,\n} from \"./actors/inter-actor/cross-actor-actions.ts\";\n// Testing\nimport { inlineClientActor } from \"./actors/testing/inline-client.ts\";\nimport { testCounter } from \"./actors/testing/test-counter.ts\";\nimport { testCounterSqlite } from \"./actors/testing/test-counter-sqlite.ts\";\nimport { testSqliteLoad } from \"./actors/testing/test-sqlite-load.ts\";\nimport { testSqliteBench } from \"./actors/testing/test-sqlite-bench.ts\";\nimport { sqliteColdStartBench } from \"./actors/testing/sqlite-cold-start-bench.ts\";\nimport { sqliteRealworldBench } from \"./actors/testing/sqlite-realworld-bench.ts\";\nimport { rawSqliteFuzzer } from \"./actors/testing/raw-sqlite-fuzzer.ts\";\nimport { sqliteMemoryPressure } from \"./actors/testing/sqlite-memory-pressure.ts\";\nimport { mockAgenticLoop } from \"./actors/testing/mock-agentic-loop.ts\";\nimport { sleepCloseFuzz } from \"./actors/testing/sleep-close-fuzz.ts\";\nimport { loadTestAgent } from \"./actors/testing/load-test-agent.ts\";\nimport { loadTestAgent2 } from \"./actors/testing/load-test-agent-2.ts\";\nimport { sigtermSleepProbe } from \"./actors/testing/sigterm-sleep-probe.ts\";\nimport { slowReconnectActor } from \"./actors/testing/slow-reconnect-actor.ts\";\n// AI\nimport { aiAgent } from \"./actors/ai/ai-agent.ts\";\n\nfunction numberFromEnv(name: string, fallback: number): number {\n\tconst value = process.env[name];\n\tif (value === undefined || value === \"\") return fallback;\n\n\tconst parsed = Number(value);\n\tif (!Number.isFinite(parsed)) {\n\t\tthrow new Error(`${name} must be a finite number`);\n\t}\n\n\treturn parsed;\n}\n\nfunction serverlessPoolConfig() {\n\t// Only the local serverless mode self-registers its pool with the engine.\n\t// In the deployed `serverless` mode the pool is configured externally on\n\t// the engine cluster, and the `serverful` mode uses a long-lived runner\n\t// connection rather than a serverless pool.\n\tif (resolveMode() !== \"serverless-local\") return undefined;\n\n\tconst url =\n\t\tprocess.env.RIVET_SERVERLESS_URL ??\n\t\tprocess.env.KITCHEN_SINK_SERVERLESS_URL ??\n\t\t\"http://127.0.0.1:3000/api/rivet\";\n\n\treturn {\n\t\tname: process.env.RIVET_POOL,\n\t\turl,\n\t\trequestLifespan: numberFromEnv(\n\t\t\t\"RIVET_SERVERLESS_REQUEST_LIFESPAN\",\n\t\t\t15 * 60,\n\t\t),\n\t\tdrainGracePeriod: numberFromEnv(\n\t\t\t\"RIVET_SERVERLESS_DRAIN_GRACE_PERIOD\",\n\t\t\t15 * 60,\n\t\t),\n\t\tmetadataPollInterval: numberFromEnv(\n\t\t\t\"RIVET_SERVERLESS_METADATA_POLL_INTERVAL_MS\",\n\t\t\t1000,\n\t\t),\n\t\tmetadata: {\n\t\t\tsource: \"kitchen-sink\",\n\t\t\tsmoke: \"raw-websocket-serverless\",\n\t\t},\n\t};\n}\n\nexport const registry = setup({\n\tconfigurePool: serverlessPoolConfig(),\n\tserverless: {\n\t\tpublicToken:\n\t\t\tprocess.env.RIVET_PUBLIC_TOKEN ?? process.env.RIVET_TOKEN ?? \"dev\",\n\t\tmaxStartPayloadBytes: numberFromEnv(\n\t\t\t\"RIVET_SERVERLESS_MAX_START_PAYLOAD_BYTES\",\n\t\t\t16 * 1024 * 1024,\n\t\t),\n\t},\n\tuse: {\n\t\t// Overview + state basics\n\t\tcounter,\n\t\tcounterConn,\n\t\tcounterWithParams,\n\t\tcounterWithLifecycle,\n\t\tpingPongCounter,\n\t\t// Core API\n\t\tinputActor,\n\t\tsyncActionActor,\n\t\tasyncActionActor,\n\t\tpromiseActor,\n\t\tshortTimeoutActor,\n\t\tlongTimeoutActor,\n\t\tdefaultTimeoutActor,\n\t\tsyncTimeoutActor,\n\t\tcustomTimeoutActor,\n\t\terrorHandlingActor,\n\t\t// State and storage\n\t\tonStateChangeActor,\n\t\tmetadataActor,\n\t\tstaticVarActor,\n\t\tnestedVarActor,\n\t\tdynamicVarActor,\n\t\tuniqueVarActor,\n\t\tdriverCtxActor,\n\t\tkvActor,\n\t\tlargePayloadActor,\n\t\tlargePayloadConnActor,\n\t\tsqliteRawActor,\n\t\tsqliteDrizzleActor,\n\t\tparallelismTest,\n\t\t// Realtime and connections\n\t\tconnStateActor,\n\t\trejectConnectionActor,\n\t\trequestAccessActor,\n\t\t// HTTP and WebSocket\n\t\trawHttpActor,\n\t\trawHttpNoHandlerActor,\n\t\trawHttpVoidReturnActor,\n\t\trawHttpHonoActor,\n\t\trawHttpRequestPropertiesActor,\n\t\trawWebSocketActor,\n\t\trawWebSocketBinaryActor,\n\t\trawFetchCounter,\n\t\trawWebSocketChatRoom,\n\t\trawWebSocketServerlessSmoke,\n\t\ttunnelStress,\n\t\t// Lifecycle and scheduling\n\t\trunWithTicks,\n\t\trunWithQueueConsumer,\n\t\trunWithEarlyExit,\n\t\trunWithError,\n\t\trunWithoutHandler,\n\t\tsleep,\n\t\tsleepWithLongRpc,\n\t\tsleepWithNoSleepOption,\n\t\tsleepWithRawHttp,\n\t\tsleepWithRawWebSocket,\n\t\tscheduled,\n\t\tdestroyActor,\n\t\tdestroyObserver,\n\t\thibernationActor,\n\t\t// Queues\n\t\tworker,\n\t\tworkerTimeout,\n\t\t// Workflows\n\t\ttimer,\n\t\torder,\n\t\tbatch,\n\t\tapproval,\n\t\tdashboard,\n\t\trace,\n\t\tpayment,\n\t\tworkflowHistorySimple,\n\t\tworkflowHistoryLoop,\n\t\tworkflowHistoryJoin,\n\t\tworkflowHistoryRace,\n\t\tworkflowHistoryFull,\n\t\tworkflowHistoryInProgress,\n\t\tworkflowHistoryRetrying,\n\t\tworkflowHistoryFailed,\n\t\tworkflowCounterActor,\n\t\tworkflowQueueActor,\n\t\tworkflowSleepActor,\n\t\tworkflowQueueTimeoutActor,\n\t\t// Inter-actor\n\t\tinventory,\n\t\tcheckout,\n\t\t// Testing fixtures\n\t\tinlineClientActor,\n\t\ttestCounter,\n\t\ttestCounterSqlite,\n\t\ttestSqliteLoad,\n\t\ttestSqliteBench,\n\t\tsqliteColdStartBench,\n\t\tsqliteRealworldBench,\n\t\trawSqliteFuzzer,\n\t\tsqliteMemoryPressure,\n\t\tmockAgenticLoop,\n\t\tsleepCloseFuzz,\n\t\tloadTestAgent,\n\t\tloadTestAgent2,\n\t\tsigtermSleepProbe,\n\t\tslowReconnectActor,\n\t\t// AI\n\t\taiAgent,\n\t},\n});\n","export type KitchenSinkMode = \"serverless\" | \"serverful\" | \"serverless-local\";\n\nexport function resolveMode(): KitchenSinkMode {\n\tconst explicit = process.env.RIVET_KITCHEN_SINK_MODE;\n\tif (\n\t\texplicit === \"serverless\" ||\n\t\texplicit === \"serverful\" ||\n\t\texplicit === \"serverless-local\"\n\t) {\n\t\treturn explicit;\n\t}\n\tif (explicit !== undefined && explicit !== \"\") {\n\t\tthrow new Error(\n\t\t\t`RIVET_KITCHEN_SINK_MODE must be one of \"serverless\", \"serverful\", or \"serverless-local\" (got \"${explicit}\")`,\n\t\t);\n\t}\n\n\tif (process.env.RIVET_RUN_ENGINE === \"1\") return \"serverless-local\";\n\tif (process.env.RIVET_SERVERLESS_URL !== undefined) return \"serverless-local\";\n\tif (process.env.KITCHEN_SINK_SERVERLESS_URL !== undefined) {\n\t\treturn \"serverless-local\";\n\t}\n\n\treturn \"serverless\";\n}\n","import { actor, event, type RivetMessageEvent, type UniversalWebSocket } from \"rivetkit\";\n\nexport const counter = actor({\n\toptions: {\n\t\tcanHibernateWebSocket: false,\n\t\tsleepGracePeriod: 5_000,\n\t},\n\tstate: { count: 0 },\n\tevents: {\n\t\tnewCount: event(),\n\t},\n\tonWebSocket(_c, websocket: UniversalWebSocket) {\n\t\t// Plain echo for the rtt counter-latency harness. Any message in →\n\t\t// the same payload back out. No state mutation, no awaits — keeps the\n\t\t// echo path as close to raw WS RTT as possible.\n\t\twebsocket.addEventListener(\"message\", (event: RivetMessageEvent) => {\n\t\t\tif (websocket.readyState !== 1) return;\n\t\t\twebsocket.send(event.data as string | ArrayBuffer);\n\t\t});\n\t},\n\tactions: {\n\t\tincrement: (c, x: number) => {\n\t\t\tc.state.count += x;\n\t\t\tc.broadcast(\"newCount\", c.state.count);\n\t\t\treturn c.state.count;\n\t\t},\n\t\tsetCount: (c, x: number) => {\n\t\t\tc.state.count = x;\n\t\t\tc.broadcast(\"newCount\", x);\n\t\t\treturn c.state.count;\n\t\t},\n\t\tgetCount: (c) => {\n\t\t\treturn c.state.count;\n\t\t},\n\t\tnoop: (_c) => {\n\t\t\treturn { ok: true };\n\t\t},\n\t\tgoToSleep: (c) => {\n\t\t\tc.sleep();\n\t\t\treturn { ok: true };\n\t\t},\n\t},\n});\n","import { actor, event } from \"rivetkit\";\n\nexport const counterConn = actor({\n\tstate: {\n\t\tconnectionCount: 0,\n\t},\n\tevents: {\n\t\tnewCount: event(),\n\t},\n\tconnState: { count: 0 },\n\tonConnect: (c, conn) => {\n\t\tc.state.connectionCount += 1;\n\t},\n\tonDisconnect: (c, conn) => {\n\t\t// Note: We can't determine if disconnect was graceful from here\n\t\t// For testing purposes, we'll decrement on all disconnects\n\t\t// In real scenarios, you'd use connection tracking with timeouts\n\t\tc.state.connectionCount -= 1;\n\t},\n\tactions: {\n\t\tincrement: (c, x: number) => {\n\t\t\tc.conn.state.count += x;\n\t\t\tc.broadcast(\"newCount\", c.conn.state.count);\n\t\t},\n\t\tsetCount: (c, x: number) => {\n\t\t\tc.conn.state.count = x;\n\t\t\tc.broadcast(\"newCount\", x);\n\t\t},\n\t\tgetCount: (c) => {\n\t\t\treturn c.conn.state.count;\n\t\t},\n\t\tgetConnectionCount: (c) => {\n\t\t\treturn c.state.connectionCount;\n\t\t},\n\t},\n});\n","import { actor, event } from \"rivetkit\";\n\nexport const counterWithParams = actor({\n\tstate: { count: 0, initializers: [] as string[] },\n\tevents: {\n\t\tnewCount: event<{ count: number; by: string }>(),\n\t},\n\tcreateConnState: (c, params: { name?: string }) => {\n\t\treturn {\n\t\t\tname: params.name || \"anonymous\",\n\t\t};\n\t},\n\tonConnect: (c, conn) => {\n\t\t// Record connection name\n\t\tc.state.initializers.push(conn.state.name);\n\t},\n\tactions: {\n\t\tincrement: (c, x: number) => {\n\t\t\tc.state.count += x;\n\t\t\tc.broadcast(\"newCount\", {\n\t\t\t\tcount: c.state.count,\n\t\t\t\tby: c.conn.state.name,\n\t\t\t});\n\t\t\treturn c.state.count;\n\t\t},\n\t\tgetInitializers: (c) => {\n\t\t\treturn c.state.initializers;\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\n\ntype ConnParams = { trackLifecycle?: boolean } | undefined;\n\nexport const counterWithLifecycle = actor({\n\tstate: {\n\t\tcount: 0,\n\t\tevents: [] as string[],\n\t},\n\tcreateConnState: (c, params: ConnParams) => ({\n\t\tjoinTime: Date.now(),\n\t}),\n\tonWake: (c) => {\n\t\tc.state.events.push(\"onWake\");\n\t},\n\tonSleep: async (c) => {\n\t\tc.state.events.push(\"onSleep:start\");\n\t\tawait new Promise((resolve) => setTimeout(resolve, 1000));\n\t\tc.state.events.push(\"onSleep:end\");\n\t},\n\tonBeforeConnect: (c, params: ConnParams) => {\n\t\tif (params?.trackLifecycle) c.state.events.push(\"onBeforeConnect\");\n\t},\n\tonConnect: (c, conn) => {\n\t\tif (conn.params?.trackLifecycle) c.state.events.push(\"onConnect\");\n\t},\n\tonDisconnect: (c, conn) => {\n\t\tif (conn.params?.trackLifecycle) c.state.events.push(\"onDisconnect\");\n\t},\n\tactions: {\n\t\tgetEvents: (c) => {\n\t\t\treturn c.state.events;\n\t\t},\n\t\tincrement: (c, x: number) => {\n\t\t\tc.state.count += x;\n\t\t\treturn c.state.count;\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\n\nexport const pingPongCounter = actor({\n\tstate: {\n\t\tpingCount: 0,\n\t},\n\tonWebSocket(ctx, websocket) {\n\t\twebsocket.addEventListener(\"message\", (event: any) => {\n\t\t\tconst data = event.data;\n\t\t\tif (typeof data !== \"string\") return;\n\n\t\t\tlet parsed: any;\n\t\t\ttry {\n\t\t\t\tparsed = JSON.parse(data);\n\t\t\t} catch {\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tif (parsed?.type === \"ping\") {\n\t\t\t\tctx.state.pingCount = ctx.state.pingCount + 1;\n\t\t\t\twebsocket.send(\n\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\ttype: \"pong\",\n\t\t\t\t\t\tpingCount: ctx.state.pingCount,\n\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t}\n\t\t});\n\t},\n\tactions: {\n\t\tgetPingCount(c) {\n\t\t\treturn c.state.pingCount;\n\t\t},\n\t\tresetPingCount(c) {\n\t\t\tc.state.pingCount = 0;\n\t\t\treturn c.state.pingCount;\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\n\nexport interface State {\n\tinitialInput?: unknown;\n\tonCreateInput?: unknown;\n}\n\n// Test actor that can capture input during creation\nexport const inputActor = actor({\n\tcreateState: (c, input): State => {\n\t\treturn {\n\t\t\tinitialInput: input,\n\t\t\tonCreateInput: undefined,\n\t\t};\n\t},\n\n\tonCreate: (c, input) => {\n\t\tc.state.onCreateInput = input;\n\t},\n\n\tactions: {\n\t\tgetInputs: (c) => {\n\t\t\treturn {\n\t\t\t\tinitialInput: c.state.initialInput,\n\t\t\t\tonCreateInput: c.state.onCreateInput,\n\t\t\t};\n\t\t},\n\t},\n});\n","import { actor, UserError } from \"rivetkit\";\n\n// Actor with synchronous actions\nexport const syncActionActor = actor({\n\tstate: { value: 0 },\n\tactions: {\n\t\t// Simple synchronous action that returns a value directly\n\t\tincrement: (c, amount = 1) => {\n\t\t\tc.state.value += amount;\n\t\t\treturn c.state.value;\n\t\t},\n\t\t// Synchronous action that returns an object\n\t\tgetInfo: (c) => {\n\t\t\treturn {\n\t\t\t\tcurrentValue: c.state.value,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t};\n\t\t},\n\t\t// Synchronous action with no return value (void)\n\t\treset: (c) => {\n\t\t\tc.state.value = 0;\n\t\t},\n\t},\n});\n\n// Actor with asynchronous actions\nexport const asyncActionActor = actor({\n\tstate: { value: 0, data: null as any },\n\tactions: {\n\t\t// Async action with a delay\n\t\tdelayedIncrement: async (c, amount = 1) => {\n\t\t\tawait Promise.resolve();\n\t\t\tc.state.value += amount;\n\t\t\treturn c.state.value;\n\t\t},\n\t\t// Async action that simulates an API call\n\t\tfetchData: async (c, id: string) => {\n\t\t\tawait Promise.resolve();\n\n\t\t\t// Simulate response data\n\t\t\tconst data = { id, timestamp: Date.now() };\n\t\t\tc.state.data = data;\n\t\t\treturn data;\n\t\t},\n\t\t// Async action with error handling\n\t\tasyncWithError: async (c, shouldError: boolean) => {\n\t\t\tawait Promise.resolve();\n\n\t\t\tif (shouldError) {\n\t\t\t\tthrow new UserError(\"Intentional error\");\n\t\t\t}\n\n\t\t\treturn \"Success\";\n\t\t},\n\t},\n});\n\n// Actor with promise actions\nexport const promiseActor = actor({\n\tstate: { results: [] as string[] },\n\tactions: {\n\t\t// Action that returns a resolved promise\n\t\tresolvedPromise: (c) => {\n\t\t\treturn Promise.resolve(\"resolved value\");\n\t\t},\n\t\t// Action that returns a promise that resolves after a delay\n\t\tdelayedPromise: (c): Promise => {\n\t\t\treturn new Promise((resolve) => {\n\t\t\t\tc.state.results.push(\"delayed\");\n\t\t\t\tresolve(\"delayed value\");\n\t\t\t});\n\t\t},\n\t\t// Action that returns a rejected promise\n\t\trejectedPromise: (c) => {\n\t\t\treturn Promise.reject(new UserError(\"promised rejection\"));\n\t\t},\n\t\t// Action to check the collected results\n\t\tgetResults: (c) => {\n\t\t\treturn c.state.results;\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\n\n// Short timeout actor\nexport const shortTimeoutActor = actor({\n\tstate: { value: 0 },\n\toptions: {\n\t\tactionTimeout: 50, // 50ms timeout\n\t},\n\tactions: {\n\t\tquickAction: async (c) => {\n\t\t\treturn \"quick response\";\n\t\t},\n\t\tslowAction: async (c) => {\n\t\t\t// This action should timeout\n\t\t\tawait new Promise((resolve) => setTimeout(resolve, 100));\n\t\t\treturn \"slow response\";\n\t\t},\n\t},\n});\n\n// Long timeout actor\nexport const longTimeoutActor = actor({\n\tstate: { value: 0 },\n\toptions: {\n\t\tactionTimeout: 200, // 200ms timeout\n\t},\n\tactions: {\n\t\tdelayedAction: async (c) => {\n\t\t\t// This action should complete within timeout\n\t\t\tawait new Promise((resolve) => setTimeout(resolve, 100));\n\t\t\treturn \"delayed response\";\n\t\t},\n\t},\n});\n\n// Default timeout actor\nexport const defaultTimeoutActor = actor({\n\tstate: { value: 0 },\n\tactions: {\n\t\tnormalAction: async (c) => {\n\t\t\tawait new Promise((resolve) => setTimeout(resolve, 50));\n\t\t\treturn \"normal response\";\n\t\t},\n\t},\n});\n\n// Sync actor (timeout shouldn't apply)\nexport const syncTimeoutActor = actor({\n\tstate: { value: 0 },\n\toptions: {\n\t\tactionTimeout: 50, // 50ms timeout\n\t},\n\tactions: {\n\t\tsyncAction: (c) => {\n\t\t\treturn \"sync response\";\n\t\t},\n\t},\n});\n","import { actor, UserError } from \"rivetkit\";\n\nexport const errorHandlingActor = actor({\n\tstate: {\n\t\terrorLog: [] as string[],\n\t},\n\tactions: {\n\t\t// Action that throws a UserError with just a message\n\t\tthrowSimpleError: () => {\n\t\t\tthrow new UserError(\"Simple error message\");\n\t\t},\n\n\t\t// Action that throws a UserError with code and metadata\n\t\tthrowDetailedError: () => {\n\t\t\tthrow new UserError(\"Detailed error message\", {\n\t\t\t\tcode: \"detailed_error\",\n\t\t\t\tmetadata: {\n\t\t\t\t\treason: \"test\",\n\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t},\n\t\t\t});\n\t\t},\n\n\t\t// Action that throws an internal error\n\t\tthrowInternalError: () => {\n\t\t\tthrow new Error(\"This is an internal error\");\n\t\t},\n\n\t\t// Action that returns successfully\n\t\tsuccessfulAction: () => {\n\t\t\treturn \"success\";\n\t\t},\n\n\t\t// Action that times out (simulated with a long delay)\n\t\ttimeoutAction: async (c) => {\n\t\t\t// This action should time out if the timeout is configured\n\t\t\treturn new Promise((resolve) => {\n\t\t\t\tsetTimeout(() => {\n\t\t\t\t\tresolve(\"This should not be reached if timeout works\");\n\t\t\t\t}, 10000); // 10 seconds\n\t\t\t});\n\t\t},\n\n\t\t// Action with configurable delay to test timeout edge cases\n\t\tdelayedAction: async (c, delayMs: number) => {\n\t\t\treturn new Promise((resolve) => {\n\t\t\t\tsetTimeout(() => {\n\t\t\t\t\tresolve(`Completed after ${delayMs}ms`);\n\t\t\t\t}, delayMs);\n\t\t\t});\n\t\t},\n\n\t\t// Log an error for inspection\n\t\tlogError: (c, error: string) => {\n\t\t\tc.state.errorLog.push(error);\n\t\t\treturn c.state.errorLog;\n\t\t},\n\n\t\t// Get the error log\n\t\tgetErrorLog: (c) => {\n\t\t\treturn c.state.errorLog;\n\t\t},\n\n\t\t// Clear the error log\n\t\tclearErrorLog: (c) => {\n\t\t\tc.state.errorLog = [];\n\t\t\treturn true;\n\t\t},\n\t},\n\toptions: {\n\t\tactionTimeout: 500, // 500ms timeout for actions\n\t},\n});\n\n// Actor with custom timeout\nexport const customTimeoutActor = actor({\n\tstate: {},\n\tactions: {\n\t\tquickAction: async () => {\n\t\t\tawait new Promise((resolve) => setTimeout(resolve, 50));\n\t\t\treturn \"Quick action completed\";\n\t\t},\n\t\tslowAction: async () => {\n\t\t\tawait new Promise((resolve) => setTimeout(resolve, 300));\n\t\t\treturn \"Slow action completed\";\n\t\t},\n\t},\n\toptions: {\n\t\tactionTimeout: 200, // 200ms timeout\n\t},\n});\n","import { actor } from \"rivetkit\";\n\nexport const onStateChangeActor = actor({\n\tstate: {\n\t\tvalue: 0,\n\t},\n\tvars: {\n\t\tchangeCount: 0,\n\t},\n\tactions: {\n\t\t// Action that modifies state - should trigger onStateChange\n\t\tsetValue: (c, newValue: number) => {\n\t\t\tc.state.value = newValue;\n\t\t\treturn c.state.value;\n\t\t},\n\t\t// Action that modifies state multiple times - should trigger onStateChange for each change\n\t\tincrementMultiple: (c, times: number) => {\n\t\t\tfor (let i = 0; i < times; i++) {\n\t\t\t\tc.state.value++;\n\t\t\t}\n\t\t\treturn c.state.value;\n\t\t},\n\t\t// Action that doesn't modify state - should NOT trigger onStateChange\n\t\tgetValue: (c) => {\n\t\t\treturn c.state.value;\n\t\t},\n\t\t// Action that reads and returns without modifying - should NOT trigger onStateChange\n\t\tgetDoubled: (c) => {\n\t\t\tconst doubled = c.state.value * 2;\n\t\t\treturn doubled;\n\t\t},\n\t\t// Get the count of how many times onStateChange was called\n\t\tgetChangeCount: (c) => {\n\t\t\treturn c.vars.changeCount;\n\t\t},\n\t\t// Reset change counter for testing\n\t\tresetChangeCount: (c) => {\n\t\t\tc.vars.changeCount = 0;\n\t\t},\n\t},\n\t// Track onStateChange calls\n\tonStateChange: (c) => {\n\t\tc.vars.changeCount++;\n\t},\n});\n","import { actor } from \"rivetkit\";\n\n// Note: For testing only - metadata API will need to be mocked\n// in tests since this is implementation-specific\nexport const metadataActor = actor({\n\tstate: {\n\t\tlastMetadata: null as any,\n\t\tactorName: \"\",\n\t\t// Store tags and region in state for testing since they may not be\n\t\t// available in the context in all environments\n\t\tstoredTags: {} as Record,\n\t\tstoredRegion: null as string | null,\n\t},\n\tonWake: (c) => {\n\t\t// Store the actor name during initialization\n\t\tc.state.actorName = c.name;\n\t},\n\tactions: {\n\t\t// Set up test tags - this will be called by tests to simulate tags\n\t\tsetupTestTags: (c, tags: Record) => {\n\t\t\tc.state.storedTags = tags;\n\t\t\treturn tags;\n\t\t},\n\n\t\t// Set up test region - this will be called by tests to simulate region\n\t\tsetupTestRegion: (c, region: string) => {\n\t\t\tc.state.storedRegion = region;\n\t\t\treturn region;\n\t\t},\n\n\t\t// Get all available metadata\n\t\tgetMetadata: (c) => {\n\t\t\t// Create metadata object from stored values\n\t\t\tconst metadata = {\n\t\t\t\tname: c.name,\n\t\t\t\ttags: c.state.storedTags,\n\t\t\t\tregion: c.state.storedRegion,\n\t\t\t};\n\n\t\t\t// Store for later inspection\n\t\t\tc.state.lastMetadata = metadata;\n\t\t\treturn metadata;\n\t\t},\n\n\t\t// Get the actor name\n\t\tgetActorName: (c) => {\n\t\t\treturn c.name;\n\t\t},\n\n\t\t// Get a specific tag by key\n\t\tgetTag: (c, key: string) => {\n\t\t\treturn c.state.storedTags[key] || null;\n\t\t},\n\n\t\t// Get all tags\n\t\tgetTags: (c) => {\n\t\t\treturn c.state.storedTags;\n\t\t},\n\n\t\t// Get the region\n\t\tgetRegion: (c) => {\n\t\t\treturn c.state.storedRegion;\n\t\t},\n\n\t\t// Get the stored actor name (from onWake)\n\t\tgetStoredActorName: (c) => {\n\t\t\treturn c.state.actorName;\n\t\t},\n\n\t\t// Get last retrieved metadata\n\t\tgetLastMetadata: (c) => {\n\t\t\treturn c.state.lastMetadata;\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\n\n// Actor with static vars\nexport const staticVarActor = actor({\n\tstate: { value: 0 },\n\tconnState: { hello: \"world\" },\n\tvars: { counter: 42, name: \"test-actor\" },\n\tactions: {\n\t\tgetVars: (c) => {\n\t\t\treturn c.vars;\n\t\t},\n\t\tgetName: (c) => {\n\t\t\treturn c.vars.name;\n\t\t},\n\t},\n});\n\n// Actor with nested vars\nexport const nestedVarActor = actor({\n\tstate: { value: 0 },\n\tconnState: { hello: \"world\" },\n\tvars: {\n\t\tcounter: 42,\n\t\tnested: {\n\t\t\tvalue: \"original\",\n\t\t\tarray: [1, 2, 3],\n\t\t\tobj: { key: \"value\" },\n\t\t},\n\t},\n\tactions: {\n\t\tgetVars: (c) => {\n\t\t\treturn c.vars;\n\t\t},\n\t\tmodifyNested: (c) => {\n\t\t\t// Attempt to modify the nested object\n\t\t\tc.vars.nested.value = \"modified\";\n\t\t\tc.vars.nested.array.push(4);\n\t\t\tc.vars.nested.obj.key = \"new-value\";\n\t\t\treturn c.vars;\n\t\t},\n\t},\n});\n\n// Actor with dynamic vars\nexport const dynamicVarActor = actor({\n\tstate: { value: 0 },\n\tconnState: { hello: \"world\" },\n\tcreateVars: () => {\n\t\treturn {\n\t\t\trandom: Math.random(),\n\t\t\tcomputed: `Actor-${Math.floor(Math.random() * 1000)}`,\n\t\t};\n\t},\n\tactions: {\n\t\tgetVars: (c: any) => {\n\t\t\treturn c.vars;\n\t\t},\n\t},\n});\n\n// Actor with unique vars per instance\nexport const uniqueVarActor = actor({\n\tstate: { value: 0 },\n\tconnState: { hello: \"world\" },\n\tcreateVars: () => {\n\t\treturn {\n\t\t\tid: Math.floor(Math.random() * 1000000),\n\t\t};\n\t},\n\tactions: {\n\t\tgetVars: (c: any) => {\n\t\t\treturn c.vars;\n\t\t},\n\t},\n});\n\n// Actor that uses driver context\nexport const driverCtxActor = actor({\n\tstate: { value: 0 },\n\tconnState: { hello: \"world\" },\n\tcreateVars: (c, driverCtx: any) => {\n\t\treturn {\n\t\t\thasDriverCtx: Boolean(driverCtx?.isTest),\n\t\t};\n\t},\n\tactions: {\n\t\tgetVars: (c: any) => {\n\t\t\treturn c.vars;\n\t\t},\n\t},\n});\n","import { actor, type ActorContext } from \"rivetkit\";\n\nexport const kvActor = actor({\n\tactions: {\n\t\tputText: async (\n\t\t\tc: ActorContext,\n\t\t\tkey: string,\n\t\t\tvalue: string,\n\t\t) => {\n\t\t\tawait c.kv.put(key, value);\n\t\t\treturn true;\n\t\t},\n\t\tgetText: async (\n\t\t\tc: ActorContext,\n\t\t\tkey: string,\n\t\t) => {\n\t\t\treturn await c.kv.get(key);\n\t\t},\n\t\tlistText: async (\n\t\t\tc: ActorContext,\n\t\t\tprefix: string,\n\t\t) => {\n\t\t\tconst results = await c.kv.list(prefix, { keyType: \"text\" });\n\t\t\treturn results.map(([key, value]) => ({\n\t\t\t\tkey,\n\t\t\t\tvalue,\n\t\t\t}));\n\t\t},\n\t\troundtripArrayBuffer: async (\n\t\t\tc: ActorContext,\n\t\t\tkey: string,\n\t\t\tvalues: number[],\n\t\t) => {\n\t\t\tconst buffer = new Uint8Array(values).buffer;\n\t\t\tawait c.kv.put(key, buffer, { type: \"arrayBuffer\" });\n\t\t\tconst result = await c.kv.get(key, { type: \"arrayBuffer\" });\n\t\t\tif (!result) {\n\t\t\t\treturn null;\n\t\t\t}\n\t\t\treturn Array.from(new Uint8Array(result));\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\n\n/**\n * Actor for testing large payloads without connections\n */\nexport const largePayloadActor = actor({\n\tstate: {},\n\tactions: {\n\t\t/**\n\t\t * Accepts a large request payload and returns its size\n\t\t */\n\t\tprocessLargeRequest: (c, data: { items: string[] }) => {\n\t\t\treturn {\n\t\t\t\titemCount: data.items.length,\n\t\t\t\tfirstItem: data.items[0],\n\t\t\t\tlastItem: data.items[data.items.length - 1],\n\t\t\t};\n\t\t},\n\n\t\t/**\n\t\t * Returns a large response payload\n\t\t */\n\t\tgetLargeResponse: (c, itemCount: number) => {\n\t\t\tconst items: string[] = [];\n\t\t\tfor (let i = 0; i < itemCount; i++) {\n\t\t\t\titems.push(`Item ${i} with some additional text to increase size`);\n\t\t\t}\n\t\t\treturn { items };\n\t\t},\n\n\t\t/**\n\t\t * Echo back the request data\n\t\t */\n\t\techo: (c, data: unknown) => {\n\t\t\treturn data;\n\t\t},\n\t},\n});\n\n/**\n * Actor for testing large payloads with connections\n */\nexport const largePayloadConnActor = actor({\n\tstate: {},\n\tconnState: {\n\t\tlastRequestSize: 0,\n\t},\n\tactions: {\n\t\t/**\n\t\t * Accepts a large request payload and returns its size\n\t\t */\n\t\tprocessLargeRequest: (c, data: { items: string[] }) => {\n\t\t\tc.conn.state.lastRequestSize = data.items.length;\n\t\t\treturn {\n\t\t\t\titemCount: data.items.length,\n\t\t\t\tfirstItem: data.items[0],\n\t\t\t\tlastItem: data.items[data.items.length - 1],\n\t\t\t};\n\t\t},\n\n\t\t/**\n\t\t * Returns a large response payload\n\t\t */\n\t\tgetLargeResponse: (c, itemCount: number) => {\n\t\t\tconst items: string[] = [];\n\t\t\tfor (let i = 0; i < itemCount; i++) {\n\t\t\t\titems.push(`Item ${i} with some additional text to increase size`);\n\t\t\t}\n\t\t\treturn { items };\n\t\t},\n\n\t\t/**\n\t\t * Echo back the request data\n\t\t */\n\t\techo: (c, data: unknown) => {\n\t\t\treturn data;\n\t\t},\n\n\t\t/**\n\t\t * Get the last request size\n\t\t */\n\t\tgetLastRequestSize: (c) => {\n\t\t\treturn c.conn.state.lastRequestSize;\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nexport const sqliteRawActor = actor({\n\tdb: db({\n\t\tonMigrate: async (db) => {\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS todos (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\ttitle TEXT NOT NULL,\n\t\t\t\t\tcompleted INTEGER DEFAULT 0,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t},\n\t}),\n\tactions: {\n\t\taddTodo: async (c, title: string) => {\n\t\t\tconst createdAt = Date.now();\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO todos (title, created_at) VALUES (?, ?)\",\n\t\t\t\ttitle,\n\t\t\t\tcreatedAt,\n\t\t\t);\n\t\t\treturn { title, createdAt };\n\t\t},\n\t\tgetTodos: async (c) => {\n\t\t\treturn await c.db.execute(\"SELECT * FROM todos ORDER BY created_at DESC\");\n\t\t},\n\t\ttoggleTodo: async (c, id: number) => {\n\t\t\tawait c.db.execute(\n\t\t\t\t\"UPDATE todos SET completed = NOT completed WHERE id = ?\",\n\t\t\t\tid,\n\t\t\t);\n\t\t\tconst rows = await c.db.execute(\"SELECT * FROM todos WHERE id = ?\", id);\n\t\t\treturn rows[0];\n\t\t},\n\t\tdeleteTodo: async (c, id: number) => {\n\t\t\tawait c.db.execute(\"DELETE FROM todos WHERE id = ?\", id);\n\t\t\treturn { deleted: id };\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport { db } from \"rivetkit/db/drizzle\";\nimport { eq } from \"drizzle-orm\";\nimport * as schema from \"./schema.ts\";\nimport migrations from \"./drizzle/migrations.js\";\n\nconst { todos } = schema;\n\nexport const sqliteDrizzleActor = actor({\n\tdb: db({ schema, migrations }),\n\tactions: {\n\t\taddTodo: async (c, title: string) => {\n\t\t\tconst result = await c.db.insert(todos).values({\n\t\t\t\ttitle,\n\t\t\t\tcreatedAt: Date.now(),\n\t\t\t}).returning();\n\t\t\treturn result[0];\n\t\t},\n\t\tgetTodos: async (c) => {\n\t\t\treturn await c.db.select().from(todos).orderBy(todos.createdAt);\n\t\t},\n\t\ttoggleTodo: async (c, id: number) => {\n\t\t\tconst existing = await c.db.select().from(todos).where(eq(todos.id, id));\n\t\t\tif (!existing[0]) return null;\n\t\t\tconst newCompleted = existing[0].completed ? 0 : 1;\n\t\t\tconst result = await c.db.update(todos)\n\t\t\t\t.set({ completed: newCompleted })\n\t\t\t\t.where(eq(todos.id, id))\n\t\t\t\t.returning();\n\t\t\treturn result[0];\n\t\t},\n\t\tdeleteTodo: async (c, id: number) => {\n\t\t\tawait c.db.delete(todos).where(eq(todos.id, id));\n\t\t\treturn { deleted: id };\n\t\t},\n\t},\n});\n","import { sqliteTable, text, integer } from \"rivetkit/db/drizzle\";\n\nexport const todos = sqliteTable(\"todos\", {\n\tid: integer(\"id\").primaryKey({ autoIncrement: true }),\n\ttitle: text(\"title\").notNull(),\n\tcompleted: integer(\"completed\").default(0),\n\tcreatedAt: integer(\"created_at\").notNull(),\n});\n","{\n \"version\": \"7\",\n \"dialect\": \"sqlite\",\n \"entries\": [\n {\n \"idx\": 0,\n \"version\": \"6\",\n \"when\": 1770921282251,\n \"tag\": \"0000_left_wrecking_crew\",\n \"breakpoints\": true\n }\n ]\n}","CREATE TABLE `todos` (\n\t`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,\n\t`title` text NOT NULL,\n\t`completed` integer DEFAULT 0,\n\t`created_at` integer NOT NULL\n);\n","import journal from './meta/_journal.json' with { type: 'json' };\nimport m0000 from './0000_left_wrecking_crew.sql';\n\n export default {\n journal,\n migrations: {\n m0000\n }\n }\n \n","import { actor, event } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nexport const parallelismTest = actor({\n\tstate: {\n\t\tstateCount: 0,\n\t},\n\tdb: db({\n\t\tonMigrate: async (db) => {\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS counter (\n\t\t\t\t\tid INTEGER PRIMARY KEY CHECK (id = 1),\n\t\t\t\t\tcount INTEGER NOT NULL DEFAULT 0\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait db.execute(`\n\t\t\t\tINSERT OR IGNORE INTO counter (id, count) VALUES (1, 0)\n\t\t\t`);\n\t\t},\n\t}),\n\tevents: {\n\t\tstateCountChanged: event<{ count: number }>(),\n\t\tsqliteCountChanged: event<{ count: number }>(),\n\t},\n\tactions: {\n\t\tincrementState: (c) => {\n\t\t\tc.state.stateCount += 1;\n\t\t\tc.broadcast(\"stateCountChanged\", { count: c.state.stateCount });\n\t\t\treturn { count: c.state.stateCount };\n\t\t},\n\t\tgetStateCount: (c) => {\n\t\t\treturn { count: c.state.stateCount };\n\t\t},\n\t\tincrementSqlite: async (c) => {\n\t\t\tawait c.db.execute(`UPDATE counter SET count = count + 1 WHERE id = 1`);\n\t\t\tconst results = await c.db.execute<{ count: number }>(\n\t\t\t\t`SELECT count FROM counter WHERE id = 1`,\n\t\t\t);\n\t\t\tconst count = results[0].count;\n\t\t\tc.broadcast(\"sqliteCountChanged\", { count });\n\t\t\treturn { count };\n\t\t},\n\t\tgetSqliteCount: async (c) => {\n\t\t\tconst results = await c.db.execute<{ count: number }>(\n\t\t\t\t`SELECT count FROM counter WHERE id = 1`,\n\t\t\t);\n\t\t\treturn { count: results[0].count };\n\t\t},\n\t},\n\toptions: {\n\t\tsleepTimeout: 30_000,\n\t},\n});\n","import { actor, event } from \"rivetkit\";\n\nexport type ConnState = {\n\tusername: string;\n\trole: string;\n\tcounter: number;\n\tcreatedAt: number;\n\tnoCount: boolean;\n};\n\nexport const connStateActor = actor({\n\tstate: {\n\t\tsharedCounter: 0,\n\t\tdisconnectionCount: 0,\n\t},\n\tevents: {\n\t\tuserConnected: event<{ id: string; username: string; role: string }>(),\n\t\tuserDisconnected: event<{ id: string }>(),\n\t\tdirectMessage: event<{ from: string; message: string }>(),\n\t},\n\t// Define connection state\n\tcreateConnState: (\n\t\tc,\n\t\tparams: { username?: string; role?: string; noCount?: boolean },\n\t): ConnState => {\n\t\treturn {\n\t\t\tusername: params?.username || \"anonymous\",\n\t\t\trole: params?.role || \"user\",\n\t\t\tcounter: 0,\n\t\t\tcreatedAt: Date.now(),\n\t\t\tnoCount: params?.noCount ?? false,\n\t\t};\n\t},\n\t// Lifecycle hook when a connection is established\n\tonConnect: (c, conn) => {\n\t\t// Broadcast event about the new connection\n\t\tc.broadcast(\"userConnected\", {\n\t\t\tid: conn.id,\n\t\t\tusername: \"anonymous\",\n\t\t\trole: \"user\",\n\t\t});\n\t},\n\t// Lifecycle hook when a connection is closed\n\tonDisconnect: (c, conn) => {\n\t\tif (!conn.state?.noCount) {\n\t\t\tc.state.disconnectionCount += 1;\n\t\t\tc.broadcast(\"userDisconnected\", {\n\t\t\t\tid: conn.id,\n\t\t\t});\n\t\t}\n\t},\n\tactions: {\n\t\t// Action to increment the connection's counter\n\t\tincrementConnCounter: (c, amount = 1) => {\n\t\t\tc.conn.state.counter += amount;\n\t\t},\n\n\t\t// Action to increment the shared counter\n\t\tincrementSharedCounter: (c, amount = 1) => {\n\t\t\tc.state.sharedCounter += amount;\n\t\t\treturn c.state.sharedCounter;\n\t\t},\n\n\t\t// Get the connection state\n\t\tgetConnectionState: (c) => {\n\t\t\treturn { id: c.conn.id, ...c.conn.state };\n\t\t},\n\n\t\t// Check all active connections\n\t\tgetConnectionIds: (c) => {\n\t\t\treturn c.conns\n\t\t\t\t.entries()\n\t\t\t\t.filter((c) => !c[1].state?.noCount)\n\t\t\t\t.map((x) => x[0])\n\t\t\t\t.toArray();\n\t\t},\n\n\t\t// Get disconnection count\n\t\tgetDisconnectionCount: (c) => {\n\t\t\treturn c.state.disconnectionCount;\n\t\t},\n\n\t\t// Get all active connection states\n\t\tgetAllConnectionStates: (c) => {\n\t\t\treturn c.conns\n\t\t\t\t.entries()\n\t\t\t\t.map(([id, conn]) => ({ id, ...conn.state }))\n\t\t\t\t.toArray();\n\t\t},\n\n\t\t// Send message to a specific connection with matching ID\n\t\tsendToConnection: (c, targetId: string, message: string) => {\n\t\t\tif (c.conns.has(targetId)) {\n\t\t\t\tc.conns\n\t\t\t\t\t.get(targetId)!\n\t\t\t\t\t.send(\"directMessage\", { from: c.conn.id, message });\n\t\t\t\treturn true;\n\t\t\t} else {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t},\n\n\t\t// Update connection state (simulated for tests)\n\t\tupdateConnection: (\n\t\t\tc,\n\t\t\tupdates: Partial<{ username: string; role: string }>,\n\t\t) => {\n\t\t\tif (updates.username) c.conn.state.username = updates.username;\n\t\t\tif (updates.role) c.conn.state.role = updates.role;\n\t\t\treturn c.conn.state;\n\t\t},\n\t\tdisconnectSelf: (c, reason?: string) => {\n\t\t\tc.conn.disconnect(reason ?? \"test.disconnect\");\n\t\t\treturn true;\n\t\t},\n\t},\n});\n","import { actor, UserError } from \"rivetkit\";\n\nexport const rejectConnectionActor = actor({\n\tonBeforeConnect: async (_c, params: { reject?: boolean }) => {\n\t\tif (params?.reject) {\n\t\t\tawait new Promise((resolve) => setTimeout(resolve, 500));\n\t\t\tthrow new UserError(\"Rejected connection\", {\n\t\t\t\tcode: \"rejected\",\n\t\t\t});\n\t\t}\n\t},\n\tactions: {\n\t\tping: () => \"pong\",\n\t},\n});\n","import { actor, type RivetMessageEvent } from \"rivetkit\";\n\n/**\n * Test fixture to verify request object access in all lifecycle hooks\n */\nexport const requestAccessActor = actor({\n\tstate: {\n\t\t// Track request info from different hooks\n\t\tonBeforeConnectRequest: {\n\t\t\thasRequest: false,\n\t\t\trequestUrl: null as string | null,\n\t\t\trequestMethod: null as string | null,\n\t\t\trequestHeaders: {} as Record,\n\t\t},\n\t\tcreateConnStateRequest: {\n\t\t\thasRequest: false,\n\t\t\trequestUrl: null as string | null,\n\t\t\trequestMethod: null as string | null,\n\t\t\trequestHeaders: {} as Record,\n\t\t},\n\t\tonRequestRequest: {\n\t\t\thasRequest: false,\n\t\t\trequestUrl: null as string | null,\n\t\t\trequestMethod: null as string | null,\n\t\t\trequestHeaders: {} as Record,\n\t\t},\n\t\tonWebSocketRequest: {\n\t\t\thasRequest: false,\n\t\t\trequestUrl: null as string | null,\n\t\t\trequestMethod: null as string | null,\n\t\t\trequestHeaders: {} as Record,\n\t\t},\n\t},\n\tcreateConnState: (c, params: { trackRequest?: boolean }) => {\n\t\t// In createConnState, the state isn't available yet.\n\n\t\tlet requestInfo: {\n\t\t\thasRequest: boolean;\n\t\t\trequestUrl: string;\n\t\t\trequestMethod: string;\n\t\t\trequestHeaders: Record;\n\t\t} | null = null;\n\n\t\tif (params?.trackRequest && c.request) {\n\t\t\tconst headers: Record = {};\n\t\t\tc.request.headers.forEach((value, key) => {\n\t\t\t\theaders[key] = value;\n\t\t\t});\n\t\t\trequestInfo = {\n\t\t\t\thasRequest: true,\n\t\t\t\trequestUrl: c.request.url,\n\t\t\t\trequestMethod: c.request.method,\n\t\t\t\trequestHeaders: headers,\n\t\t\t};\n\t\t}\n\n\t\treturn {\n\t\t\ttrackRequest: params?.trackRequest || false,\n\t\t\trequestInfo,\n\t\t};\n\t},\n\tonConnect: (c, conn) => {\n\t\t// Copy request info from connection state if it was tracked\n\t\tif (conn.state.requestInfo) {\n\t\t\tc.state.createConnStateRequest = conn.state.requestInfo;\n\t\t}\n\t},\n\tonBeforeConnect: (c, params) => {\n\t\tif (params?.trackRequest) {\n\t\t\tif (c.request) {\n\t\t\t\tc.state.onBeforeConnectRequest.hasRequest = true;\n\t\t\t\tc.state.onBeforeConnectRequest.requestUrl = c.request.url;\n\t\t\t\tc.state.onBeforeConnectRequest.requestMethod = c.request.method;\n\n\t\t\t\t// Store select headers\n\t\t\t\tconst headers: Record = {};\n\t\t\t\tc.request.headers.forEach((value, key) => {\n\t\t\t\t\theaders[key] = value;\n\t\t\t\t});\n\t\t\t\tc.state.onBeforeConnectRequest.requestHeaders = headers;\n\t\t\t} else {\n\t\t\t\t// Track that we tried but request was not available\n\t\t\t\tc.state.onBeforeConnectRequest.hasRequest = false;\n\t\t\t}\n\t\t}\n\t},\n\tonRequest: (c, request) => {\n\t\t// Store request info\n\t\tc.state.onRequestRequest.hasRequest = true;\n\t\tc.state.onRequestRequest.requestUrl = request.url;\n\t\tc.state.onRequestRequest.requestMethod = request.method;\n\n\t\t// Store select headers\n\t\tconst headers: Record = {};\n\t\trequest.headers.forEach((value, key) => {\n\t\t\theaders[key] = value;\n\t\t});\n\t\tc.state.onRequestRequest.requestHeaders = headers;\n\n\t\t// Return response with request info\n\t\treturn new Response(\n\t\t\tJSON.stringify({\n\t\t\t\thasRequest: true,\n\t\t\t\trequestUrl: request.url,\n\t\t\t\trequestMethod: request.method,\n\t\t\t\trequestHeaders: headers,\n\t\t\t}),\n\t\t\t{\n\t\t\t\tstatus: 200,\n\t\t\t\theaders: { \"Content-Type\": \"application/json\" },\n\t\t\t},\n\t\t);\n\t},\n\tonWebSocket: (c, websocket) => {\n\t\tif (!c.request) throw \"Missing request\";\n\t\t// Store request info\n\t\tc.state.onWebSocketRequest.hasRequest = true;\n\t\tc.state.onWebSocketRequest.requestUrl = c.request.url;\n\t\tc.state.onWebSocketRequest.requestMethod = c.request.method;\n\n\t\t// Store select headers\n\t\tconst headers: Record = {};\n\t\tc.request.headers.forEach((value, key) => {\n\t\t\theaders[key] = value;\n\t\t});\n\t\tc.state.onWebSocketRequest.requestHeaders = headers;\n\n\t\t// Send request info on connection\n\t\twebsocket.send(\n\t\t\tJSON.stringify({\n\t\t\t\thasRequest: true,\n\t\t\t\trequestUrl: c.request.url,\n\t\t\t\trequestMethod: c.request.method,\n\t\t\t\trequestHeaders: headers,\n\t\t\t}),\n\t\t);\n\n\t\t// Echo messages back\n\t\twebsocket.addEventListener(\"message\", (event: RivetMessageEvent) => {\n\t\t\twebsocket.send(event.data);\n\t\t});\n\t},\n\tactions: {\n\t\tgetRequestInfo: (c) => {\n\t\t\treturn {\n\t\t\t\tonBeforeConnect: c.state.onBeforeConnectRequest,\n\t\t\t\tcreateConnState: c.state.createConnStateRequest,\n\t\t\t\tonRequest: c.state.onRequestRequest,\n\t\t\t\tonWebSocket: c.state.onWebSocketRequest,\n\t\t\t};\n\t\t},\n\t},\n});\n","import { Hono } from \"hono\";\nimport { actor, type RequestContext } from \"rivetkit\";\n\nexport const rawHttpActor = actor({\n\tstate: {\n\t\trequestCount: 0,\n\t},\n\tonRequest(\n\t\tctx: RequestContext,\n\t\trequest: Request,\n\t) {\n\t\tconst url = new URL(request.url);\n\t\tconst method = request.method;\n\n\t\t// Track request count\n\t\tctx.state.requestCount++;\n\n\t\t// Handle different endpoints\n\t\tif (url.pathname === \"/api/hello\") {\n\t\t\treturn new Response(\n\t\t\t\tJSON.stringify({ message: \"Hello from actor!\" }),\n\t\t\t\t{\n\t\t\t\t\theaders: { \"Content-Type\": \"application/json\" },\n\t\t\t\t},\n\t\t\t);\n\t\t}\n\n\t\tif (url.pathname === \"/api/echo\" && method === \"POST\") {\n\t\t\treturn new Response(request.body, {\n\t\t\t\theaders: request.headers,\n\t\t\t});\n\t\t}\n\n\t\tif (url.pathname === \"/api/state\") {\n\t\t\treturn new Response(\n\t\t\t\tJSON.stringify({\n\t\t\t\t\trequestCount: ctx.state.requestCount,\n\t\t\t\t}),\n\t\t\t\t{\n\t\t\t\t\theaders: { \"Content-Type\": \"application/json\" },\n\t\t\t\t},\n\t\t\t);\n\t\t}\n\n\t\tif (url.pathname === \"/api/headers\") {\n\t\t\tconst headers: Record = {};\n\t\t\trequest.headers.forEach((value, key) => {\n\t\t\t\theaders[key] = value;\n\t\t\t});\n\t\t\treturn new Response(JSON.stringify(headers), {\n\t\t\t\theaders: { \"Content-Type\": \"application/json\" },\n\t\t\t});\n\t\t}\n\n\t\t// Return 404 for unhandled paths\n\t\treturn new Response(\"Not Found\", { status: 404 });\n\t},\n\tactions: {},\n});\n\nexport const rawHttpNoHandlerActor = actor({\n\tactions: {},\n});\n\nexport const rawHttpVoidReturnActor = actor({\n\tonRequest(ctx, request) {\n\t\t// Intentionally return void to test error handling\n\t\treturn undefined as any;\n\t},\n\tactions: {},\n});\n\nexport const rawHttpHonoActor = actor({\n\tcreateVars() {\n\t\tconst router = new Hono();\n\n\t\t// Set up routes\n\t\trouter.get(\"/\", (c: any) =>\n\t\t\tc.json({ message: \"Welcome to Hono actor!\" }),\n\t\t);\n\n\t\trouter.get(\"/users\", (c: any) =>\n\t\t\tc.json([\n\t\t\t\t{ id: 1, name: \"Alice\" },\n\t\t\t\t{ id: 2, name: \"Bob\" },\n\t\t\t]),\n\t\t);\n\n\t\trouter.get(\"/users/:id\", (c: any) => {\n\t\t\tconst id = c.req.param(\"id\");\n\t\t\treturn c.json({\n\t\t\t\tid: parseInt(id),\n\t\t\t\tname: id === \"1\" ? \"Alice\" : \"Bob\",\n\t\t\t});\n\t\t});\n\n\t\trouter.post(\"/users\", async (c: any) => {\n\t\t\tconst body = await c.req.json();\n\t\t\treturn c.json({ id: 3, ...body }, 201);\n\t\t});\n\n\t\trouter.put(\"/users/:id\", async (c: any) => {\n\t\t\tconst id = c.req.param(\"id\");\n\t\t\tconst body = await c.req.json();\n\t\t\treturn c.json({ id: parseInt(id), ...body });\n\t\t});\n\n\t\trouter.delete(\"/users/:id\", (c: any) => {\n\t\t\tconst id = c.req.param(\"id\");\n\t\t\treturn c.json({ message: `User ${id} deleted` });\n\t\t});\n\n\t\t// Return the router as a var\n\t\treturn { router };\n\t},\n\tonRequest(\n\t\tctx: RequestContext,\n\t\trequest: Request,\n\t) {\n\t\t// Use the Hono router from vars\n\t\treturn ctx.vars.router.fetch(request);\n\t},\n\tactions: {},\n});\n","import { actor, type RequestContext } from \"rivetkit\";\n\nexport const rawHttpRequestPropertiesActor = actor({\n\tactions: {},\n\tonRequest(\n\t\tctx: RequestContext,\n\t\trequest: Request,\n\t) {\n\t\t// Extract all relevant Request properties\n\t\tconst url = new URL(request.url);\n\t\tconst method = request.method;\n\n\t\t// Get all headers\n\t\tconst headers: Record = {};\n\t\trequest.headers.forEach((value, key) => {\n\t\t\theaders[key] = value;\n\t\t});\n\n\t\t// Handle body based on content type\n\t\tconst handleBody = async () => {\n\t\t\tif (!request.body) {\n\t\t\t\treturn null;\n\t\t\t}\n\n\t\t\tconst contentType = request.headers.get(\"content-type\") || \"\";\n\n\t\t\ttry {\n\t\t\t\tif (contentType.includes(\"application/json\")) {\n\t\t\t\t\tconst text = await request.text();\n\t\t\t\t\treturn text ? JSON.parse(text) : null;\n\t\t\t\t} else {\n\t\t\t\t\t// For non-JSON, return as text\n\t\t\t\t\tconst text = await request.text();\n\t\t\t\t\treturn text || null; // Return null for empty bodies\n\t\t\t\t}\n\t\t\t} catch (error) {\n\t\t\t\t// If body parsing fails, return null\n\t\t\t\treturn null;\n\t\t\t}\n\t\t};\n\n\t\t// Special handling for HEAD and OPTIONS requests\n\t\tif (method === \"HEAD\") {\n\t\t\treturn new Response(null, {\n\t\t\t\tstatus: 200,\n\t\t\t});\n\t\t}\n\n\t\tif (method === \"OPTIONS\") {\n\t\t\treturn new Response(null, {\n\t\t\t\tstatus: 204,\n\t\t\t});\n\t\t}\n\n\t\t// Return all request properties as JSON\n\t\treturn handleBody().then((body) => {\n\t\t\tconst responseData = {\n\t\t\t\t// URL properties\n\t\t\t\turl: request.url,\n\t\t\t\tpathname: url.pathname,\n\t\t\t\tsearch: url.search,\n\t\t\t\tsearchParams: Object.fromEntries(url.searchParams.entries()),\n\t\t\t\thash: url.hash,\n\n\t\t\t\t// Method\n\t\t\t\tmethod: request.method,\n\n\t\t\t\t// Headers\n\t\t\t\theaders: headers,\n\n\t\t\t\t// Body\n\t\t\t\tbody,\n\t\t\t\tbodyText:\n\t\t\t\t\ttypeof body === \"string\"\n\t\t\t\t\t\t? body\n\t\t\t\t\t\t: body === null && request.body !== null\n\t\t\t\t\t\t\t? \"\"\n\t\t\t\t\t\t\t: null,\n\n\t\t\t\t// Additional properties that might be available\n\t\t\t\t// Note: Some properties like cache, credentials, mode, etc.\n\t\t\t\t// might not be available in all environments\n\t\t\t\tcache: request.cache || null,\n\t\t\t\tcredentials: request.credentials || null,\n\t\t\t\tmode: request.mode || null,\n\t\t\t\tredirect: request.redirect || null,\n\t\t\t\treferrer: request.referrer || null,\n\t\t\t};\n\n\t\t\treturn new Response(JSON.stringify(responseData), {\n\t\t\t\theaders: { \"Content-Type\": \"application/json\" },\n\t\t\t});\n\t\t});\n\t},\n});\n","import { type ActorContext, actor, type UniversalWebSocket } from \"rivetkit\";\n\nexport const rawWebSocketActor = actor({\n\tstate: {\n\t\tconnectionCount: 0,\n\t\tmessageCount: 0,\n\t},\n\tonWebSocket(ctx, websocket) {\n\t\tctx.state.connectionCount = ctx.state.connectionCount + 1;\n\t\tconsole.log(\n\t\t\t`[ACTOR] New connection, count: ${ctx.state.connectionCount}`,\n\t\t);\n\n\t\t// Send welcome message\n\t\twebsocket.send(\n\t\t\tJSON.stringify({\n\t\t\t\ttype: \"welcome\",\n\t\t\t\tconnectionCount: ctx.state.connectionCount,\n\t\t\t}),\n\t\t);\n\t\tconsole.log(\"[ACTOR] Sent welcome message\");\n\n\t\t// Echo messages back\n\t\twebsocket.addEventListener(\"message\", (event: any) => {\n\t\t\tctx.state.messageCount = ctx.state.messageCount + 1;\n\t\t\tconsole.log(\n\t\t\t\t`[ACTOR] Message received, total count: ${ctx.state.messageCount}, data:`,\n\t\t\t\tevent.data,\n\t\t\t);\n\n\t\t\tconst data = event.data;\n\t\t\tif (typeof data === \"string\") {\n\t\t\t\ttry {\n\t\t\t\t\tconst parsed = JSON.parse(data);\n\t\t\t\t\tif (parsed.type === \"ping\") {\n\t\t\t\t\t\twebsocket.send(\n\t\t\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\t\t\ttype: \"pong\",\n\t\t\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t);\n\t\t\t\t\t} else if (parsed.type === \"getStats\") {\n\t\t\t\t\t\tconsole.log(\n\t\t\t\t\t\t\t`[ACTOR] Sending stats - connections: ${ctx.state.connectionCount}, messages: ${ctx.state.messageCount}`,\n\t\t\t\t\t\t);\n\t\t\t\t\t\twebsocket.send(\n\t\t\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\t\t\ttype: \"stats\",\n\t\t\t\t\t\t\t\tconnectionCount: ctx.state.connectionCount,\n\t\t\t\t\t\t\t\tmessageCount: ctx.state.messageCount,\n\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t);\n\t\t\t\t\t} else if (parsed.type === \"getRequestInfo\") {\n\t\t\t\t\t\t// Send back the request URL info if available\n\t\t\t\t\t\tconst url = ctx.request?.url || \"ws://actor/websocket\";\n\t\t\t\t\t\tconst urlObj = new URL(url);\n\t\t\t\t\t\twebsocket.send(\n\t\t\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\t\t\ttype: \"requestInfo\",\n\t\t\t\t\t\t\t\turl: url,\n\t\t\t\t\t\t\t\tpathname: urlObj.pathname,\n\t\t\t\t\t\t\t\tsearch: urlObj.search,\n\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t);\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// Echo back\n\t\t\t\t\t\twebsocket.send(data);\n\t\t\t\t\t}\n\t\t\t\t} catch {\n\t\t\t\t\t// If not JSON, just echo it back\n\t\t\t\t\twebsocket.send(data);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// Echo binary data\n\t\t\t\twebsocket.send(data);\n\t\t\t}\n\t\t});\n\n\t\t// Handle close\n\t\twebsocket.addEventListener(\"close\", () => {\n\t\t\tctx.state.connectionCount = ctx.state.connectionCount - 1;\n\t\t\tconsole.log(\n\t\t\t\t`[ACTOR] Connection closed, count: ${ctx.state.connectionCount}`,\n\t\t\t);\n\t\t});\n\t},\n\tactions: {\n\t\tgetStats(ctx: any) {\n\t\t\treturn {\n\t\t\t\tconnectionCount: ctx.state.connectionCount,\n\t\t\t\tmessageCount: ctx.state.messageCount,\n\t\t\t};\n\t\t},\n\t},\n});\n\nexport const rawWebSocketBinaryActor = actor({\n\tonWebSocket(ctx, websocket) {\n\t\t// Handle binary data\n\t\twebsocket.addEventListener(\"message\", (event: any) => {\n\t\t\tconst data = event.data;\n\t\t\tif (data instanceof ArrayBuffer || data instanceof Uint8Array) {\n\t\t\t\t// Reverse the bytes and send back\n\t\t\t\tconst bytes = new Uint8Array(data);\n\t\t\t\tconst reversed = new Uint8Array(bytes.length);\n\t\t\t\tfor (let i = 0; i < bytes.length; i++) {\n\t\t\t\t\treversed[i] = bytes[bytes.length - 1 - i];\n\t\t\t\t}\n\t\t\t\twebsocket.send(reversed);\n\t\t\t}\n\t\t});\n\t},\n\tactions: {},\n});\n","import { Hono } from \"hono\";\nimport { type ActorContextOf, actor } from \"rivetkit\";\n\nexport const rawFetchCounter = actor({\n\tstate: {\n\t\tcount: 0,\n\t},\n\tcreateVars: () => {\n\t\t// Setup router\n\t\treturn { router: createCounterRouter() };\n\t},\n\tonRequest: (c, request) => {\n\t\treturn c.vars.router.fetch(request, { actor: c });\n\t},\n\tactions: {\n\t\t// ...actions...\n\t},\n});\n\nfunction createCounterRouter(): Hono {\n\tconst app = new Hono<{\n\t\tBindings: { actor: ActorContextOf };\n\t}>();\n\n\tapp.get(\"/count\", (c) => {\n\t\tconst { actor } = c.env;\n\n\t\treturn c.json({\n\t\t\tcount: actor.state.count,\n\t\t});\n\t});\n\n\tapp.post(\"/increment\", (c) => {\n\t\tconst { actor } = c.env;\n\n\t\tactor.state.count++;\n\t\treturn c.json({\n\t\t\tcount: actor.state.count,\n\t\t});\n\t});\n\n\treturn app;\n}\n","import { actor } from \"rivetkit\";\n\nexport const rawWebSocketChatRoom = actor({\n\tstate: {\n\t\tmessages: [] as Array<{\n\t\t\tid: string;\n\t\t\ttext: string;\n\t\t\ttimestamp: number;\n\t\t}>,\n\t},\n\tcreateVars: () => {\n\t\treturn {\n\t\t\tsockets: new Set(),\n\t\t};\n\t},\n\tonWebSocket(ctx, socket) {\n\t\t// Add socket to the set\n\t\tctx.vars.sockets.add(socket);\n\n\t\t// Send recent messages to new connection\n\t\tsocket.send(\n\t\t\tJSON.stringify({\n\t\t\t\ttype: \"init\",\n\t\t\t\tmessages: ctx.state.messages,\n\t\t\t}),\n\t\t);\n\n\t\t// Handle incoming messages\n\t\tsocket.addEventListener(\"message\", (event: any) => {\n\t\t\ttry {\n\t\t\t\tconst data = JSON.parse(event.data);\n\n\t\t\t\tif (data.type === \"message\" && data.text) {\n\t\t\t\t\tconst message = {\n\t\t\t\t\t\tid: crypto.randomUUID(),\n\t\t\t\t\t\ttext: data.text,\n\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t};\n\n\t\t\t\t\t// Add to state\n\t\t\t\t\tctx.state.messages.push(message);\n\t\t\t\t\tctx.saveState({});\n\n\t\t\t\t\t// Keep only last 50 messages\n\t\t\t\t\tif (ctx.state.messages.length > 50) {\n\t\t\t\t\t\tctx.state.messages.shift();\n\t\t\t\t\t}\n\n\t\t\t\t\t// Broadcast to all connected clients\n\t\t\t\t\tconst broadcast = JSON.stringify({\n\t\t\t\t\t\ttype: \"message\",\n\t\t\t\t\t\t...message,\n\t\t\t\t\t});\n\n\t\t\t\t\tfor (const ws of ctx.vars.sockets) {\n\t\t\t\t\t\tif (ws.readyState === 1) {\n\t\t\t\t\t\t\t// OPEN\n\t\t\t\t\t\t\tws.send(broadcast);\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} catch (e) {\n\t\t\t\tconsole.error(\"Failed to process message:\", e);\n\t\t\t}\n\t\t});\n\n\t\t// Remove socket on close\n\t\tsocket.addEventListener(\"close\", () => {\n\t\t\tctx.vars.sockets.delete(socket);\n\t\t});\n\t},\n\tactions: {},\n});\n","import { actor, type RivetMessageEvent, type UniversalWebSocket } from \"rivetkit\";\n\nfunction sleep(ms: number): Promise {\n\treturn new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nexport const rawWebSocketServerlessSmoke = actor({\n\toptions: {\n\t\tcanHibernateWebSocket: false,\n\t\tsleepGracePeriod: 5_000,\n\t},\n\tstate: {\n\t\tconnectionCount: 0,\n\t\tsleepCount: 0,\n\t\ttotalTickCount: 0,\n\t\ttotalMessageCount: 0,\n\t},\n\tasync onSleep(c) {\n\t\tconst delayMs = 10 + Math.floor(Math.random() * 1_991);\n\t\tc.state.sleepCount += 1;\n\t\tc.log.info({\n\t\t\tmsg: \"raw websocket serverless smoke onSleep delay\",\n\t\t\tdelayMs,\n\t\t\tsleepCount: c.state.sleepCount,\n\t\t});\n\t\tawait sleep(delayMs);\n\t},\n\tonWebSocket(c, websocket: UniversalWebSocket) {\n\t\tc.state.connectionCount += 1;\n\t\tconst connectionId = crypto.randomUUID();\n\t\tlet index = 0;\n\n\t\tconst sendTick = () => {\n\t\t\tif (websocket.readyState !== 1) return;\n\n\t\t\tconst timestamp = Date.now();\n\t\t\tconst message = {\n\t\t\t\ttype: \"tick\",\n\t\t\t\tconnectionId,\n\t\t\t\tindex,\n\t\t\t\ttimestamp,\n\t\t\t\tiso: new Date(timestamp).toISOString(),\n\t\t\t\ttotalTickCount: c.state.totalTickCount,\n\t\t\t};\n\n\t\t\tc.state.totalTickCount += 1;\n\t\t\tindex += 1;\n\t\t\twebsocket.send(JSON.stringify(message));\n\t\t};\n\n\t\tc.log.info({\n\t\t\tmsg: \"raw websocket serverless smoke connected\",\n\t\t\tconnectionId,\n\t\t\tconnectionCount: c.state.connectionCount,\n\t\t});\n\n\t\tsendTick();\n\t\tconst interval = setInterval(sendTick, 1_000);\n\n\t\twebsocket.addEventListener(\"message\", (event: RivetMessageEvent) => {\n\t\t\tc.state.totalMessageCount += 1;\n\t\t\tc.log.info({\n\t\t\t\tmsg: \"raw websocket serverless smoke received message\",\n\t\t\t\tconnectionId,\n\t\t\t\ttotalMessageCount: c.state.totalMessageCount,\n\t\t\t});\n\t\t\twebsocket.send(\n\t\t\t\tJSON.stringify({\n\t\t\t\t\ttype: \"ack\",\n\t\t\t\t\tconnectionId,\n\t\t\t\t\tindex,\n\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\treceived: event.data,\n\t\t\t\t}),\n\t\t\t);\n\t\t});\n\n\t\twebsocket.addEventListener(\"close\", () => {\n\t\t\tclearInterval(interval);\n\t\t\tc.state.connectionCount -= 1;\n\t\t\tc.log.info({\n\t\t\t\tmsg: \"raw websocket serverless smoke disconnected\",\n\t\t\t\tconnectionId,\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t});\n\t\t});\n\t},\n\tactions: {\n\t\tgetStats(c) {\n\t\t\treturn {\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t\ttotalTickCount: c.state.totalTickCount,\n\t\t\t\ttotalMessageCount: c.state.totalMessageCount,\n\t\t\t};\n\t\t},\n\t},\n});\n","import { actor, type RivetMessageEvent, type UniversalWebSocket } from \"rivetkit\";\n\nexport const tunnelStress = actor({\n\toptions: {\n\t\tcanHibernateWebSocket: false,\n\t\tsleepGracePeriod: 5_000,\n\t},\n\tstate: {\n\t\tconnectionCount: 0,\n\t\tmessageCount: 0,\n\t\theartbeatCount: 0,\n\t},\n\tonWebSocket(c, websocket: UniversalWebSocket) {\n\t\tc.state.connectionCount += 1;\n\t\tconst connectionId = crypto.randomUUID();\n\n\t\tconst sendHeartbeat = () => {\n\t\t\tif (websocket.readyState !== 1) return;\n\n\t\t\tc.state.heartbeatCount += 1;\n\t\t\twebsocket.send(\n\t\t\t\tJSON.stringify({\n\t\t\t\t\ttype: \"heartbeat\",\n\t\t\t\t\tconnectionId,\n\t\t\t\t\theartbeatCount: c.state.heartbeatCount,\n\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t}),\n\t\t\t);\n\t\t};\n\n\t\tconst heartbeat = setInterval(sendHeartbeat, 1_000);\n\t\tsendHeartbeat();\n\n\t\twebsocket.addEventListener(\"message\", async (event: RivetMessageEvent) => {\n\t\t\t// Fast-path ping: echo back without touching KV so the client can measure raw RTT\n\t\t\t// without the per-message storage write. Used by the counter-latency client's first\n\t\t\t// two probes after WS open.\n\t\t\tif (typeof event.data === \"string\") {\n\t\t\t\tlet parsed: unknown;\n\t\t\t\ttry {\n\t\t\t\t\tparsed = JSON.parse(event.data);\n\t\t\t\t} catch {\n\t\t\t\t\tparsed = undefined;\n\t\t\t\t}\n\t\t\t\tif (\n\t\t\t\t\tparsed &&\n\t\t\t\t\ttypeof parsed === \"object\" &&\n\t\t\t\t\t(parsed as { type?: unknown }).type === \"ping\"\n\t\t\t\t) {\n\t\t\t\t\tconst id = (parsed as { id?: unknown }).id;\n\t\t\t\t\tif (websocket.readyState === 1) {\n\t\t\t\t\t\twebsocket.send(\n\t\t\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\t\t\ttype: \"pong\",\n\t\t\t\t\t\t\t\tconnectionId,\n\t\t\t\t\t\t\t\tid,\n\t\t\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tc.state.messageCount += 1;\n\t\t\tawait c.kv.put(\"counter\", String(c.state.messageCount));\n\t\t\twebsocket.send(\n\t\t\t\tJSON.stringify({\n\t\t\t\t\ttype: \"reply\",\n\t\t\t\t\tconnectionId,\n\t\t\t\t\tmessageCount: c.state.messageCount,\n\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\treceived: event.data,\n\t\t\t\t}),\n\t\t\t);\n\t\t});\n\n\t\twebsocket.addEventListener(\"close\", () => {\n\t\t\tclearInterval(heartbeat);\n\t\t\tc.state.connectionCount -= 1;\n\t\t});\n\t},\n\tactions: {\n\t\tgetStats(c) {\n\t\t\treturn {\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t\tmessageCount: c.state.messageCount,\n\t\t\t\theartbeatCount: c.state.heartbeatCount,\n\t\t\t};\n\t\t},\n\t},\n});\n","import { actor, event, queue } from \"rivetkit\";\nimport type { registry } from \"../../index.ts\";\n\nexport const RUN_SLEEP_TIMEOUT = 500;\n\n// Actor that tracks tick counts and respects abort signal\nexport const runWithTicks = actor({\n\tstate: {\n\t\ttickCount: 0,\n\t\tlastTickAt: 0,\n\t\trunStarted: false,\n\t\trunExited: false,\n\t},\n\trun: async (c) => {\n\t\tc.state.runStarted = true;\n\t\tc.log.info(\"run handler started\");\n\n\t\twhile (!c.aborted) {\n\t\t\tc.state.tickCount += 1;\n\t\t\tc.state.lastTickAt = Date.now();\n\t\t\tc.log.info({ msg: \"tick\", tickCount: c.state.tickCount });\n\n\t\t\t// Wait 50ms between ticks, or exit early if aborted\n\t\t\tawait new Promise((resolve) => {\n\t\t\t\tconst timeout = setTimeout(resolve, 50);\n\t\t\t\tc.abortSignal.addEventListener(\n\t\t\t\t\t\"abort\",\n\t\t\t\t\t() => {\n\t\t\t\t\t\tclearTimeout(timeout);\n\t\t\t\t\t\tresolve();\n\t\t\t\t\t},\n\t\t\t\t\t{ once: true },\n\t\t\t\t);\n\t\t\t});\n\t\t}\n\n\t\tc.state.runExited = true;\n\t\tc.log.info(\"run handler exiting gracefully\");\n\t},\n\tactions: {\n\t\tgetState: (c) => ({\n\t\t\ttickCount: c.state.tickCount,\n\t\t\tlastTickAt: c.state.lastTickAt,\n\t\t\trunStarted: c.state.runStarted,\n\t\t\trunExited: c.state.runExited,\n\t\t}),\n\t},\n\toptions: {\n\t\tsleepTimeout: RUN_SLEEP_TIMEOUT,\n\t},\n});\n\n// Actor that consumes from a queue in the run handler\nexport const runWithQueueConsumer = actor({\n\tstate: {\n\t\tmessagesReceived: [] as Array<{ name: string; body: unknown }>,\n\t\trunStarted: false,\n\t},\n\tqueues: {\n\t\tmessages: queue(),\n\t},\n\trun: async (c) => {\n\t\tc.state.runStarted = true;\n\t\tc.log.info(\"run handler started, waiting for messages\");\n\n\t\tfor await (const message of c.queue.iter()) {\n\t\t\tc.log.info({ msg: \"received message\", body: message.body });\n\t\t\tc.state.messagesReceived.push({\n\t\t\t\tname: message.name,\n\t\t\t\tbody: message.body,\n\t\t\t});\n\t\t}\n\n\t\tc.log.info(\"run handler exiting gracefully\");\n\t},\n\tactions: {\n\t\tgetState: (c) => ({\n\t\t\tmessagesReceived: c.state.messagesReceived,\n\t\t\trunStarted: c.state.runStarted,\n\t\t}),\n\t\tsendMessage: async (c, body: unknown) => {\n\t\t\tconst client = c.client();\n\t\t\tconst handle = client.runWithQueueConsumer.getForId(c.actorId);\n\t\t\tawait handle.send(\"messages\", body);\n\t\t\treturn true;\n\t\t},\n\t},\n\toptions: {\n\t\tsleepTimeout: RUN_SLEEP_TIMEOUT,\n\t},\n});\n\n// Actor that exits the run handler after a short delay to test crash behavior\nexport const runWithEarlyExit = actor({\n\tstate: {\n\t\trunStarted: false,\n\t\tdestroyCalled: false,\n\t},\n\trun: async (c) => {\n\t\tc.state.runStarted = true;\n\t\tc.log.info(\"run handler started, will exit after delay\");\n\t\t// Wait a bit so we can observe the runStarted state before exit\n\t\tawait new Promise((resolve) => setTimeout(resolve, 200));\n\t\tc.log.info(\"run handler exiting early\");\n\t\t// Exit without respecting abort signal\n\t},\n\tonDestroy: (c) => {\n\t\tc.state.destroyCalled = true;\n\t},\n\tactions: {\n\t\tgetState: (c) => ({\n\t\t\trunStarted: c.state.runStarted,\n\t\t\tdestroyCalled: c.state.destroyCalled,\n\t\t}),\n\t},\n\toptions: {\n\t\tsleepTimeout: RUN_SLEEP_TIMEOUT,\n\t},\n});\n\n// Actor that throws an error in the run handler to test crash behavior\nexport const runWithError = actor({\n\tstate: {\n\t\trunStarted: false,\n\t\tdestroyCalled: false,\n\t},\n\trun: async (c) => {\n\t\tc.state.runStarted = true;\n\t\tc.log.info(\"run handler started, will throw error\");\n\t\tawait new Promise((resolve) => setTimeout(resolve, 50));\n\t\tthrow new Error(\"intentional error in run handler\");\n\t},\n\tonDestroy: (c) => {\n\t\tc.state.destroyCalled = true;\n\t},\n\tactions: {\n\t\tgetState: (c) => ({\n\t\t\trunStarted: c.state.runStarted,\n\t\t\tdestroyCalled: c.state.destroyCalled,\n\t\t}),\n\t},\n\toptions: {\n\t\tsleepTimeout: RUN_SLEEP_TIMEOUT,\n\t},\n});\n\n// Actor without a run handler for comparison\nexport const runWithoutHandler = actor({\n\tstate: {\n\t\twakeCount: 0,\n\t},\n\tonWake: (c) => {\n\t\tc.state.wakeCount += 1;\n\t},\n\tactions: {\n\t\tgetState: (c) => ({\n\t\t\twakeCount: c.state.wakeCount,\n\t\t}),\n\t},\n\toptions: {\n\t\tsleepTimeout: RUN_SLEEP_TIMEOUT,\n\t},\n});\n","import { actor, event, type UniversalWebSocket } from \"rivetkit\";\nimport { promiseWithResolvers } from \"rivetkit/utils\";\n\nexport const SLEEP_TIMEOUT = 1000;\n\nexport const sleep = actor({\n\tstate: { startCount: 0, sleepCount: 0 },\n\tonWake: (c) => {\n\t\tc.state.startCount += 1;\n\t},\n\tonSleep: (c) => {\n\t\tc.state.sleepCount += 1;\n\t},\n\tactions: {\n\t\ttriggerSleep: (c) => {\n\t\t\tc.sleep();\n\t\t},\n\t\tgetCounts: (c) => {\n\t\t\treturn {\n\t\t\t\tstartCount: c.state.startCount,\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t};\n\t\t},\n\t\tsetAlarm: async (c, duration: number) => {\n\t\t\tawait c.schedule.after(duration, \"onAlarm\");\n\t\t},\n\t\tonAlarm: (c) => {\n\t\t\tc.log.info(\"alarm called\");\n\t\t},\n\t},\n\toptions: {\n\t\tsleepTimeout: SLEEP_TIMEOUT,\n\t},\n});\n\nexport const sleepWithLongRpc = actor({\n\tstate: { startCount: 0, sleepCount: 0 },\n\tcreateVars: () =>\n\t\t({}) as { longRunningResolve: PromiseWithResolvers },\n\tevents: {\n\t\twaiting: event<[]>(),\n\t},\n\tonWake: (c) => {\n\t\tc.state.startCount += 1;\n\t},\n\tonSleep: (c) => {\n\t\tc.state.sleepCount += 1;\n\t},\n\tactions: {\n\t\tgetCounts: (c) => {\n\t\t\treturn {\n\t\t\t\tstartCount: c.state.startCount,\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t};\n\t\t},\n\t\tlongRunningRpc: async (c) => {\n\t\t\tc.log.info(\"starting long running rpc\");\n\t\t\tc.vars.longRunningResolve = promiseWithResolvers(() => {});\n\t\t\tc.broadcast(\"waiting\");\n\t\t\tawait c.vars.longRunningResolve.promise;\n\t\t\tc.log.info(\"finished long running rpc\");\n\t\t},\n\t\tfinishLongRunningRpc: (c) => c.vars.longRunningResolve?.resolve(),\n\t},\n\toptions: {\n\t\tsleepTimeout: SLEEP_TIMEOUT,\n\t},\n});\n\nexport const sleepWithRawHttp = actor({\n\tstate: { startCount: 0, sleepCount: 0, requestCount: 0 },\n\tonWake: (c) => {\n\t\tc.state.startCount += 1;\n\t},\n\tonSleep: (c) => {\n\t\tc.state.sleepCount += 1;\n\t},\n\tonRequest: async (c, request) => {\n\t\tc.state.requestCount += 1;\n\t\tconst url = new URL(request.url);\n\n\t\tif (url.pathname === \"/long-request\") {\n\t\t\tconst duration = parseInt(\n\t\t\t\turl.searchParams.get(\"duration\") || \"1000\",\n\t\t\t);\n\t\t\tc.log.info({ msg: \"starting long fetch request\", duration });\n\t\t\tawait new Promise((resolve) => setTimeout(resolve, duration));\n\t\t\tc.log.info(\"finished long fetch request\");\n\t\t\treturn new Response(JSON.stringify({ completed: true }), {\n\t\t\t\theaders: { \"Content-Type\": \"application/json\" },\n\t\t\t});\n\t\t}\n\n\t\treturn new Response(\"Not Found\", { status: 404 });\n\t},\n\tactions: {\n\t\tgetCounts: (c) => {\n\t\t\treturn {\n\t\t\t\tstartCount: c.state.startCount,\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t\trequestCount: c.state.requestCount,\n\t\t\t};\n\t\t},\n\t},\n\toptions: {\n\t\tsleepTimeout: SLEEP_TIMEOUT,\n\t},\n});\n\nexport const sleepWithRawWebSocket = actor({\n\tstate: { startCount: 0, sleepCount: 0, connectionCount: 0 },\n\tonWake: (c) => {\n\t\tc.state.startCount += 1;\n\t},\n\tonSleep: (c) => {\n\t\tc.state.sleepCount += 1;\n\t},\n\tonWebSocket: (c, websocket: UniversalWebSocket) => {\n\t\tc.state.connectionCount += 1;\n\t\tc.log.info({\n\t\t\tmsg: \"websocket connected\",\n\t\t\tconnectionCount: c.state.connectionCount,\n\t\t});\n\n\t\twebsocket.send(\n\t\t\tJSON.stringify({\n\t\t\t\ttype: \"connected\",\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t}),\n\t\t);\n\n\t\twebsocket.addEventListener(\"message\", (event: any) => {\n\t\t\tconst data = event.data;\n\t\t\tif (typeof data === \"string\") {\n\t\t\t\ttry {\n\t\t\t\t\tconst parsed = JSON.parse(data);\n\t\t\t\t\tif (parsed.type === \"getCounts\") {\n\t\t\t\t\t\twebsocket.send(\n\t\t\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\t\t\ttype: \"counts\",\n\t\t\t\t\t\t\t\tstartCount: c.state.startCount,\n\t\t\t\t\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t\t\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t);\n\t\t\t\t\t} else if (parsed.type === \"keepAlive\") {\n\t\t\t\t\t\t// Just acknowledge to keep connection alive\n\t\t\t\t\t\twebsocket.send(JSON.stringify({ type: \"ack\" }));\n\t\t\t\t\t}\n\t\t\t\t} catch {\n\t\t\t\t\t// Echo non-JSON messages\n\t\t\t\t\twebsocket.send(data);\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\n\t\twebsocket.addEventListener(\"close\", () => {\n\t\t\tc.state.connectionCount -= 1;\n\t\t\tc.log.info({\n\t\t\t\tmsg: \"websocket disconnected\",\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t});\n\t\t});\n\t},\n\tactions: {\n\t\tgetCounts: (c) => {\n\t\t\treturn {\n\t\t\t\tstartCount: c.state.startCount,\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t};\n\t\t},\n\t},\n\toptions: {\n\t\tsleepTimeout: SLEEP_TIMEOUT,\n\t},\n});\n\nexport const sleepWithNoSleepOption = actor({\n\tstate: { startCount: 0, sleepCount: 0 },\n\tonWake: (c) => {\n\t\tc.state.startCount += 1;\n\t},\n\tonSleep: (c) => {\n\t\tc.state.sleepCount += 1;\n\t},\n\tactions: {\n\t\tgetCounts: (c) => {\n\t\t\treturn {\n\t\t\t\tstartCount: c.state.startCount,\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t};\n\t\t},\n\t},\n\toptions: {\n\t\tsleepTimeout: SLEEP_TIMEOUT,\n\t\tnoSleep: true,\n\t},\n});\n","import { actor, event } from \"rivetkit\";\n\nexport const scheduled = actor({\n\tstate: {\n\t\tlastRun: 0,\n\t\tscheduledCount: 0,\n\t\ttaskHistory: [] as string[],\n\t},\n\tevents: {\n\t\tscheduled: event<{ time: number; count: number }>(),\n\t\tscheduledWithId: event<{ taskId: string; time: number; count: number }>(),\n\t},\n\tactions: {\n\t\t// Schedule using 'at' with specific timestamp\n\t\tscheduleTaskAt: (c, timestamp: number) => {\n\t\t\tc.schedule.at(timestamp, \"onScheduledTask\");\n\t\t\treturn timestamp;\n\t\t},\n\n\t\t// Schedule using 'after' with delay\n\t\tscheduleTaskAfter: (c, delayMs: number) => {\n\t\t\tc.schedule.after(delayMs, \"onScheduledTask\");\n\t\t\treturn Date.now() + delayMs;\n\t\t},\n\n\t\t// Schedule with a task ID for ordering tests\n\t\tscheduleTaskAfterWithId: (c, taskId: string, delayMs: number) => {\n\t\t\tc.schedule.after(delayMs, \"onScheduledTaskWithId\", taskId);\n\t\t\treturn { taskId, scheduledFor: Date.now() + delayMs };\n\t\t},\n\n\t\t// Original method for backward compatibility\n\t\tscheduleTask: (c, delayMs: number) => {\n\t\t\tconst timestamp = Date.now() + delayMs;\n\t\t\tc.schedule.at(timestamp, \"onScheduledTask\");\n\t\t\treturn timestamp;\n\t\t},\n\n\t\t// Getters for state\n\t\tgetLastRun: (c) => {\n\t\t\treturn c.state.lastRun;\n\t\t},\n\n\t\tgetScheduledCount: (c) => {\n\t\t\treturn c.state.scheduledCount;\n\t\t},\n\n\t\tgetTaskHistory: (c) => {\n\t\t\treturn c.state.taskHistory;\n\t\t},\n\n\t\tclearHistory: (c) => {\n\t\t\tc.state.taskHistory = [];\n\t\t\tc.state.scheduledCount = 0;\n\t\t\tc.state.lastRun = 0;\n\t\t\treturn true;\n\t\t},\n\n\t\t// Scheduled task handlers\n\t\tonScheduledTask: (c) => {\n\t\t\tc.state.lastRun = Date.now();\n\t\t\tc.state.scheduledCount++;\n\t\t\tc.broadcast(\"scheduled\", {\n\t\t\t\ttime: c.state.lastRun,\n\t\t\t\tcount: c.state.scheduledCount,\n\t\t\t});\n\t\t},\n\n\t\tonScheduledTaskWithId: (c, taskId: string) => {\n\t\t\tc.state.lastRun = Date.now();\n\t\t\tc.state.scheduledCount++;\n\t\t\tc.state.taskHistory.push(taskId);\n\t\t\tc.broadcast(\"scheduledWithId\", {\n\t\t\t\ttaskId,\n\t\t\t\ttime: c.state.lastRun,\n\t\t\t\tcount: c.state.scheduledCount,\n\t\t\t});\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport type { registry } from \"../../index.ts\";\n\nexport const destroyObserver = actor({\n\tstate: { destroyedActors: [] as string[] },\n\tactions: {\n\t\tnotifyDestroyed: (c, actorKey: string) => {\n\t\t\tc.state.destroyedActors.push(actorKey);\n\t\t},\n\t\twasDestroyed: (c, actorKey: string) => {\n\t\t\treturn c.state.destroyedActors.includes(actorKey);\n\t\t},\n\t\treset: (c) => {\n\t\t\tc.state.destroyedActors = [];\n\t\t},\n\t},\n});\n\nexport const destroyActor = actor({\n\tstate: { value: 0, key: \"\" },\n\tonWake: (c) => {\n\t\t// Store the actor key so we can reference it in onDestroy\n\t\tc.state.key = c.key.join(\"/\");\n\t},\n\tonDestroy: async (c) => {\n\t\tconst client = c.client();\n\t\tconst observer = client.destroyObserver.getOrCreate([\"observer\"]);\n\t\tawait observer.notifyDestroyed(c.state.key);\n\t},\n\tactions: {\n\t\tsetValue: async (c, newValue: number) => {\n\t\t\tc.state.value = newValue;\n\t\t\tawait c.saveState({ immediate: true });\n\t\t\treturn c.state.value;\n\t\t},\n\t\tgetValue: (c) => {\n\t\t\treturn c.state.value;\n\t\t},\n\t\tdestroy: (c) => {\n\t\t\tc.destroy();\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\n\nexport const HIBERNATION_SLEEP_TIMEOUT = 500;\n\nexport type HibernationConnState = {\n\tcount: number;\n\tconnectCount: number;\n\tdisconnectCount: number;\n};\n\nexport const hibernationActor = actor({\n\tstate: {\n\t\tsleepCount: 0,\n\t\twakeCount: 0,\n\t},\n\tcreateConnState: (c): HibernationConnState => {\n\t\treturn {\n\t\t\tcount: 0,\n\t\t\tconnectCount: 0,\n\t\t\tdisconnectCount: 0,\n\t\t};\n\t},\n\tonWake: (c) => {\n\t\tc.state.wakeCount += 1;\n\t},\n\tonSleep: (c) => {\n\t\tc.state.sleepCount += 1;\n\t},\n\tonConnect: (c, conn) => {\n\t\tconn.state.connectCount += 1;\n\t},\n\tonDisconnect: (c, conn) => {\n\t\tconn.state.disconnectCount += 1;\n\t},\n\tactions: {\n\t\t// Basic RPC that returns a simple value\n\t\tping: (c) => {\n\t\t\treturn \"pong\";\n\t\t},\n\t\t// Increment the connection's count\n\t\tconnIncrement: (c) => {\n\t\t\tc.conn.state.count += 1;\n\t\t\treturn c.conn.state.count;\n\t\t},\n\t\t// Get the connection's count\n\t\tgetConnCount: (c) => {\n\t\t\treturn c.conn.state.count;\n\t\t},\n\t\t// Get the connection's lifecycle counts\n\t\tgetConnLifecycleCounts: (c) => {\n\t\t\treturn {\n\t\t\t\tconnectCount: c.conn.state.connectCount,\n\t\t\t\tdisconnectCount: c.conn.state.disconnectCount,\n\t\t\t};\n\t\t},\n\t\t// Get all connection IDs\n\t\tgetConnectionIds: (c) => {\n\t\t\treturn c.conns\n\t\t\t\t.entries()\n\t\t\t\t.map((x) => x[0])\n\t\t\t\t.toArray();\n\t\t},\n\t\t// Get actor sleep/wake counts\n\t\tgetActorCounts: (c) => {\n\t\t\treturn {\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t\twakeCount: c.state.wakeCount,\n\t\t\t};\n\t\t},\n\t\t// Trigger sleep\n\t\ttriggerSleep: (c) => {\n\t\t\tc.sleep();\n\t\t},\n\t},\n\toptions: {\n\t\tsleepTimeout: HIBERNATION_SLEEP_TIMEOUT,\n\t},\n});\n","import { actor, event, queue } from \"rivetkit\";\n\nexport interface WorkerJob {\n\tid: string;\n\tpayload: string;\n}\n\nexport interface WorkerState {\n\tstatus: \"idle\" | \"running\";\n\tprocessed: number;\n\tlastJob: WorkerJob | null;\n}\n\nexport const worker = actor({\n\tstate: {\n\t\tstatus: \"idle\" as \"idle\" | \"running\",\n\t\tprocessed: 0,\n\t\tlastJob: null as WorkerJob | null,\n\t},\n\tevents: {\n\t\tstatusChanged: event<{ status: \"idle\" | \"running\"; processed: number }>(),\n\t\tjobProcessed: event<{ processed: number; job: WorkerJob }>(),\n\t},\n\tqueues: {\n\t\tjobs: queue(),\n\t},\n\tasync run(c) {\n\t\tc.state.status = \"running\";\n\t\tc.broadcast(\"statusChanged\", {\n\t\t\tstatus: c.state.status,\n\t\t\tprocessed: c.state.processed,\n\t\t});\n\n\t\tfor await (const job of c.queue.iter()) {\n\t\t\tc.state.processed += 1;\n\t\t\tc.state.lastJob = job.body;\n\t\t\tc.broadcast(\"jobProcessed\", {\n\t\t\t\tprocessed: c.state.processed,\n\t\t\t\tjob: job.body,\n\t\t\t});\n\t\t}\n\n\t\tc.state.status = \"idle\";\n\t},\n\tactions: {\n\t\tgetState(c): WorkerState {\n\t\t\treturn {\n\t\t\t\tstatus: c.state.status,\n\t\t\t\tprocessed: c.state.processed,\n\t\t\t\tlastJob: c.state.lastJob,\n\t\t\t};\n\t\t},\n\t},\n});\n","import { actor, event, queue } from \"rivetkit\";\n\nconst DEFAULT_TIMEOUT_MS = 2_000;\n\nexport interface WorkerTimeoutJob {\n\tid: string;\n\tpayload: string;\n}\n\nexport interface WorkerTimeoutState {\n\tstatus: \"idle\" | \"running\";\n\tprocessed: number;\n\tticks: number;\n\tlastTickAt: number | null;\n\tlastJob: WorkerTimeoutJob | null;\n\ttimeoutMs: number;\n}\n\nexport const workerTimeout = actor({\n\tstate: {\n\t\tstatus: \"idle\" as \"idle\" | \"running\",\n\t\tprocessed: 0,\n\t\tticks: 0,\n\t\tlastTickAt: null as number | null,\n\t\tlastJob: null as WorkerTimeoutJob | null,\n\t\ttimeoutMs: DEFAULT_TIMEOUT_MS,\n\t},\n\tevents: {\n\t\ttick: event<{ ticks: number; at: number }>(),\n\t\tjobProcessed: event<{ processed: number; job: WorkerTimeoutJob }>(),\n\t},\n\tqueues: {\n\t\tjobs: queue(),\n\t},\n\trun: async (c) => {\n\t\tc.state.status = \"running\";\n\n\t\twhile (!c.aborted) {\n\t\t\tconst message = await c.queue.next({\n\t\t\t\tnames: [\"jobs\"],\n\t\t\t\ttimeout: c.state.timeoutMs,\n\t\t\t});\n\n\t\t\tif (!message) {\n\t\t\t\tconst at = Date.now();\n\t\t\t\tc.state.ticks += 1;\n\t\t\t\tc.state.lastTickAt = at;\n\t\t\t\tc.broadcast(\"tick\", {\n\t\t\t\t\tticks: c.state.ticks,\n\t\t\t\t\tat,\n\t\t\t\t});\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tc.state.processed += 1;\n\t\t\tc.state.lastJob = message.body;\n\t\t\tc.broadcast(\"jobProcessed\", {\n\t\t\t\tprocessed: c.state.processed,\n\t\t\t\tjob: message.body,\n\t\t\t});\n\t\t}\n\n\t\tc.state.status = \"idle\";\n\t},\n\tactions: {\n\t\tenqueueJob: async (c, payload: string) => {\n\t\t\tconst job = {\n\t\t\t\tid: crypto.randomUUID(),\n\t\t\t\tpayload,\n\t\t\t};\n\t\t\tawait c.queue.send(\"jobs\", job);\n\t\t\treturn job;\n\t\t},\n\t\tsetTimeoutMs: (c, timeoutMs: number) => {\n\t\t\tc.state.timeoutMs = Math.max(100, Math.floor(timeoutMs));\n\t\t\treturn c.state.timeoutMs;\n\t\t},\n\t\tgetState: (c): WorkerTimeoutState => c.state,\n\t},\n});\n","import { actor, event, queue } from \"rivetkit\";\nimport { Loop, type WorkflowLoopContextOf, workflow } from \"rivetkit/workflow\";\n\nconst WORKFLOW_GUARD_KV_KEY = \"__rivet_actor_workflow_guard_triggered\";\n\nconst WORKFLOW_QUEUE_NAME = \"workflow-default\";\nconst WORKFLOW_TIMEOUT_QUEUE_NAME = \"workflow-timeout\";\n\nexport const workflowCounterActor = actor({\n\tstate: {\n\t\trunCount: 0,\n\t\tguardTriggered: false,\n\t\thistory: [] as number[],\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"counter\", async (loopCtx) => {\n\t\t\t\ttry {\n\t\t\t\t\t// Accessing state outside a step should throw.\n\t\t\t\t\t// biome-ignore lint/style/noUnusedExpressions: intentionally checking accessor.\n\t\t\t\t\tloopCtx.state;\n\t\t\t\t} catch {}\n\n\t\t\t\tawait loopCtx.step(\"increment\", async () => {\n\t\t\t\t\tincrementWorkflowCounter(loopCtx);\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.sleep(\"idle\", 25);\n\t\t\t\treturn Loop.continue(undefined);\n\t\t\t});\n\t}),\n\tactions: {\n\t\tgetState: async (c) => {\n\t\t\tconst guardFlag = await c.kv.get(WORKFLOW_GUARD_KV_KEY);\n\t\t\tif (guardFlag === \"true\") {\n\t\t\t\tc.state.guardTriggered = true;\n\t\t\t}\n\t\t\treturn c.state;\n\t\t},\n\t},\n\toptions: {\n\t\tsleepTimeout: 50,\n\t},\n});\n\nexport const workflowQueueActor = actor({\n\tstate: {\n\t\treceived: [] as unknown[],\n\t},\n\tqueues: {\n\t\t[WORKFLOW_QUEUE_NAME]: queue(),\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"queue\", async (loopCtx) => {\n\t\t\t\tconst message = await loopCtx.queue.next(\"queue-wait\", {\n\t\t\t\t\tnames: [WORKFLOW_QUEUE_NAME],\n\t\t\t\t\tcompletable: true,\n\t\t\t\t});\n\t\t\t\tif (!message.complete) {\n\t\t\t\t\treturn Loop.continue(undefined);\n\t\t\t\t}\n\t\t\t\tconst complete = message.complete;\n\t\t\t\tawait loopCtx.step(\"store-message\", async () => {\n\t\t\t\t\tawait storeWorkflowQueueMessage(loopCtx, message.body, complete);\n\t\t\t\t});\n\t\t\t\treturn Loop.continue(undefined);\n\t\t\t});\n\t}),\n\tactions: {\n\t\tgetMessages: (c) => c.state.received,\n\t},\n});\n\nexport const workflowSleepActor = actor({\n\tstate: {\n\t\tticks: 0,\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"sleep\", async (loopCtx) => {\n\t\t\t\tawait loopCtx.step(\"tick\", async () => {\n\t\t\t\t\tincrementWorkflowSleepTick(loopCtx);\n\t\t\t\t});\n\t\t\t\tawait loopCtx.sleep(\"delay\", 40);\n\t\t\t\treturn Loop.continue(undefined);\n\t\t\t});\n\t}),\n\tactions: {\n\t\tgetState: (c) => c.state,\n\t},\n\toptions: {\n\t\tsleepTimeout: 50,\n\t},\n});\n\nexport const workflowQueueTimeoutActor = actor({\n\tstate: {\n\t\tprocessed: 0,\n\t\tticks: 0,\n\t\tlastTickAt: null as number | null,\n\t\tlastJob: null as { id: string; payload: string } | null,\n\t\ttimeoutMs: 2_000,\n\t},\n\tevents: {\n\t\ttick: event<{ ticks: number; at: number }>(),\n\t\tjobProcessed: event<{ processed: number; job: { id: string; payload: string } }>(),\n\t},\n\tqueues: {\n\t\t[WORKFLOW_TIMEOUT_QUEUE_NAME]: queue<{ id: string; payload: string }>(),\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"queue-timeout-loop\", async (loopCtx) => {\n\t\t\t\tconst timeoutMs = await loopCtx.step(\"read-timeout\", async () => {\n\t\t\t\t\treturn readWorkflowTimeoutMs(loopCtx);\n\t\t\t\t});\n\n\t\t\t\tconst [message] = await loopCtx.queue.nextBatch(\"wait-job-or-timeout\", {\n\t\t\t\t\tnames: [WORKFLOW_TIMEOUT_QUEUE_NAME],\n\t\t\t\t\ttimeout: timeoutMs,\n\t\t\t\t});\n\n\t\t\t\tif (!message) {\n\t\t\t\t\tawait loopCtx.step(\"tick\", async () => {\n\t\t\t\t\t\trecordWorkflowTimeoutTick(loopCtx);\n\t\t\t\t\t});\n\t\t\t\t\treturn Loop.continue(undefined);\n\t\t\t\t}\n\n\t\t\t\tawait loopCtx.step(\"process-job\", async () => {\n\t\t\t\t\tprocessWorkflowTimeoutJob(loopCtx, message.body);\n\t\t\t\t});\n\t\t\t\treturn Loop.continue(undefined);\n\t\t\t});\n\t}),\n\tactions: {\n\t\tenqueueJob: async (c, payload: string) => {\n\t\t\tconst job = { id: crypto.randomUUID(), payload };\n\t\t\tawait c.queue.send(WORKFLOW_TIMEOUT_QUEUE_NAME, job);\n\t\t\treturn job;\n\t\t},\n\t\tsetTimeoutMs: (c, timeoutMs: number) => {\n\t\t\tc.state.timeoutMs = Math.max(100, Math.floor(timeoutMs));\n\t\t\treturn c.state.timeoutMs;\n\t\t},\n\t\tgetState: (c) => c.state,\n\t},\n});\n\nfunction incrementWorkflowCounter(\n\tctx: WorkflowLoopContextOf,\n): void {\n\tctx.state.runCount += 1;\n\tctx.state.history.push(ctx.state.runCount);\n}\n\nasync function storeWorkflowQueueMessage(\n\tctx: WorkflowLoopContextOf,\n\tbody: unknown,\n\tcomplete: (response: { echo: unknown }) => Promise,\n): Promise {\n\tctx.state.received.push(body);\n\tawait complete({ echo: body });\n}\n\nfunction incrementWorkflowSleepTick(\n\tctx: WorkflowLoopContextOf,\n): void {\n\tctx.state.ticks += 1;\n}\n\nfunction readWorkflowTimeoutMs(\n\tctx: WorkflowLoopContextOf,\n): number {\n\treturn ctx.state.timeoutMs;\n}\n\nfunction recordWorkflowTimeoutTick(\n\tctx: WorkflowLoopContextOf,\n): void {\n\tconst at = Date.now();\n\tctx.state.ticks += 1;\n\tctx.state.lastTickAt = at;\n\tctx.broadcast(\"tick\", {\n\t\tticks: ctx.state.ticks,\n\t\tat,\n\t});\n}\n\nfunction processWorkflowTimeoutJob(\n\tctx: WorkflowLoopContextOf,\n\tjob: { id: string; payload: string },\n): void {\n\tctx.state.processed += 1;\n\tctx.state.lastJob = job;\n\tctx.broadcast(\"jobProcessed\", {\n\t\tprocessed: ctx.state.processed,\n\t\tjob,\n\t});\n}\n\nexport { WORKFLOW_QUEUE_NAME, WORKFLOW_TIMEOUT_QUEUE_NAME };\n","// TIMER (Sleep Demo)\n// Demonstrates: Durable sleep that survives restarts\n// One actor per timer - actor key is the timer ID\n\nimport { actor, event } from \"rivetkit\";\nimport { Loop, workflow } from \"rivetkit/workflow\";\nimport { actorCtx } from \"./_helpers.ts\";\n\nexport type Timer = {\n\tid: string;\n\tname: string;\n\tdurationMs: number;\n\tstartedAt: number;\n\tcompletedAt?: number;\n};\n\ntype State = Timer;\n\nexport type TimerInput = {\n\tname?: string;\n\tdurationMs?: number;\n};\n\nexport const timer = actor({\n\tcreateState: (c, input?: TimerInput): Timer => ({\n\t\tid: c.key[0] as string,\n\t\tname: input?.name ?? \"Timer\",\n\t\tdurationMs: input?.durationMs ?? 10000,\n\t\tstartedAt: Date.now(),\n\t}),\n\tevents: {\n\t\ttimerStarted: event(),\n\t\ttimerCompleted: event(),\n\t},\n\n\tactions: {\n\t\tgetTimer: (c): Timer => c.state,\n\t},\n\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"timer-loop\", async (loopCtx) => {\n\t\t\t\tconst c = actorCtx(loopCtx);\n\n\t\t\t\t// Get duration inside a step since state is only available in steps\n\t\t\t\tconst durationMs = await loopCtx.step(\"start-timer\", async () => {\n\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\tmsg: \"starting timer\",\n\t\t\t\t\t\ttimerId: c.state.id,\n\t\t\t\t\t\tdurationMs: c.state.durationMs,\n\t\t\t\t\t});\n\t\t\t\t\tc.broadcast(\"timerStarted\", c.state);\n\t\t\t\t\treturn c.state.durationMs;\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.sleep(\"countdown\", durationMs);\n\n\t\t\t\tawait loopCtx.step(\"complete-timer\", async () => {\n\t\t\t\t\tc.state.completedAt = Date.now();\n\t\t\t\t\tc.broadcast(\"timerCompleted\", c.state);\n\t\t\t\t\tctx.log.info({ msg: \"timer completed\", timerId: c.state.id });\n\t\t\t\t});\n\n\t\t\t\treturn Loop.break(undefined);\n\t\t\t});\n\t}),\n\n\toptions: {\n\t\tsleepTimeout: 1000,\n\t},\n});\n","// Type helper - cast loop context to access actor-specific properties\n// Only call these helpers INSIDE a step callback where state access is allowed\n// biome-ignore lint/suspicious/noExplicitAny: Workflow context typing workaround\nexport type ActorLoopContext = {\n\tstate: S;\n\tbroadcast: (name: string, ...args: unknown[]) => void;\n};\n\n// biome-ignore lint/suspicious/noExplicitAny: Workflow context typing workaround\nexport function actorCtx(ctx: unknown): ActorLoopContext {\n\treturn ctx as any;\n}\n","// ORDER PROCESSOR (Steps Demo)\n// Demonstrates: Sequential workflow steps with automatic retries\n// One actor per order - actor key is the order ID\n\nimport { actor, event } from \"rivetkit\";\nimport { Loop, workflow } from \"rivetkit/workflow\";\nimport { actorCtx } from \"./_helpers.ts\";\n\nexport type OrderStatus =\n\t| \"pending\"\n\t| \"validating\"\n\t| \"charging\"\n\t| \"fulfilling\"\n\t| \"completed\"\n\t| \"failed\";\n\nexport type Order = {\n\tid: string;\n\tstatus: OrderStatus;\n\tstep: number;\n\terror?: string;\n\tcreatedAt: number;\n\tcompletedAt?: number;\n};\n\ntype State = Order;\n\nasync function simulateWork(name: string, failChance = 0.1): Promise {\n\tawait new Promise((resolve) =>\n\t\tsetTimeout(resolve, 500 + Math.random() * 1000)\n\t);\n\tif (Math.random() < failChance) {\n\t\tthrow new Error(`${name} failed (simulated)`);\n\t}\n}\n\nexport const order = actor({\n\tcreateState: (c): Order => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"pending\",\n\t\tstep: 0,\n\t\tcreatedAt: Date.now(),\n\t}),\n\tevents: {\n\t\torderUpdated: event(),\n\t},\n\n\tactions: {\n\t\tgetOrder: (c): Order => c.state,\n\t},\n\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"process-order\", async (loopCtx) => {\n\t\t\t\tconst c = actorCtx(loopCtx);\n\n\t\t\t\tawait loopCtx.step(\"validate\", async () => {\n\t\t\t\t\tctx.log.info({ msg: \"processing order\", orderId: c.state.id });\n\t\t\t\t\tc.state.status = \"validating\";\n\t\t\t\t\tc.state.step = 1;\n\t\t\t\t\tc.broadcast(\"orderUpdated\", c.state);\n\t\t\t\t\tawait simulateWork(\"validation\", 0.05);\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.step(\"charge\", async () => {\n\t\t\t\t\tc.state.status = \"charging\";\n\t\t\t\t\tc.state.step = 2;\n\t\t\t\t\tc.broadcast(\"orderUpdated\", c.state);\n\t\t\t\t\tawait simulateWork(\"payment\", 0.1);\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.step(\"fulfill\", async () => {\n\t\t\t\t\tc.state.status = \"fulfilling\";\n\t\t\t\t\tc.state.step = 3;\n\t\t\t\t\tc.broadcast(\"orderUpdated\", c.state);\n\t\t\t\t\tawait simulateWork(\"fulfillment\", 0.05);\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.step(\"complete\", async () => {\n\t\t\t\t\tc.state.status = \"completed\";\n\t\t\t\t\tc.state.step = 4;\n\t\t\t\t\tc.state.completedAt = Date.now();\n\t\t\t\t\tc.broadcast(\"orderUpdated\", c.state);\n\t\t\t\t\tctx.log.info({ msg: \"order completed\", orderId: c.state.id });\n\t\t\t\t});\n\n\t\t\t\treturn Loop.break(undefined);\n\t\t\t});\n\t}),\n});\n","// BATCH PROCESSOR (Loops Demo)\n// Demonstrates: Loop with persistent state (cursor) for batch processing\n// One actor per batch job - actor key is the job ID\n\nimport { actor, event } from \"rivetkit\";\nimport { Loop, workflow } from \"rivetkit/workflow\";\nimport { actorCtx } from \"./_helpers.ts\";\n\nexport type BatchInfo = {\n\tid: number;\n\tcount: number;\n\tprocessedAt: number;\n};\n\nexport type BatchJob = {\n\tid: string;\n\ttotalItems: number;\n\tbatchSize: number;\n\tstatus: \"running\" | \"stopped\" | \"completed\";\n\tprocessedTotal: number;\n\tcurrentBatch: number;\n\tbatches: BatchInfo[];\n\tstartedAt: number;\n\tcompletedAt?: number;\n};\n\ntype State = BatchJob;\n\nfunction fetchBatch(\n\tcursor: number,\n\tbatchSize: number,\n\ttotalItems: number\n): { items: number[]; hasMore: boolean } {\n\tconst start = cursor * batchSize;\n\tconst end = Math.min(start + batchSize, totalItems);\n\tconst items = [];\n\tfor (let i = start; i < end; i++) {\n\t\titems.push(i);\n\t}\n\treturn {\n\t\titems,\n\t\thasMore: end < totalItems,\n\t};\n}\n\nexport type BatchJobInput = {\n\ttotalItems?: number;\n\tbatchSize?: number;\n};\n\nexport const batch = actor({\n\tcreateState: (c, input?: BatchJobInput): BatchJob => ({\n\t\tid: c.key[0] as string,\n\t\ttotalItems: input?.totalItems ?? 50,\n\t\tbatchSize: input?.batchSize ?? 5,\n\t\tstatus: \"running\",\n\t\tprocessedTotal: 0,\n\t\tcurrentBatch: 0,\n\t\tbatches: [],\n\t\tstartedAt: Date.now(),\n\t}),\n\tevents: {\n\t\tbatchProcessed: event(),\n\t\tstateChanged: event(),\n\t\tprocessingComplete: event<{ totalBatches: number; totalItems: number }>(),\n\t},\n\n\tactions: {\n\t\tgetJob: (c): BatchJob => c.state,\n\t},\n\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop({\n\t\t\tname: \"batch-loop\",\n\t\t\tstate: { cursor: 0 },\n\t\t\trun: async (batchCtx, loopState: { cursor: number }) => {\n\t\t\t\tconst c = actorCtx(batchCtx);\n\n\t\t\t\tconst batch = await batchCtx.step(\"fetch-batch\", async () => {\n\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\tmsg: \"processing batch\",\n\t\t\t\t\t\tjobId: c.state.id,\n\t\t\t\t\t\tcursor: loopState.cursor,\n\t\t\t\t\t});\n\t\t\t\t\tawait new Promise((r) => setTimeout(r, 200 + Math.random() * 300));\n\t\t\t\t\treturn fetchBatch(loopState.cursor, c.state.batchSize, c.state.totalItems);\n\t\t\t\t});\n\n\t\t\t\tawait batchCtx.step(\"process-batch\", async () => {\n\t\t\t\t\tawait new Promise((r) => setTimeout(r, 300 + Math.random() * 500));\n\n\t\t\t\t\tconst batchInfo: BatchInfo = {\n\t\t\t\t\t\tid: loopState.cursor,\n\t\t\t\t\t\tcount: batch.items.length,\n\t\t\t\t\t\tprocessedAt: Date.now(),\n\t\t\t\t\t};\n\n\t\t\t\t\tc.state.currentBatch = loopState.cursor;\n\t\t\t\t\tc.state.processedTotal += batch.items.length;\n\t\t\t\t\tc.state.batches.push(batchInfo);\n\n\t\t\t\t\tc.broadcast(\"batchProcessed\", batchInfo);\n\t\t\t\t\tc.broadcast(\"stateChanged\", c.state);\n\n\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\tmsg: \"batch processed\",\n\t\t\t\t\t\tjobId: c.state.id,\n\t\t\t\t\t\tcursor: loopState.cursor,\n\t\t\t\t\t\tcount: batch.items.length,\n\t\t\t\t\t});\n\t\t\t\t});\n\n\t\t\t\tif (!batch.hasMore) {\n\t\t\t\t\tawait batchCtx.step(\"mark-complete\", async () => {\n\t\t\t\t\t\tc.state.status = \"completed\";\n\t\t\t\t\t\tc.state.completedAt = Date.now();\n\t\t\t\t\t\tc.broadcast(\"stateChanged\", c.state);\n\t\t\t\t\t\tc.broadcast(\"processingComplete\", {\n\t\t\t\t\t\t\ttotalBatches: loopState.cursor + 1,\n\t\t\t\t\t\t\ttotalItems: c.state.processedTotal,\n\t\t\t\t\t\t});\n\t\t\t\t\t});\n\t\t\t\t\treturn Loop.break(loopState.cursor + 1);\n\t\t\t\t}\n\n\t\t\t\treturn Loop.continue({ cursor: loopState.cursor + 1 });\n\t\t\t},\n\t\t});\n\t}),\n});\n","// APPROVAL REQUEST (Queue Wait Demo)\n// Demonstrates: Queue waits with timeout for approval workflows\n// One actor per approval request - actor key is the request ID\n\nimport { actor, event, queue } from \"rivetkit\";\nimport { Loop, workflow } from \"rivetkit/workflow\";\nimport { actorCtx } from \"./_helpers.ts\";\n\nexport type RequestStatus = \"pending\" | \"approved\" | \"rejected\" | \"timeout\";\n\nexport type ApprovalRequest = {\n\tid: string;\n\ttitle: string;\n\tdescription: string;\n\tstatus: RequestStatus;\n\tcreatedAt: number;\n\tdecidedAt?: number;\n\tdecidedBy?: string;\n\tdeciding?: boolean; // True when a decision is being processed\n};\n\ntype State = ApprovalRequest;\n\nconst QUEUE_DECISION = \"decision\" as const;\n\nconst APPROVAL_TIMEOUT_MS = 30000;\n\nexport type ApprovalRequestInput = {\n\ttitle?: string;\n\tdescription?: string;\n};\n\ntype ApprovalDecision = {\n\tapproved: boolean;\n\tapprover: string;\n};\n\nexport const approval = actor({\n\tcreateState: (c, input?: ApprovalRequestInput): ApprovalRequest => ({\n\t\tid: c.key[0] as string,\n\t\ttitle: input?.title ?? \"Untitled Request\",\n\t\tdescription: input?.description ?? \"\",\n\t\tstatus: \"pending\",\n\t\tcreatedAt: Date.now(),\n\t}),\n\tqueues: {\n\t\tdecision: queue(),\n\t},\n\tevents: {\n\t\trequestUpdated: event(),\n\t\trequestCreated: event(),\n\t},\n\n\tactions: {\n\t\tgetRequest: (c): ApprovalRequest => c.state,\n\n\t\tapprove: async (c, approver: string) => {\n\t\t\tif (c.state.status !== \"pending\") return;\n\t\t\tc.state.deciding = true;\n\t\t\tc.broadcast(\"requestUpdated\", c.state);\n\t\t\tawait c.queue.send(QUEUE_DECISION, { approved: true, approver });\n\t\t},\n\n\t\treject: async (c, approver: string) => {\n\t\t\tif (c.state.status !== \"pending\") return;\n\t\t\tc.state.deciding = true;\n\t\t\tc.broadcast(\"requestUpdated\", c.state);\n\t\t\tawait c.queue.send(QUEUE_DECISION, { approved: false, approver });\n\t\t},\n\t},\n\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"approval-loop\", async (loopCtx) => {\n\t\t\t\tconst c = actorCtx(loopCtx);\n\n\t\t\t\tawait loopCtx.step(\"init-request\", async () => {\n\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\tmsg: \"waiting for approval decision\",\n\t\t\t\t\t\trequestId: c.state.id,\n\t\t\t\t\t\ttitle: c.state.title,\n\t\t\t\t\t});\n\t\t\t\t\tc.broadcast(\"requestCreated\", c.state);\n\t\t\t\t});\n\n\t\t\t\tconst [decisionMessage] = await loopCtx.queue.nextBatch(\n\t\t\t\t\t\"wait-decision\",\n\t\t\t\t\t{\n\t\t\t\t\t\tnames: [QUEUE_DECISION],\n\t\t\t\t\t\ttimeout: APPROVAL_TIMEOUT_MS,\n\t\t\t\t\t},\n\t\t\t\t);\n\t\t\t\tconst decision = decisionMessage?.body ?? null;\n\n\t\t\t\tawait loopCtx.step(\"update-status\", async () => {\n\t\t\t\t\tc.state.deciding = false;\n\t\t\t\t\tif (decision === null) {\n\t\t\t\t\t\tc.state.status = \"timeout\";\n\t\t\t\t\t\tctx.log.info({ msg: \"request timed out\", requestId: c.state.id });\n\t\t\t\t\t} else if (decision.approved) {\n\t\t\t\t\t\tc.state.status = \"approved\";\n\t\t\t\t\t\tc.state.decidedBy = decision.approver;\n\t\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\t\tmsg: \"request approved\",\n\t\t\t\t\t\t\trequestId: c.state.id,\n\t\t\t\t\t\t\tapprover: decision.approver,\n\t\t\t\t\t\t});\n\t\t\t\t\t} else {\n\t\t\t\t\t\tc.state.status = \"rejected\";\n\t\t\t\t\t\tc.state.decidedBy = decision.approver;\n\t\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\t\tmsg: \"request rejected\",\n\t\t\t\t\t\t\trequestId: c.state.id,\n\t\t\t\t\t\t\tapprover: decision.approver,\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\t\t\t\t\tc.state.decidedAt = Date.now();\n\t\t\t\t\tc.broadcast(\"requestUpdated\", c.state);\n\t\t\t\t});\n\n\t\t\t\treturn Loop.break(undefined);\n\t\t\t});\n\t}),\n});\n","// DASHBOARD (Join Demo)\n// Demonstrates: Parallel data fetching with join (wait-all)\n\nimport { actor, event, queue } from \"rivetkit\";\nimport { Loop, workflow } from \"rivetkit/workflow\";\nimport { actorCtx } from \"./_helpers.ts\";\n\nexport type UserStats = {\n\tcount: number;\n\tactiveToday: number;\n\tnewThisWeek: number;\n};\n\nexport type OrderStats = {\n\tcount: number;\n\trevenue: number;\n\tavgOrderValue: number;\n};\n\nexport type MetricsStats = {\n\tpageViews: number;\n\tsessions: number;\n\tbounceRate: number;\n};\n\nexport type DashboardData = {\n\tusers: UserStats;\n\torders: OrderStats;\n\tmetrics: MetricsStats;\n\tfetchedAt: number;\n};\n\nexport type BranchStatus = \"pending\" | \"running\" | \"completed\" | \"failed\";\n\nexport type DashboardState = {\n\tdata: DashboardData | null;\n\tloading: boolean;\n\tbranches: {\n\t\tusers: BranchStatus;\n\t\torders: BranchStatus;\n\t\tmetrics: BranchStatus;\n\t};\n\tlastRefresh: number | null;\n};\n\ntype State = DashboardState;\n\nconst QUEUE_REFRESH = \"refresh\";\ntype RefreshMessage = Record;\n\nasync function fetchUserStats(): Promise {\n\tawait new Promise((r) => setTimeout(r, 800 + Math.random() * 1200));\n\treturn {\n\t\tcount: Math.floor(1000 + Math.random() * 500),\n\t\tactiveToday: Math.floor(100 + Math.random() * 200),\n\t\tnewThisWeek: Math.floor(20 + Math.random() * 80),\n\t};\n}\n\nasync function fetchOrderStats(): Promise {\n\tawait new Promise((r) => setTimeout(r, 600 + Math.random() * 1000));\n\tconst count = Math.floor(50 + Math.random() * 150);\n\tconst revenue = Math.floor(5000 + Math.random() * 15000);\n\treturn {\n\t\tcount,\n\t\trevenue,\n\t\tavgOrderValue: Math.round(revenue / count),\n\t};\n}\n\nasync function fetchMetricsStats(): Promise {\n\tawait new Promise((r) => setTimeout(r, 400 + Math.random() * 800));\n\treturn {\n\t\tpageViews: Math.floor(10000 + Math.random() * 50000),\n\t\tsessions: Math.floor(2000 + Math.random() * 8000),\n\t\tbounceRate: Math.round(30 + Math.random() * 40),\n\t};\n}\n\nexport const dashboard = actor({\n\tstate: {\n\t\tdata: null as DashboardData | null,\n\t\tloading: false,\n\t\tbranches: {\n\t\t\tusers: \"pending\" as BranchStatus,\n\t\t\torders: \"pending\" as BranchStatus,\n\t\t\tmetrics: \"pending\" as BranchStatus,\n\t\t},\n\t\tlastRefresh: null as number | null,\n\t},\n\tqueues: {\n\t\t[QUEUE_REFRESH]: queue(),\n\t},\n\tevents: {\n\t\tstateChanged: event(),\n\t\trefreshComplete: event(),\n\t},\n\n\tactions: {\n\t\trefresh: async (c) => {\n\t\t\tif (!c.state.loading) {\n\t\t\t\tc.state.loading = true;\n\t\t\t\tc.state.branches = {\n\t\t\t\t\tusers: \"pending\",\n\t\t\t\t\torders: \"pending\",\n\t\t\t\t\tmetrics: \"pending\",\n\t\t\t\t};\n\t\t\t\tc.broadcast(\"stateChanged\", c.state);\n\t\t\t\tawait c.queue.send(QUEUE_REFRESH, {});\n\t\t\t}\n\t\t},\n\n\t\tgetState: (c): DashboardState => c.state,\n\t},\n\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"refresh-loop\", async (loopCtx) => {\n\t\t\t\tconst c = actorCtx(loopCtx);\n\n\t\t\t\tawait loopCtx.queue.next(\"wait-refresh\", {\n\t\t\t\t\tnames: [QUEUE_REFRESH],\n\t\t\t\t});\n\n\t\t\t\tctx.log.info({ msg: \"starting dashboard refresh\" });\n\n\t\t\t\tconst results = await loopCtx.join(\"fetch-all\", {\n\t\t\t\t\tusers: {\n\t\t\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\t\t\tconst bc = actorCtx(branchCtx);\n\n\t\t\t\t\t\t\tawait branchCtx.step(\"mark-running\", async () => {\n\t\t\t\t\t\t\t\tbc.state.branches.users = \"running\";\n\t\t\t\t\t\t\t\tbc.broadcast(\"stateChanged\", bc.state);\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\tconst data = await branchCtx.step(\"fetch-users\", async () => {\n\t\t\t\t\t\t\t\treturn await fetchUserStats();\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\tawait branchCtx.step(\"mark-complete\", async () => {\n\t\t\t\t\t\t\t\tbc.state.branches.users = \"completed\";\n\t\t\t\t\t\t\t\tbc.broadcast(\"stateChanged\", bc.state);\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\treturn data;\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\torders: {\n\t\t\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\t\t\tconst bc = actorCtx(branchCtx);\n\n\t\t\t\t\t\t\tawait branchCtx.step(\"mark-running\", async () => {\n\t\t\t\t\t\t\t\tbc.state.branches.orders = \"running\";\n\t\t\t\t\t\t\t\tbc.broadcast(\"stateChanged\", bc.state);\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\tconst data = await branchCtx.step(\"fetch-orders\", async () => {\n\t\t\t\t\t\t\t\treturn await fetchOrderStats();\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\tawait branchCtx.step(\"mark-complete\", async () => {\n\t\t\t\t\t\t\t\tbc.state.branches.orders = \"completed\";\n\t\t\t\t\t\t\t\tbc.broadcast(\"stateChanged\", bc.state);\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\treturn data;\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tmetrics: {\n\t\t\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\t\t\tconst bc = actorCtx(branchCtx);\n\n\t\t\t\t\t\t\tawait branchCtx.step(\"mark-running\", async () => {\n\t\t\t\t\t\t\t\tbc.state.branches.metrics = \"running\";\n\t\t\t\t\t\t\t\tbc.broadcast(\"stateChanged\", bc.state);\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\tconst data = await branchCtx.step(\"fetch-metrics\", async () => {\n\t\t\t\t\t\t\t\treturn await fetchMetricsStats();\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\tawait branchCtx.step(\"mark-complete\", async () => {\n\t\t\t\t\t\t\t\tbc.state.branches.metrics = \"completed\";\n\t\t\t\t\t\t\t\tbc.broadcast(\"stateChanged\", bc.state);\n\t\t\t\t\t\t\t});\n\n\t\t\t\t\t\t\treturn data;\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.step(\"save-data\", async () => {\n\t\t\t\t\tc.state.data = {\n\t\t\t\t\t\tusers: results.users,\n\t\t\t\t\t\torders: results.orders,\n\t\t\t\t\t\tmetrics: results.metrics,\n\t\t\t\t\t\tfetchedAt: Date.now(),\n\t\t\t\t\t};\n\t\t\t\t\tc.state.loading = false;\n\t\t\t\t\tc.state.lastRefresh = Date.now();\n\t\t\t\t\tc.broadcast(\"stateChanged\", c.state);\n\t\t\t\t\tc.broadcast(\"refreshComplete\", c.state.data);\n\t\t\t\t});\n\n\t\t\t\tctx.log.info({ msg: \"dashboard refresh complete\" });\n\n\t\t\t\treturn Loop.continue(undefined);\n\t\t\t});\n\t}),\n});\n","// RACE RUNNER (Race Demo)\n// Demonstrates: Race (parallel first-wins) for timeout patterns\n// One actor per race task - actor key is the task ID\n\nimport { actor, event } from \"rivetkit\";\nimport { Loop, workflow } from \"rivetkit/workflow\";\nimport { actorCtx } from \"./_helpers.ts\";\n\nexport type RaceTask = {\n\tid: string;\n\tworkDurationMs: number;\n\ttimeoutMs: number;\n\tstatus: \"running\" | \"work_won\" | \"timeout_won\";\n\tresult?: string;\n\tstartedAt: number;\n\tcompletedAt?: number;\n\tactualDurationMs?: number;\n};\n\ntype State = RaceTask;\n\nexport type RaceTaskInput = {\n\tworkDurationMs?: number;\n\ttimeoutMs?: number;\n};\n\nexport const race = actor({\n\tcreateState: (c, input?: RaceTaskInput): RaceTask => ({\n\t\tid: c.key[0] as string,\n\t\tworkDurationMs: input?.workDurationMs ?? 2000,\n\t\ttimeoutMs: input?.timeoutMs ?? 3000,\n\t\tstatus: \"running\",\n\t\tstartedAt: Date.now(),\n\t}),\n\tevents: {\n\t\traceStarted: event(),\n\t\traceCompleted: event(),\n\t},\n\n\tactions: {\n\t\tgetTask: (c): RaceTask => c.state,\n\t},\n\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"race-loop\", async (loopCtx) => {\n\t\t\t\tconst c = actorCtx(loopCtx);\n\n\t\t\t\t// Get durations inside a step since state is only available in steps\n\t\t\t\tconst { workDurationMs, timeoutMs, taskId } = await loopCtx.step(\n\t\t\t\t\t\"start-race\",\n\t\t\t\t\tasync () => {\n\t\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\t\tmsg: \"starting race\",\n\t\t\t\t\t\t\ttaskId: c.state.id,\n\t\t\t\t\t\t\tworkDurationMs: c.state.workDurationMs,\n\t\t\t\t\t\t\ttimeoutMs: c.state.timeoutMs,\n\t\t\t\t\t\t});\n\t\t\t\t\t\tc.broadcast(\"raceStarted\", c.state);\n\t\t\t\t\t\treturn {\n\t\t\t\t\t\t\tworkDurationMs: c.state.workDurationMs,\n\t\t\t\t\t\t\ttimeoutMs: c.state.timeoutMs,\n\t\t\t\t\t\t\ttaskId: c.state.id,\n\t\t\t\t\t\t};\n\t\t\t\t\t}\n\t\t\t\t);\n\n\t\t\t\tconst { winner, value } = await loopCtx.race(\"work-vs-timeout\", [\n\t\t\t\t\t{\n\t\t\t\t\t\tname: \"work\",\n\t\t\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\t\t\tawait branchCtx.sleep(\"simulate-work\", workDurationMs);\n\t\t\t\t\t\t\treturn await branchCtx.step(\"complete-work\", async () => {\n\t\t\t\t\t\t\t\treturn `Result for task ${taskId}`;\n\t\t\t\t\t\t\t});\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tname: \"timeout\",\n\t\t\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\t\t\tawait branchCtx.sleep(\"timeout-wait\", timeoutMs);\n\t\t\t\t\t\t\treturn null;\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t]);\n\n\t\t\t\tawait loopCtx.step(\"save-result\", async () => {\n\t\t\t\t\tc.state.completedAt = Date.now();\n\t\t\t\t\tc.state.actualDurationMs = c.state.completedAt - c.state.startedAt;\n\n\t\t\t\t\tif (winner === \"work\") {\n\t\t\t\t\t\tc.state.status = \"work_won\";\n\t\t\t\t\t\tc.state.result = value as string;\n\t\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\t\tmsg: \"work completed before timeout\",\n\t\t\t\t\t\t\ttaskId: c.state.id,\n\t\t\t\t\t\t\tdurationMs: c.state.actualDurationMs,\n\t\t\t\t\t\t});\n\t\t\t\t\t} else {\n\t\t\t\t\t\tc.state.status = \"timeout_won\";\n\t\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\t\tmsg: \"timeout won the race\",\n\t\t\t\t\t\t\ttaskId: c.state.id,\n\t\t\t\t\t\t\tdurationMs: c.state.actualDurationMs,\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\n\t\t\t\t\tc.broadcast(\"raceCompleted\", c.state);\n\t\t\t\t});\n\n\t\t\t\treturn Loop.break(undefined);\n\t\t\t});\n\t}),\n});\n","// PAYMENT PROCESSOR (Rollback Demo)\n// Demonstrates: Rollback checkpoints with compensating actions\n// One actor per transaction - actor key is the transaction ID\n\nimport { actor, event } from \"rivetkit\";\nimport { Loop, workflow } from \"rivetkit/workflow\";\nimport { actorCtx } from \"./_helpers.ts\";\n\nexport type TransactionStep = {\n\tname: string;\n\tstatus: \"pending\" | \"completed\" | \"rolled_back\";\n\tcompletedAt?: number;\n\trolledBackAt?: number;\n};\n\nexport type Transaction = {\n\tid: string;\n\tamount: number;\n\tshouldFail: boolean;\n\tstatus:\n\t\t| \"pending\"\n\t\t| \"reserving\"\n\t\t| \"charging\"\n\t\t| \"completing\"\n\t\t| \"completed\"\n\t\t| \"rolling_back\"\n\t\t| \"failed\";\n\tsteps: TransactionStep[];\n\terror?: string;\n\tstartedAt: number;\n\tcompletedAt?: number;\n};\n\ntype State = Transaction;\n\nexport type TransactionInput = {\n\tamount?: number;\n\tshouldFail?: boolean;\n};\n\nexport const payment = actor({\n\tcreateState: (c, input?: TransactionInput): Transaction => ({\n\t\tid: c.key[0] as string,\n\t\tamount: input?.amount ?? 100,\n\t\tshouldFail: input?.shouldFail ?? false,\n\t\tstatus: \"pending\",\n\t\tsteps: [\n\t\t\t{ name: \"reserve-inventory\", status: \"pending\" },\n\t\t\t{ name: \"charge-card\", status: \"pending\" },\n\t\t\t{ name: \"complete-order\", status: \"pending\" },\n\t\t],\n\t\tstartedAt: Date.now(),\n\t}),\n\tevents: {\n\t\ttransactionStarted: event(),\n\t\ttransactionUpdated: event(),\n\t\ttransactionCompleted: event(),\n\t},\n\n\tactions: {\n\t\tgetTransaction: (c): Transaction => c.state,\n\t},\n\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.loop(\"payment-loop\", async (loopCtx) => {\n\t\t\t\tconst c = actorCtx(loopCtx);\n\n\t\t\t\tawait loopCtx.step(\"init-payment\", async () => {\n\t\t\t\t\tctx.log.info({\n\t\t\t\t\t\tmsg: \"starting payment processing\",\n\t\t\t\t\t\ttxId: c.state.id,\n\t\t\t\t\t\tamount: c.state.amount,\n\t\t\t\t\t\tshouldFail: c.state.shouldFail,\n\t\t\t\t\t});\n\t\t\t\t\tc.broadcast(\"transactionStarted\", c.state);\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.rollbackCheckpoint(\"payment-checkpoint\");\n\n\t\t\t\t// Step 1: Reserve inventory\n\t\t\t\tawait loopCtx.step({\n\t\t\t\t\tname: \"reserve-inventory\",\n\t\t\t\t\trun: async () => {\n\t\t\t\t\t\tc.state.status = \"reserving\";\n\t\t\t\t\t\tconst step = c.state.steps.find(\n\t\t\t\t\t\t\t(s) => s.name === \"reserve-inventory\"\n\t\t\t\t\t\t);\n\t\t\t\t\t\tif (step) {\n\t\t\t\t\t\t\tstep.status = \"completed\";\n\t\t\t\t\t\t\tstep.completedAt = Date.now();\n\t\t\t\t\t\t}\n\t\t\t\t\t\tc.broadcast(\"transactionUpdated\", c.state);\n\n\t\t\t\t\t\tawait new Promise((r) =>\n\t\t\t\t\t\t\tsetTimeout(r, 500 + Math.random() * 500)\n\t\t\t\t\t\t);\n\t\t\t\t\t\tctx.log.info({ msg: \"inventory reserved\", txId: c.state.id });\n\t\t\t\t\t\treturn { reserved: true };\n\t\t\t\t\t},\n\t\t\t\t\trollback: async () => {\n\t\t\t\t\t\t// Set rolling_back status on first rollback\n\t\t\t\t\t\tc.state.status = \"rolling_back\";\n\t\t\t\t\t\tconst step = c.state.steps.find(\n\t\t\t\t\t\t\t(s) => s.name === \"reserve-inventory\"\n\t\t\t\t\t\t);\n\t\t\t\t\t\tif (step) {\n\t\t\t\t\t\t\tstep.status = \"rolled_back\";\n\t\t\t\t\t\t\tstep.rolledBackAt = Date.now();\n\t\t\t\t\t\t}\n\t\t\t\t\t\tctx.log.info({ msg: \"inventory released\", txId: c.state.id });\n\t\t\t\t\t\tc.broadcast(\"transactionUpdated\", c.state);\n\t\t\t\t\t\t// Small delay so UI can show the rollback\n\t\t\t\t\t\tawait new Promise((r) => setTimeout(r, 400));\n\t\t\t\t\t},\n\t\t\t\t});\n\n\t\t\t\t// Step 2: Charge card\n\t\t\t\tawait loopCtx.step({\n\t\t\t\t\tname: \"charge-card\",\n\t\t\t\t\trun: async () => {\n\t\t\t\t\t\tc.state.status = \"charging\";\n\t\t\t\t\t\tconst step = c.state.steps.find((s) => s.name === \"charge-card\");\n\t\t\t\t\t\tif (step) {\n\t\t\t\t\t\t\tstep.status = \"completed\";\n\t\t\t\t\t\t\tstep.completedAt = Date.now();\n\t\t\t\t\t\t}\n\t\t\t\t\t\tc.broadcast(\"transactionUpdated\", c.state);\n\n\t\t\t\t\t\tawait new Promise((r) =>\n\t\t\t\t\t\t\tsetTimeout(r, 500 + Math.random() * 500)\n\t\t\t\t\t\t);\n\n\t\t\t\t\t\tif (c.state.shouldFail) {\n\t\t\t\t\t\t\tthrow new Error(\"Payment declined (simulated)\");\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tctx.log.info({ msg: \"card charged\", txId: c.state.id });\n\t\t\t\t\t\treturn { chargeId: `ch_${c.state.id}` };\n\t\t\t\t\t},\n\t\t\t\t\trollback: async () => {\n\t\t\t\t\t\tc.state.status = \"rolling_back\";\n\t\t\t\t\t\tconst step = c.state.steps.find((s) => s.name === \"charge-card\");\n\t\t\t\t\t\tif (step) {\n\t\t\t\t\t\t\tstep.status = \"rolled_back\";\n\t\t\t\t\t\t\tstep.rolledBackAt = Date.now();\n\t\t\t\t\t\t}\n\t\t\t\t\t\tctx.log.info({ msg: \"charge refunded\", txId: c.state.id });\n\t\t\t\t\t\tc.broadcast(\"transactionUpdated\", c.state);\n\t\t\t\t\t\t// Small delay so UI can show the rollback\n\t\t\t\t\t\tawait new Promise((r) => setTimeout(r, 400));\n\t\t\t\t\t},\n\t\t\t\t});\n\n\t\t\t\t// Step 3: Complete order\n\t\t\t\tawait loopCtx.step(\"complete-order\", async () => {\n\t\t\t\t\t\tc.state.status = \"completing\";\n\t\t\t\t\t\tconst step = c.state.steps.find((s) => s.name === \"complete-order\");\n\t\t\t\t\t\tif (step) step.status = \"completed\";\n\t\t\t\t\t\tc.broadcast(\"transactionUpdated\", c.state);\n\n\t\t\t\t\t\tawait new Promise((r) =>\n\t\t\t\t\t\t\tsetTimeout(r, 300 + Math.random() * 300)\n\t\t\t\t\t\t);\n\n\t\t\t\t\t\tc.state.status = \"completed\";\n\t\t\t\t\t\tc.state.completedAt = Date.now();\n\t\t\t\t\t\tctx.log.info({ msg: \"order completed\", txId: c.state.id });\n\t\t\t\t\t\tc.broadcast(\"transactionCompleted\", c.state);\n\t\t\t\t\t});\n\n\t\t\t\treturn Loop.break(undefined);\n\t\t\t});\n\t}),\n});\n","import { actor, event, queue } from \"rivetkit\";\nimport { Loop, workflow } from \"rivetkit/workflow\";\nimport { actorCtx } from \"./_helpers.ts\";\n\nfunction delay(ms: number): Promise {\n\treturn new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nexport type WorkflowHistorySimpleState = {\n\tid: string;\n\tstatus: \"pending\" | \"running\" | \"completed\";\n\tlastStep?: string;\n\tstartedAt?: number;\n\tcompletedAt?: number;\n\toutput?: { success: boolean; processedItems: number };\n};\n\nexport const workflowHistorySimple = actor({\n\tcreateState: (c): WorkflowHistorySimpleState => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"pending\",\n\t}),\n\tactions: {\n\t\tgetState: (c): WorkflowHistorySimpleState => c.state,\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.step(\"start\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"running\";\n\t\t\tc.state.lastStep = \"start\";\n\t\t\tc.state.startedAt = Date.now();\n\t\t\treturn { initialized: true };\n\t\t});\n\n\t\tawait delay(700);\n\n\t\tawait ctx.step(\"process\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.lastStep = \"process\";\n\t\t\treturn { processed: true, items: 5 };\n\t\t});\n\n\t\tawait delay(2200);\n\n\t\tawait ctx.step(\"validate\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.lastStep = \"validate\";\n\t\t\treturn { valid: true };\n\t\t});\n\n\t\tawait delay(600);\n\n\t\tawait ctx.step(\"complete\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.lastStep = \"complete\";\n\t\t\tc.state.status = \"completed\";\n\t\t\tc.state.completedAt = Date.now();\n\t\t\tc.state.output = { success: true, processedItems: 3 };\n\t\t\treturn { success: true };\n\t\t});\n\t}),\n});\n\nconst LOOP_ITEMS = [\"A\", \"B\", \"C\"];\n\nexport type WorkflowHistoryLoopState = {\n\tid: string;\n\tstatus: \"running\" | \"completed\";\n\tprocessed: number;\n\tbatches: Array<{ index: number; item: string }>;\n\tcompletedAt?: number;\n};\n\nexport const workflowHistoryLoop = actor({\n\tcreateState: (c): WorkflowHistoryLoopState => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"running\",\n\t\tprocessed: 0,\n\t\tbatches: [],\n\t}),\n\tactions: {\n\t\tgetState: (c): WorkflowHistoryLoopState => c.state,\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.step(\"init\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"running\";\n\t\t\treturn { batchSize: LOOP_ITEMS.length };\n\t\t});\n\n\t\tawait ctx.loop({\n\t\t\tname: \"batch-loop\",\n\t\t\tstate: { index: 0 },\n\t\t\tcommitInterval: 1,\n\t\t\thistoryEvery: 1,\n\t\t\thistoryKeep: LOOP_ITEMS.length,\n\t\t\trun: async (loopCtx, loopState: { index: number }) => {\n\t\t\t\tconst item = LOOP_ITEMS[loopState.index];\n\n\t\t\t\tawait loopCtx.step(`process-${loopState.index}`, async () => {\n\t\t\t\t\tconst c = actorCtx(loopCtx);\n\t\t\t\t\tc.state.processed += 1;\n\t\t\t\t\tc.state.batches.push({ index: loopState.index, item });\n\t\t\t\t\treturn { item, status: \"done\" };\n\t\t\t\t});\n\n\t\t\t\tif (loopState.index >= LOOP_ITEMS.length - 1) {\n\t\t\t\t\treturn Loop.break({ processed: LOOP_ITEMS.length });\n\t\t\t\t}\n\n\t\t\t\treturn Loop.continue({ index: loopState.index + 1 });\n\t\t\t},\n\t\t});\n\n\t\tawait ctx.step(\"finalize\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"completed\";\n\t\t\tc.state.completedAt = Date.now();\n\t\t\treturn { allProcessed: true };\n\t\t});\n\t}),\n});\n\nexport type WorkflowHistoryJoinState = {\n\tid: string;\n\tstatus: \"pending\" | \"running\" | \"completed\";\n\tresult?: {\n\t\tapi: string;\n\t\trows: number;\n\t\tcacheHit: boolean;\n\t};\n};\n\nexport const workflowHistoryJoin = actor({\n\tcreateState: (c): WorkflowHistoryJoinState => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"pending\",\n\t}),\n\tactions: {\n\t\tgetState: (c): WorkflowHistoryJoinState => c.state,\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.step(\"start\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"running\";\n\t\t\treturn { ready: true };\n\t\t});\n\n\t\tconst results = await ctx.join(\"parallel-tasks\", {\n\t\t\t\"fetch-api\": {\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tawait branchCtx.step(\"task-a\", async () => {\n\t\t\t\t\t\tawait delay(120);\n\t\t\t\t\t\treturn { fetched: true };\n\t\t\t\t\t});\n\t\t\t\t\treturn { data: \"api-response\" };\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"query-db\": {\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tawait branchCtx.step(\"task-b\", async () => {\n\t\t\t\t\t\tawait delay(200);\n\t\t\t\t\t\treturn { queried: true };\n\t\t\t\t\t});\n\t\t\t\t\treturn { rows: 42 };\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"check-cache\": {\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tawait branchCtx.step(\"task-c\", async () => {\n\t\t\t\t\t\tawait delay(60);\n\t\t\t\t\t\treturn { checked: true };\n\t\t\t\t\t});\n\t\t\t\t\treturn { hit: true };\n\t\t\t\t},\n\t\t\t},\n\t\t});\n\n\t\tawait ctx.step(\"merge-results\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"completed\";\n\t\t\tc.state.result = {\n\t\t\t\tapi: results[\"fetch-api\"].data,\n\t\t\t\trows: results[\"query-db\"].rows,\n\t\t\t\tcacheHit: results[\"check-cache\"].hit,\n\t\t\t};\n\t\t\treturn { merged: true };\n\t\t});\n\t}),\n});\n\nexport type WorkflowHistoryRaceState = {\n\tid: string;\n\tstatus: \"running\" | \"completed\";\n\twinner?: string;\n\tresult?: string;\n};\n\nexport const workflowHistoryRace = actor({\n\tcreateState: (c): WorkflowHistoryRaceState => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"running\",\n\t}),\n\tactions: {\n\t\tgetState: (c): WorkflowHistoryRaceState => c.state,\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.step(\"begin\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"running\";\n\t\t\treturn { started: true };\n\t\t});\n\n\t\tconst { winner, value } = await ctx.race<{\n\t\t\tprovider: string;\n\t\t\tlatency: number;\n\t\t}>(\"race-providers\", [\n\t\t\t{\n\t\t\t\tname: \"provider-fast\",\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tawait branchCtx.sleep(\"provider-fast-step\", 50);\n\t\t\t\t\treturn { provider: \"cdn-edge\", latency: 12 };\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"provider-slow\",\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tawait branchCtx.sleep(\"provider-slow-step\", 200);\n\t\t\t\t\treturn { provider: \"origin\", latency: 120 };\n\t\t\t\t},\n\t\t\t},\n\t\t]);\n\n\t\tawait ctx.step(\"use-result\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"completed\";\n\t\t\tc.state.winner = winner;\n\t\t\tc.state.result = value.provider;\n\t\t\treturn { used: value.provider };\n\t\t});\n\t}),\n});\n\nexport type WorkflowHistoryFullState = {\n\tid: string;\n\tstatus: \"pending\" | \"running\" | \"waiting\" | \"completed\" | \"failed\";\n\tseededMessages: boolean;\n\tlastStep?: string;\n\tstartedAt?: number;\n\tcompletedAt?: number;\n};\n\nconst QUEUE_ORDER_CREATED = \"order:created\";\nconst QUEUE_ORDER_UPDATED = \"order:updated\";\nconst QUEUE_ORDER_ITEM = \"order:item\";\nconst QUEUE_ORDER_ARTIFACT = \"order:artifact\";\nconst QUEUE_ORDER_READY = \"order:ready\";\nconst QUEUE_ORDER_OPTIONAL = \"order:optional\";\n\ntype OrderCreatedMessage = { id: string };\ntype OrderUpdatedMessage = { id: string; status: string };\ntype OrderItemMessage = { sku: string; qty: number };\ntype OrderArtifactMessage = { artifactId: string };\ntype OrderReadyMessage = { batch: number };\ntype OrderOptionalMessage = { note?: string };\n\ntype MessageSeed =\n\t| { name: typeof QUEUE_ORDER_CREATED; payload: OrderCreatedMessage }\n\t| { name: typeof QUEUE_ORDER_UPDATED; payload: OrderUpdatedMessage }\n\t| { name: typeof QUEUE_ORDER_ITEM; payload: OrderItemMessage }\n\t| { name: typeof QUEUE_ORDER_ARTIFACT; payload: OrderArtifactMessage }\n\t| { name: typeof QUEUE_ORDER_READY; payload: OrderReadyMessage };\n\nconst FULL_WORKFLOW_MESSAGE_SEEDS: MessageSeed[] = [\n\t{ name: QUEUE_ORDER_CREATED, payload: { id: \"order-1\" } },\n\t{ name: QUEUE_ORDER_UPDATED, payload: { id: \"order-1\", status: \"paid\" } },\n\t{ name: QUEUE_ORDER_ITEM, payload: { sku: \"sku-0\", qty: 1 } },\n\t{ name: QUEUE_ORDER_ITEM, payload: { sku: \"sku-4\", qty: 1 } },\n\t{ name: QUEUE_ORDER_ARTIFACT, payload: { artifactId: \"artifact-0\" } },\n\t{ name: QUEUE_ORDER_ARTIFACT, payload: { artifactId: \"artifact-1\" } },\n\t{ name: QUEUE_ORDER_ARTIFACT, payload: { artifactId: \"artifact-2\" } },\n\t{ name: QUEUE_ORDER_READY, payload: { batch: 3 } },\n\t{ name: QUEUE_ORDER_READY, payload: { batch: 0 } },\n\t{ name: QUEUE_ORDER_READY, payload: { batch: 2 } },\n];\n\nconst FULL_WORKFLOW_ITEMS = [\n\t{ id: \"item-1\", basePrice: 100, tax: 8 },\n\t{ id: \"item-2\", basePrice: 115, tax: 9 },\n\t{ id: \"item-3\", basePrice: 130, tax: 10 },\n\t{ id: \"item-4\", basePrice: 145, tax: 12 },\n];\n\nexport const workflowHistoryFull = actor({\n\tcreateState: (c): WorkflowHistoryFullState => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"pending\",\n\t\tseededMessages: false,\n\t}),\n\tqueues: {\n\t\t[QUEUE_ORDER_CREATED]: queue(),\n\t\t[QUEUE_ORDER_UPDATED]: queue(),\n\t\t[QUEUE_ORDER_ITEM]: queue(),\n\t\t[QUEUE_ORDER_ARTIFACT]: queue(),\n\t\t[QUEUE_ORDER_READY]: queue(),\n\t\t[QUEUE_ORDER_OPTIONAL]: queue(),\n\t},\n\tactions: {\n\t\tgetState: (c): WorkflowHistoryFullState => c.state,\n\t\tseedMessages: async (c) => {\n\t\t\tif (c.state.seededMessages) return;\n\t\t\tfor (const seed of FULL_WORKFLOW_MESSAGE_SEEDS) {\n\t\t\t\tawait c.queue.send(seed.name, seed.payload);\n\t\t\t}\n\t\t\tc.state.seededMessages = true;\n\t\t},\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.step(\"bootstrap\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"running\";\n\t\t\tc.state.lastStep = \"bootstrap\";\n\t\t\tc.state.startedAt = Date.now();\n\t\t\treturn {\n\t\t\t\trequestId: `req-${c.state.id}`,\n\t\t\t\tstartedAt: Date.now(),\n\t\t\t};\n\t\t});\n\n\t\tawait ctx.step(\"validate-input\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.lastStep = \"validate-input\";\n\t\t\treturn true;\n\t\t});\n\n\t\tawait ctx.rollbackCheckpoint(\"checkpoint-after-validation\");\n\n\t\tawait ctx.step(\"load-user-profile\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.lastStep = \"load-user-profile\";\n\t\t\treturn {\n\t\t\t\tid: \"user-123\",\n\t\t\t\ttier: \"standard\",\n\t\t\t\tflags: [\"email-verified\", \"promo-eligible\"],\n\t\t\t};\n\t\t});\n\n\t\tawait ctx.step(\"compute-discount\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.lastStep = \"compute-discount\";\n\t\t\treturn { percent: 5, reason: \"tier-discount\" };\n\t\t});\n\n\t\tawait ctx.step(\"ephemeral-cache-check\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.lastStep = \"ephemeral-cache-check\";\n\t\t\treturn { cacheHit: false, tier: \"standard\" };\n\t\t});\n\n\t\tawait ctx.rollbackCheckpoint(\"checkpoint-before-reserve\");\n\n\t\tawait ctx.loop({\n\t\t\tname: \"process-items-loop\",\n\t\t\tstate: { index: 0 },\n\t\t\tcommitInterval: 1,\n\t\t\thistoryEvery: 1,\n\t\t\thistoryKeep: 2,\n\t\t\trun: async (loopCtx, loopState: { index: number }) => {\n\t\t\t\tconst item = FULL_WORKFLOW_ITEMS[loopState.index];\n\t\t\t\tif (!item) {\n\t\t\t\t\treturn Loop.break({ count: FULL_WORKFLOW_ITEMS.length });\n\t\t\t\t}\n\n\t\t\t\tawait loopCtx.step(`fetch-item-${loopState.index}`, async () => {\n\t\t\t\t\treturn { itemId: item.id, basePrice: item.basePrice };\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.step(`compute-tax-${loopState.index}`, async () => {\n\t\t\t\t\treturn item.tax;\n\t\t\t\t});\n\n\t\t\t\tawait loopCtx.step(\n\t\t\t\t\t`reserve-inventory-${loopState.index}`,\n\t\t\t\t\tasync () => ({\n\t\t\t\t\t\treservationId: `res-${loopState.index}`,\n\t\t\t\t\t\titemId: item.id,\n\t\t\t\t\t}),\n\t\t\t\t);\n\n\t\t\t\tif (loopState.index >= FULL_WORKFLOW_ITEMS.length - 1) {\n\t\t\t\t\treturn Loop.break({\n\t\t\t\t\t\tcount: FULL_WORKFLOW_ITEMS.length,\n\t\t\t\t\t\ttotal: 504,\n\t\t\t\t\t});\n\t\t\t\t}\n\n\t\t\t\treturn Loop.continue({ index: loopState.index + 1 });\n\t\t\t},\n\t\t});\n\n\t\tawait ctx.sleep(\"short-cooldown\", 40);\n\t\tawait ctx.sleep(\"cooldown-sleep\", 60);\n\t\tawait ctx.sleep(\"wait-until-deadline\", 45);\n\n\t\tawait ctx.step(\"compute-deadlines\", async () => {\n\t\t\tconst readyBy = Date.now() + 800;\n\t\t\tconst readyBatchBy = Date.now() + 1100;\n\t\t\treturn { readyBy, readyBatchBy };\n\t\t});\n\n\t\tawait ctx.queue.next(\"listen-order-created\", {\n\t\t\tnames: [QUEUE_ORDER_CREATED],\n\t\t});\n\t\tawait ctx.queue.nextBatch(\"listen-order-updated-timeout\", {\n\t\t\tnames: [QUEUE_ORDER_UPDATED],\n\t\t\ttimeout: 250,\n\t\t});\n\t\tawait ctx.queue.nextBatch(\"listen-batch-two\", {\n\t\t\tnames: [QUEUE_ORDER_ITEM],\n\t\t\tcount: 2,\n\t\t});\n\t\tawait ctx.queue.nextBatch(\"listen-artifacts-timeout\", {\n\t\t\tnames: [QUEUE_ORDER_ARTIFACT],\n\t\t\tcount: 3,\n\t\t\ttimeout: 300,\n\t\t});\n\t\tawait ctx.queue.nextBatch(\"listen-optional\", {\n\t\t\tnames: [QUEUE_ORDER_OPTIONAL],\n\t\t\ttimeout: 200,\n\t\t});\n\t\tawait ctx.queue.nextBatch(\"listen-until\", {\n\t\t\tnames: [QUEUE_ORDER_READY],\n\t\t\ttimeout: 300,\n\t\t});\n\t\tawait ctx.queue.nextBatch(\"listen-batch-until\", {\n\t\t\tnames: [QUEUE_ORDER_READY],\n\t\t\tcount: 2,\n\t\t\ttimeout: 400,\n\t\t});\n\n\t\tawait ctx.join(\"join-dependencies\", {\n\t\t\tinventory: {\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tconst reserved = await branchCtx.step(\n\t\t\t\t\t\t\"inventory-audit\",\n\t\t\t\t\t\tasync () => 4,\n\t\t\t\t\t);\n\t\t\t\t\tawait branchCtx.sleep(\"join-inventory-sleep\", 35);\n\t\t\t\t\treturn {\n\t\t\t\t\t\treserved,\n\t\t\t\t\t\tchecked: 4,\n\t\t\t\t\t\tnotes: [\"inventory-ok\", \"items=4\"],\n\t\t\t\t\t};\n\t\t\t\t},\n\t\t\t},\n\t\t\tpricing: {\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tconst method = await branchCtx.step(\n\t\t\t\t\t\t\"pricing-method\",\n\t\t\t\t\t\tasync () => \"promo\",\n\t\t\t\t\t);\n\t\t\t\t\treturn {\n\t\t\t\t\t\tsubtotal: 504,\n\t\t\t\t\t\tdiscount: 25,\n\t\t\t\t\t\ttotal: 479,\n\t\t\t\t\t\tmethod,\n\t\t\t\t\t};\n\t\t\t\t},\n\t\t\t},\n\t\t\tshipping: {\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tconst zone = await branchCtx.step(\n\t\t\t\t\t\t\"shipping-zone\",\n\t\t\t\t\t\tasync () => \"us-east\",\n\t\t\t\t\t);\n\t\t\t\t\tawait branchCtx.sleep(\"join-shipping-sleep\", 35);\n\t\t\t\t\treturn { method: \"ground\", etaDays: 4, zone };\n\t\t\t\t},\n\t\t\t},\n\t\t});\n\n\t\tawait ctx.race(\"race-fulfillment\", [\n\t\t\t{\n\t\t\t\tname: \"race-fast\",\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tawait branchCtx.sleep(\"race-fast-sleep\", 70);\n\t\t\t\t\treturn { method: \"express\", cost: 18, etaDays: 1 };\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"race-slow\",\n\t\t\t\trun: async (branchCtx) => {\n\t\t\t\t\tawait branchCtx.sleep(\"race-slow-sleep\", 250);\n\t\t\t\t\treturn { method: \"ground\", cost: 8, etaDays: 4 };\n\t\t\t\t},\n\t\t\t},\n\t\t]);\n\n\t\tawait ctx.removed(\"legacy-step-placeholder\", \"step\");\n\n\t\tawait ctx.step(\"finalize\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.lastStep = \"finalize\";\n\t\t\tc.state.status = \"completed\";\n\t\t\tc.state.completedAt = Date.now();\n\t\t\treturn true;\n\t\t});\n\t}),\n});\n\nexport type WorkflowHistoryInProgressState = {\n\tid: string;\n\tstatus: \"running\" | \"completed\";\n\tprocessingDurationMs: number;\n\tprogress: number;\n\tstartedAt?: number;\n\tcompletedAt?: number;\n};\n\nexport type WorkflowHistoryInProgressInput = {\n\tprocessingDurationMs?: number;\n};\n\nexport const workflowHistoryInProgress = actor({\n\tcreateState: (\n\t\tc,\n\t\tinput?: WorkflowHistoryInProgressInput,\n\t): WorkflowHistoryInProgressState => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"running\",\n\t\tprocessingDurationMs: input?.processingDurationMs ?? 30000,\n\t\tprogress: 0,\n\t}),\n\tactions: {\n\t\tgetState: (c): WorkflowHistoryInProgressState => c.state,\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.step(\"init\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.startedAt = Date.now();\n\t\t\tc.state.progress = 10;\n\t\t\treturn { initialized: true };\n\t\t});\n\n\t\tawait ctx.step(\"fetch-data\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.progress = 25;\n\t\t\treturn { fetched: true, records: 100 };\n\t\t});\n\n\t\tawait ctx.step(\"process\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.progress = 42;\n\t\t\tawait delay(c.state.processingDurationMs);\n\t\t\tc.state.status = \"completed\";\n\t\t\tc.state.completedAt = Date.now();\n\t\t\treturn { processed: true };\n\t\t});\n\t}),\n});\n\nexport type WorkflowHistoryRetryingState = {\n\tid: string;\n\tstatus: \"running\" | \"completed\";\n\tattempts: number;\n\tlastError?: string;\n\tsucceedAfter: number;\n};\n\nconst RETRY_MAX_RETRIES = 20;\n\nexport const workflowHistoryRetrying = actor({\n\tcreateState: (c): WorkflowHistoryRetryingState => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"running\",\n\t\tattempts: 0,\n\t\tsucceedAfter: 999,\n\t}),\n\tactions: {\n\t\tgetState: (c): WorkflowHistoryRetryingState => c.state,\n\t\tallowSuccess: (c, afterAttempt?: number) => {\n\t\t\tc.state.succeedAfter = afterAttempt ?? c.state.attempts + 1;\n\t\t},\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.step(\"start\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"running\";\n\t\t\treturn { ready: true };\n\t\t});\n\n\t\tawait ctx.step({\n\t\t\tname: \"api-call\",\n\t\t\tmaxRetries: RETRY_MAX_RETRIES,\n\t\t\tretryBackoffBase: 250,\n\t\t\tretryBackoffMax: 1500,\n\t\t\trun: async () => {\n\t\t\t\tconst c = actorCtx(ctx);\n\t\t\t\tc.state.attempts += 1;\n\t\t\t\tif (c.state.attempts < c.state.succeedAfter) {\n\t\t\t\t\tconst error = new Error(\"Connection timeout after 5000ms\");\n\t\t\t\t\tc.state.lastError = error.message;\n\t\t\t\t\tthrow error;\n\t\t\t\t}\n\t\t\t\tc.state.status = \"completed\";\n\t\t\t\tc.state.lastError = undefined;\n\t\t\t\treturn { success: true };\n\t\t\t},\n\t\t});\n\t}),\n});\n\nexport type WorkflowHistoryFailedState = {\n\tid: string;\n\tstatus: \"running\" | \"failed\";\n\tattempts: number;\n\tlastError?: string;\n};\n\nconst FAILED_MAX_RETRIES = 3;\n\nexport const workflowHistoryFailed = actor({\n\tcreateState: (c): WorkflowHistoryFailedState => ({\n\t\tid: c.key[0] as string,\n\t\tstatus: \"running\",\n\t\tattempts: 0,\n\t}),\n\tactions: {\n\t\tgetState: (c): WorkflowHistoryFailedState => c.state,\n\t},\n\trun: workflow(async (ctx) => {\n\t\tawait ctx.step(\"init\", async () => {\n\t\t\tconst c = actorCtx(ctx);\n\t\t\tc.state.status = \"running\";\n\t\t\treturn { initialized: true };\n\t\t});\n\n\t\tawait ctx.step(\"validate\", async () => {\n\t\t\treturn { valid: true };\n\t\t});\n\n\t\tawait ctx.step({\n\t\t\tname: \"process\",\n\t\t\tmaxRetries: FAILED_MAX_RETRIES,\n\t\t\tretryBackoffBase: 200,\n\t\t\tretryBackoffMax: 800,\n\t\t\trun: async () => {\n\t\t\t\tconst c = actorCtx(ctx);\n\t\t\t\tc.state.attempts += 1;\n\t\t\t\tconst error = new Error(\n\t\t\t\t\t\"Database connection failed: ECONNREFUSED\",\n\t\t\t\t);\n\t\t\t\tc.state.lastError = error.message;\n\t\t\t\tthrow error;\n\t\t\t},\n\t\t});\n\t}),\n});\n","import { actor } from \"rivetkit\";\n\nexport interface InventoryInput {\n\tinitialStock: number;\n\titemName: string;\n}\n\nexport interface InventoryState {\n\titemName: string;\n\tstock: number;\n\treservations: string[]; // Track which checkouts have reserved items\n}\n\nexport interface CheckoutInput {\n\tcustomerName: string;\n}\n\nexport interface CheckoutItem {\n\titemId: string;\n\titemName: string;\n\tquantity: number;\n}\n\nexport interface CheckoutResult {\n\tsuccess: boolean;\n\tmessage: string;\n\tremainingStock?: number;\n}\n\nexport interface CheckoutState {\n\tcustomerName: string;\n\titems: CheckoutItem[];\n\tcompleted: boolean;\n}\n\n// Inventory actor manages stock for a specific item\nexport const inventory = actor({\n\t// Each item has its own inventory actor instance\n\tcreateState: (_c, input?: InventoryInput): InventoryState => ({\n\t\titemName: input?.itemName ?? \"Widget\",\n\t\tstock: input?.initialStock ?? 100,\n\t\treservations: [],\n\t}),\n\n\tactions: {\n\t\t// Check current stock\n\t\tgetStock: (c) => ({\n\t\t\titemName: c.state.itemName,\n\t\t\tstock: c.state.stock,\n\t\t}),\n\n\t\t// Reserve items for checkout (called by checkout actor)\n\t\treserveItems: (c, checkoutId: string, quantity: number) => {\n\t\t\tif (c.state.stock < quantity) {\n\t\t\t\treturn {\n\t\t\t\t\tsuccess: false,\n\t\t\t\t\tmessage: `Insufficient stock. Available: ${c.state.stock}, Requested: ${quantity}`,\n\t\t\t\t\tavailableStock: c.state.stock,\n\t\t\t\t};\n\t\t\t}\n\n\t\t\t// Reserve the items\n\t\t\tc.state.stock -= quantity;\n\t\t\tc.state.reservations.push(checkoutId);\n\n\t\t\treturn {\n\t\t\t\tsuccess: true,\n\t\t\t\tmessage: `Reserved ${quantity} items for checkout ${checkoutId}`,\n\t\t\t\tremainingStock: c.state.stock,\n\t\t\t};\n\t\t},\n\n\t\t// Release reserved items if checkout is cancelled\n\t\treleaseItems: (c, checkoutId: string, quantity: number) => {\n\t\t\tconst index = c.state.reservations.indexOf(checkoutId);\n\t\t\tif (index > -1) {\n\t\t\t\tc.state.reservations.splice(index, 1);\n\t\t\t\tc.state.stock += quantity;\n\t\t\t}\n\t\t\treturn {\n\t\t\t\tsuccess: true,\n\t\t\t\tremainingStock: c.state.stock,\n\t\t\t};\n\t\t},\n\t},\n});\n\n// Checkout actor manages the checkout process and communicates with inventory\nexport const checkout = actor({\n\tcreateState: (_c, input?: CheckoutInput): CheckoutState => ({\n\t\tcustomerName: input?.customerName ?? \"Guest\",\n\t\titems: [],\n\t\tcompleted: false,\n\t}),\n\n\tactions: {\n\t\t// Add item to checkout and reserve from inventory\n\t\taddItem: async (\n\t\t\tc,\n\t\t\titemId: string,\n\t\t\tquantity: number,\n\t\t): Promise => {\n\t\t\t// Use server-side client to communicate with inventory actor\n\t\t\t// https://rivet.dev/docs/actors/communicating-between-actors\n\t\t\tconst inventoryActor = c.client().inventory.getOrCreate([itemId]);\n\n\t\t\t// Get item details\n\t\t\tconst itemInfo = await inventoryActor.getStock();\n\n\t\t\t// Try to reserve items from inventory\n\t\t\tconst reservation = await inventoryActor.reserveItems(\n\t\t\t\tc.actorId, // Use checkout ID as reservation ID\n\t\t\t\tquantity,\n\t\t\t);\n\n\t\t\tif (!reservation.success) {\n\t\t\t\treturn {\n\t\t\t\t\tsuccess: false,\n\t\t\t\t\tmessage: reservation.message,\n\t\t\t\t};\n\t\t\t}\n\n\t\t\t// Add item to checkout\n\t\t\tc.state.items.push({\n\t\t\t\titemId,\n\t\t\t\titemName: itemInfo.itemName,\n\t\t\t\tquantity,\n\t\t\t});\n\n\t\t\treturn {\n\t\t\t\tsuccess: true,\n\t\t\t\tmessage: `Added ${quantity} ${itemInfo.itemName} to checkout`,\n\t\t\t\tremainingStock: reservation.remainingStock,\n\t\t\t};\n\t\t},\n\n\t\t// Get checkout summary\n\t\tgetSummary: (c) => ({\n\t\t\tcustomerName: c.state.customerName,\n\t\t\titems: c.state.items,\n\t\t\tcompleted: c.state.completed,\n\t\t\ttotalItems: c.state.items.reduce(\n\t\t\t\t(sum, item) => sum + item.quantity,\n\t\t\t\t0,\n\t\t\t),\n\t\t}),\n\n\t\t// Complete the checkout\n\t\tcompleteCheckout: (c) => {\n\t\t\tc.state.completed = true;\n\t\t\treturn {\n\t\t\t\tsuccess: true,\n\t\t\t\tmessage: \"Checkout completed successfully\",\n\t\t\t};\n\t\t},\n\n\t\t// Cancel checkout and release all reservations\n\t\tcancelCheckout: async (c) => {\n\t\t\t// Release all reserved items\n\t\t\tfor (const item of c.state.items) {\n\t\t\t\tconst inventoryActor = c\n\t\t\t\t\t.client()\n\t\t\t\t\t.inventory.getOrCreate([item.itemId]);\n\t\t\t\tawait inventoryActor.releaseItems(c.actorId, item.quantity);\n\t\t\t}\n\n\t\t\t// Clear the cart\n\t\t\tc.state.items = [];\n\n\t\t\treturn {\n\t\t\t\tsuccess: true,\n\t\t\t\tmessage: \"Checkout cancelled, items returned to inventory\",\n\t\t\t};\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport type { registry } from \"../../index.ts\";\n\n// The dynamic-sandbox WebSocket connect path is failing intermittently and\n// silently swapping to HTTP-only would skip event-bridge validation entirely.\n// Re-enable WebSocket coverage on every runtime; if the dynamic-sandbox\n// regression returns, root-cause it instead of restoring this gate.\nfunction isDynamicSandboxRuntime(): boolean {\n\treturn false;\n}\n\nasync function waitForConnectionOpen(connection: {\n\tconnStatus: string;\n\tonOpen(callback: () => void): () => void;\n\tonError(callback: (error: unknown) => void): () => void;\n}) {\n\tif (connection.connStatus === \"connected\") {\n\t\treturn;\n\t}\n\n\tawait new Promise((resolve, reject) => {\n\t\tconst unsubscribeOpen = connection.onOpen(() => {\n\t\t\tunsubscribeOpen();\n\t\t\tunsubscribeError();\n\t\t\tresolve();\n\t\t});\n\t\tconst unsubscribeError = connection.onError((error) => {\n\t\t\tunsubscribeOpen();\n\t\t\tunsubscribeError();\n\t\t\treject(error);\n\t\t});\n\t});\n}\n\nexport const inlineClientActor = actor({\n\tstate: { messages: [] as string[] },\n\tactions: {\n\t\t// Action that uses client to call another actor (stateless)\n\t\tcallCounterIncrement: async (c, amount: number) => {\n\t\t\tconst client = c.client();\n\t\t\tconst result = await client.counter\n\t\t\t\t.getOrCreate([\"inline-test\"])\n\t\t\t\t.increment(amount);\n\t\t\tc.state.messages.push(\n\t\t\t\t`Called counter.increment(${amount}), result: ${result}`,\n\t\t\t);\n\t\t\treturn result;\n\t\t},\n\n\t\t// Action that uses client to get counter state (stateless)\n\t\tgetCounterState: async (c) => {\n\t\t\tconst client = c.client();\n\t\t\tconst count = await client.counter\n\t\t\t\t.getOrCreate([\"inline-test\"])\n\t\t\t\t.getCount();\n\t\t\tc.state.messages.push(`Got counter state: ${count}`);\n\t\t\treturn count;\n\t\t},\n\n\t\t// Action that uses client with .connect() for stateful communication\n\t\tconnectToCounterAndIncrement: async (c, amount: number) => {\n\t\t\tconst client = c.client();\n\t\t\tconst handle = client.counter.getOrCreate([\"inline-test-stateful\"]);\n\n\t\t\tif (isDynamicSandboxRuntime()) {\n\t\t\t\tconst events: number[] = [];\n\t\t\t\tconst result1 = await handle.increment(amount);\n\t\t\t\tevents.push(result1);\n\t\t\t\tconst result2 = await handle.increment(amount * 2);\n\t\t\t\tevents.push(result2);\n\n\t\t\t\tc.state.messages.push(\n\t\t\t\t\t`Connected to counter, incremented by ${amount} and ${amount * 2}, results: ${result1}, ${result2}, events: ${JSON.stringify(events)}`,\n\t\t\t\t);\n\n\t\t\t\treturn { result1, result2, events };\n\t\t\t}\n\n\t\t\tawait handle.getCount();\n\t\t\tconst connection = handle.connect();\n\t\t\tawait waitForConnectionOpen(connection);\n\n\t\t\t// Set up event listener\n\t\t\tconst events: number[] = [];\n\t\t\tconnection.on(\"newCount\", (count: number) => {\n\t\t\t\tevents.push(count);\n\t\t\t});\n\n\t\t\t// Perform increments\n\t\t\tconst result1 = await connection.increment(amount);\n\t\t\tconst result2 = await connection.increment(amount * 2);\n\n\t\t\tawait connection.dispose();\n\n\t\t\tc.state.messages.push(\n\t\t\t\t`Connected to counter, incremented by ${amount} and ${amount * 2}, results: ${result1}, ${result2}, events: ${JSON.stringify(events)}`,\n\t\t\t);\n\n\t\t\treturn { result1, result2, events };\n\t\t},\n\n\t\t// Get all messages from this actor's state\n\t\tgetMessages: (c) => {\n\t\t\treturn c.state.messages;\n\t\t},\n\n\t\t// Clear messages\n\t\tclearMessages: (c) => {\n\t\t\tc.state.messages = [];\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\n\nexport const testCounter = actor({\n\tstate: { count: 0 },\n\tactions: {\n\t\tincrement: (c, amount: number = 1) => {\n\t\t\tc.state.count += amount;\n\t\t\treturn c.state.count;\n\t\t},\n\t\tgetCount: (c) => {\n\t\t\treturn c.state.count;\n\t\t},\n\t\treset: (c) => {\n\t\t\tc.state.count = 0;\n\t\t\treturn c.state.count;\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nexport const testCounterSqlite = actor({\n\tdb: db({\n\t\tonMigrate: async (db) => {\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS counter (\n\t\t\t\t\tid INTEGER PRIMARY KEY CHECK (id = 1),\n\t\t\t\t\tvalue INTEGER NOT NULL DEFAULT 0\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait db.execute(\n\t\t\t\t\"INSERT OR IGNORE INTO counter (id, value) VALUES (1, 0)\",\n\t\t\t);\n\t\t},\n\t}),\n\tactions: {\n\t\tincrement: async (c, amount: number = 1) => {\n\t\t\tawait c.db.execute(\n\t\t\t\t\"UPDATE counter SET value = value + ? WHERE id = 1\",\n\t\t\t\tamount,\n\t\t\t);\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT value FROM counter WHERE id = 1\",\n\t\t\t);\n\t\t\treturn (rows[0] as { value: number }).value;\n\t\t},\n\t\tgetCount: async (c) => {\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT value FROM counter WHERE id = 1\",\n\t\t\t);\n\t\t\treturn (rows[0] as { value: number }).value;\n\t\t},\n\t\treset: async (c) => {\n\t\t\tawait c.db.execute(\"UPDATE counter SET value = 0 WHERE id = 1\");\n\t\t\treturn 0;\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nexport const testSqliteLoad = actor({\n\tdb: db({\n\t\tonMigrate: async (db) => {\n\t\t\t// Migration 1: schema version tracking\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS schema_version (\n\t\t\t\t\tid INTEGER PRIMARY KEY CHECK (id = 1),\n\t\t\t\t\tversion INTEGER NOT NULL DEFAULT 50\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait db.execute(\n\t\t\t\t\"INSERT OR IGNORE INTO schema_version (id, version) VALUES (1, 50)\",\n\t\t\t);\n\n\t\t\t// Migration 2\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS users (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tname TEXT NOT NULL,\n\t\t\t\t\temail TEXT,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 3\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS products (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tname TEXT NOT NULL,\n\t\t\t\t\tprice REAL NOT NULL DEFAULT 0,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 4\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS orders (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tuser_id INTEGER NOT NULL,\n\t\t\t\t\ttotal REAL NOT NULL DEFAULT 0,\n\t\t\t\t\tstatus TEXT NOT NULL DEFAULT 'pending',\n\t\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 5\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS order_items (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\torder_id INTEGER NOT NULL,\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\tquantity INTEGER NOT NULL DEFAULT 1,\n\t\t\t\t\tprice REAL NOT NULL,\n\t\t\t\t\tFOREIGN KEY (order_id) REFERENCES orders(id),\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 6\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS categories (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tname TEXT NOT NULL UNIQUE,\n\t\t\t\t\tdescription TEXT\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 7\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS product_categories (\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\tcategory_id INTEGER NOT NULL,\n\t\t\t\t\tPRIMARY KEY (product_id, category_id),\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id),\n\t\t\t\t\tFOREIGN KEY (category_id) REFERENCES categories(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 8\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS reviews (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tuser_id INTEGER NOT NULL,\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\trating INTEGER NOT NULL CHECK (rating BETWEEN 1 AND 5),\n\t\t\t\t\tcomment TEXT,\n\t\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id),\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 9\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS addresses (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tuser_id INTEGER NOT NULL,\n\t\t\t\t\tstreet TEXT NOT NULL,\n\t\t\t\t\tcity TEXT NOT NULL,\n\t\t\t\t\tstate TEXT,\n\t\t\t\t\tzip TEXT,\n\t\t\t\t\tcountry TEXT NOT NULL DEFAULT 'US',\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 10\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS payments (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\torder_id INTEGER NOT NULL,\n\t\t\t\t\tamount REAL NOT NULL,\n\t\t\t\t\tmethod TEXT NOT NULL DEFAULT 'card',\n\t\t\t\t\tstatus TEXT NOT NULL DEFAULT 'pending',\n\t\t\t\t\tprocessed_at INTEGER,\n\t\t\t\t\tFOREIGN KEY (order_id) REFERENCES orders(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 11\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS inventory (\n\t\t\t\t\tproduct_id INTEGER PRIMARY KEY,\n\t\t\t\t\tquantity INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\treserved INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\tlast_restocked_at INTEGER,\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 12\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS coupons (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tcode TEXT NOT NULL UNIQUE,\n\t\t\t\t\tdiscount_percent REAL NOT NULL,\n\t\t\t\t\tmax_uses INTEGER,\n\t\t\t\t\tused_count INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\texpires_at INTEGER\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 13\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS shipping (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\torder_id INTEGER NOT NULL,\n\t\t\t\t\taddress_id INTEGER NOT NULL,\n\t\t\t\t\tcarrier TEXT,\n\t\t\t\t\ttracking_number TEXT,\n\t\t\t\t\tshipped_at INTEGER,\n\t\t\t\t\tdelivered_at INTEGER,\n\t\t\t\t\tFOREIGN KEY (order_id) REFERENCES orders(id),\n\t\t\t\t\tFOREIGN KEY (address_id) REFERENCES addresses(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 14\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS tags (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tname TEXT NOT NULL UNIQUE\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 15\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS product_tags (\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\ttag_id INTEGER NOT NULL,\n\t\t\t\t\tPRIMARY KEY (product_id, tag_id),\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id),\n\t\t\t\t\tFOREIGN KEY (tag_id) REFERENCES tags(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 16\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS wishlists (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tuser_id INTEGER NOT NULL,\n\t\t\t\t\tname TEXT NOT NULL DEFAULT 'Default',\n\t\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 17\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS wishlist_items (\n\t\t\t\t\twishlist_id INTEGER NOT NULL,\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\tadded_at INTEGER NOT NULL,\n\t\t\t\t\tPRIMARY KEY (wishlist_id, product_id),\n\t\t\t\t\tFOREIGN KEY (wishlist_id) REFERENCES wishlists(id),\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 18\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS notifications (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tuser_id INTEGER NOT NULL,\n\t\t\t\t\ttype TEXT NOT NULL,\n\t\t\t\t\tmessage TEXT NOT NULL,\n\t\t\t\t\tread INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 19\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS audit_log (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tentity_type TEXT NOT NULL,\n\t\t\t\t\tentity_id INTEGER NOT NULL,\n\t\t\t\t\taction TEXT NOT NULL,\n\t\t\t\t\tdetails TEXT,\n\t\t\t\t\tperformed_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 20\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS sessions (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tuser_id INTEGER NOT NULL,\n\t\t\t\t\ttoken TEXT NOT NULL UNIQUE,\n\t\t\t\t\texpires_at INTEGER NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 21\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_users_email ON users(email)\",\n\t\t\t);\n\n\t\t\t// Migration 22\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_orders_user ON orders(user_id)\",\n\t\t\t);\n\n\t\t\t// Migration 23\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_orders_status ON orders(status)\",\n\t\t\t);\n\n\t\t\t// Migration 24\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_reviews_product ON reviews(product_id)\",\n\t\t\t);\n\n\t\t\t// Migration 25\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_reviews_user ON reviews(user_id)\",\n\t\t\t);\n\n\t\t\t// Migration 26\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_notifications_user ON notifications(user_id)\",\n\t\t\t);\n\n\t\t\t// Migration 27\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_audit_entity ON audit_log(entity_type, entity_id)\",\n\t\t\t);\n\n\t\t\t// Migration 28\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id)\",\n\t\t\t);\n\n\t\t\t// Migration 29\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_sessions_token ON sessions(token)\",\n\t\t\t);\n\n\t\t\t// Migration 30\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_inventory_quantity ON inventory(quantity)\",\n\t\t\t);\n\n\t\t\t// Migration 31\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS returns (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\torder_id INTEGER NOT NULL,\n\t\t\t\t\treason TEXT NOT NULL,\n\t\t\t\t\tstatus TEXT NOT NULL DEFAULT 'requested',\n\t\t\t\t\trequested_at INTEGER NOT NULL,\n\t\t\t\t\tprocessed_at INTEGER,\n\t\t\t\t\tFOREIGN KEY (order_id) REFERENCES orders(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 32\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS return_items (\n\t\t\t\t\treturn_id INTEGER NOT NULL,\n\t\t\t\t\torder_item_id INTEGER NOT NULL,\n\t\t\t\t\tquantity INTEGER NOT NULL,\n\t\t\t\t\tPRIMARY KEY (return_id, order_item_id),\n\t\t\t\t\tFOREIGN KEY (return_id) REFERENCES returns(id),\n\t\t\t\t\tFOREIGN KEY (order_item_id) REFERENCES order_items(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 33\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS suppliers (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tname TEXT NOT NULL,\n\t\t\t\t\tcontact_email TEXT,\n\t\t\t\t\tcountry TEXT\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 34\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS product_suppliers (\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\tsupplier_id INTEGER NOT NULL,\n\t\t\t\t\tcost REAL NOT NULL,\n\t\t\t\t\tlead_time_days INTEGER,\n\t\t\t\t\tPRIMARY KEY (product_id, supplier_id),\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id),\n\t\t\t\t\tFOREIGN KEY (supplier_id) REFERENCES suppliers(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 35\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS price_history (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\told_price REAL NOT NULL,\n\t\t\t\t\tnew_price REAL NOT NULL,\n\t\t\t\t\tchanged_at INTEGER NOT NULL,\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 36\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS user_preferences (\n\t\t\t\t\tuser_id INTEGER PRIMARY KEY,\n\t\t\t\t\ttheme TEXT NOT NULL DEFAULT 'dark',\n\t\t\t\t\tlanguage TEXT NOT NULL DEFAULT 'en',\n\t\t\t\t\tnotifications_enabled INTEGER NOT NULL DEFAULT 1,\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 37\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS cart (\n\t\t\t\t\tuser_id INTEGER NOT NULL,\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\tquantity INTEGER NOT NULL DEFAULT 1,\n\t\t\t\t\tadded_at INTEGER NOT NULL,\n\t\t\t\t\tPRIMARY KEY (user_id, product_id),\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id),\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 38\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS saved_searches (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tuser_id INTEGER NOT NULL,\n\t\t\t\t\tquery TEXT NOT NULL,\n\t\t\t\t\tfilters TEXT,\n\t\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES users(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 39\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS product_images (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\turl TEXT NOT NULL,\n\t\t\t\t\talt_text TEXT,\n\t\t\t\t\tsort_order INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 40\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS discounts (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tproduct_id INTEGER NOT NULL,\n\t\t\t\t\tdiscount_percent REAL NOT NULL,\n\t\t\t\t\tstarts_at INTEGER NOT NULL,\n\t\t\t\t\tends_at INTEGER,\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES products(id)\n\t\t\t\t)\n\t\t\t`);\n\n\t\t\t// Migration 41\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_order_items_order ON order_items(order_id)\",\n\t\t\t);\n\n\t\t\t// Migration 42\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_order_items_product ON order_items(product_id)\",\n\t\t\t);\n\n\t\t\t// Migration 43\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_payments_order ON payments(order_id)\",\n\t\t\t);\n\n\t\t\t// Migration 44\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_shipping_order ON shipping(order_id)\",\n\t\t\t);\n\n\t\t\t// Migration 45\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_returns_order ON returns(order_id)\",\n\t\t\t);\n\n\t\t\t// Migration 46\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_price_history_product ON price_history(product_id)\",\n\t\t\t);\n\n\t\t\t// Migration 47\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_product_images_product ON product_images(product_id)\",\n\t\t\t);\n\n\t\t\t// Migration 48\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_discounts_product ON discounts(product_id)\",\n\t\t\t);\n\n\t\t\t// Migration 49\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_cart_user ON cart(user_id)\",\n\t\t\t);\n\n\t\t\t// Migration 50\n\t\t\tawait db.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_saved_searches_user ON saved_searches(user_id)\",\n\t\t\t);\n\n\t\t\t// Migration 51: counter for benchmarking\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS counter (\n\t\t\t\t\tid INTEGER PRIMARY KEY CHECK (id = 1),\n\t\t\t\t\tvalue INTEGER NOT NULL DEFAULT 0\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait db.execute(\n\t\t\t\t\"INSERT OR IGNORE INTO counter (id, value) VALUES (1, 0)\",\n\t\t\t);\n\t\t},\n\t}),\n\tactions: {\n\t\tincrement: async (c, amount: number = 1) => {\n\t\t\tawait c.db.execute(\n\t\t\t\t\"UPDATE counter SET value = value + ? WHERE id = 1\",\n\t\t\t\tamount,\n\t\t\t);\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT value FROM counter WHERE id = 1\",\n\t\t\t);\n\t\t\treturn (rows[0] as { value: number }).value;\n\t\t},\n\t\tgetCount: async (c) => {\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT value FROM counter WHERE id = 1\",\n\t\t\t);\n\t\t\treturn (rows[0] as { value: number }).value;\n\t\t},\n\t\treset: async (c) => {\n\t\t\tawait c.db.execute(\"UPDATE counter SET value = 0 WHERE id = 1\");\n\t\t\treturn 0;\n\t\t},\n\t\trunLoadTest: async (c) => {\n\t\t\tconst now = Date.now();\n\t\t\tconst results: string[] = [];\n\n\t\t\t// Query 1: Insert a user\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO users (name, email, created_at) VALUES (?, ?, ?)\",\n\t\t\t\t\"Load Test User\",\n\t\t\t\t`load-${now}@test.com`,\n\t\t\t\tnow,\n\t\t\t);\n\t\t\tresults.push(\"inserted user\");\n\n\t\t\t// Query 2: Get the user back\n\t\t\tconst users = await c.db.execute(\n\t\t\t\t\"SELECT * FROM users WHERE email = ?\",\n\t\t\t\t`load-${now}@test.com`,\n\t\t\t);\n\t\t\tresults.push(`fetched user: ${(users[0] as { name: string }).name}`);\n\t\t\tconst userId = (users[0] as { id: number }).id;\n\n\t\t\t// Query 3: Insert a product\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO products (name, price, created_at) VALUES (?, ?, ?)\",\n\t\t\t\t\"Test Widget\",\n\t\t\t\t29.99,\n\t\t\t\tnow,\n\t\t\t);\n\t\t\tresults.push(\"inserted product\");\n\n\t\t\t// Query 4: Get products\n\t\t\tconst products = await c.db.execute(\"SELECT * FROM products LIMIT 10\");\n\t\t\tresults.push(`fetched ${(products as unknown[]).length} products`);\n\t\t\tconst productId = (products[0] as { id: number }).id;\n\n\t\t\t// Query 5: Insert a category\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT OR IGNORE INTO categories (name, description) VALUES (?, ?)\",\n\t\t\t\t`test-cat-${now}`,\n\t\t\t\t\"A test category\",\n\t\t\t);\n\t\t\tresults.push(\"inserted category\");\n\n\t\t\t// Query 6: Get categories\n\t\t\tconst categories = await c.db.execute(\"SELECT * FROM categories\");\n\t\t\tresults.push(`fetched ${(categories as unknown[]).length} categories`);\n\n\t\t\t// Query 7: Insert an order\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO orders (user_id, total, status, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\tuserId,\n\t\t\t\t29.99,\n\t\t\t\t\"pending\",\n\t\t\t\tnow,\n\t\t\t);\n\t\t\tresults.push(\"inserted order\");\n\n\t\t\t// Query 8: Get orders for user\n\t\t\tconst orders = await c.db.execute(\n\t\t\t\t\"SELECT * FROM orders WHERE user_id = ?\",\n\t\t\t\tuserId,\n\t\t\t);\n\t\t\tresults.push(`fetched ${(orders as unknown[]).length} orders for user`);\n\t\t\tconst orderId = (orders[0] as { id: number }).id;\n\n\t\t\t// Query 9: Insert order item\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO order_items (order_id, product_id, quantity, price) VALUES (?, ?, ?, ?)\",\n\t\t\t\torderId,\n\t\t\t\tproductId,\n\t\t\t\t2,\n\t\t\t\t29.99,\n\t\t\t);\n\t\t\tresults.push(\"inserted order item\");\n\n\t\t\t// Query 10: Insert inventory\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT OR REPLACE INTO inventory (product_id, quantity, reserved, last_restocked_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\tproductId,\n\t\t\t\t100,\n\t\t\t\t2,\n\t\t\t\tnow,\n\t\t\t);\n\t\t\tresults.push(\"inserted inventory\");\n\n\t\t\t// Query 11: Insert a review\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO reviews (user_id, product_id, rating, comment, created_at) VALUES (?, ?, ?, ?, ?)\",\n\t\t\t\tuserId,\n\t\t\t\tproductId,\n\t\t\t\t5,\n\t\t\t\t\"Great product!\",\n\t\t\t\tnow,\n\t\t\t);\n\t\t\tresults.push(\"inserted review\");\n\n\t\t\t// Query 12: Get reviews with join\n\t\t\tconst reviews = await c.db.execute(\n\t\t\t\t\"SELECT r.*, u.name as reviewer FROM reviews r JOIN users u ON r.user_id = u.id WHERE r.product_id = ?\",\n\t\t\t\tproductId,\n\t\t\t);\n\t\t\tresults.push(`fetched ${(reviews as unknown[]).length} reviews`);\n\n\t\t\t// Query 13: Insert notification\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO notifications (user_id, type, message, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\tuserId,\n\t\t\t\t\"order\",\n\t\t\t\t\"Your order has been placed\",\n\t\t\t\tnow,\n\t\t\t);\n\t\t\tresults.push(\"inserted notification\");\n\n\t\t\t// Query 14: Insert audit log entry\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO audit_log (entity_type, entity_id, action, details, performed_at) VALUES (?, ?, ?, ?, ?)\",\n\t\t\t\t\"order\",\n\t\t\t\torderId,\n\t\t\t\t\"created\",\n\t\t\t\t`Order created by user ${userId}`,\n\t\t\t\tnow,\n\t\t\t);\n\t\t\tresults.push(\"inserted audit log\");\n\n\t\t\t// Query 15: Aggregate query on orders\n\t\t\tconst orderStats = await c.db.execute(\n\t\t\t\t\"SELECT status, COUNT(*) as count, SUM(total) as total_value FROM orders GROUP BY status\",\n\t\t\t);\n\t\t\tresults.push(\n\t\t\t\t`order stats: ${(orderStats as unknown[]).length} statuses`,\n\t\t\t);\n\n\t\t\t// Query 16: Insert address\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO addresses (user_id, street, city, state, zip, country) VALUES (?, ?, ?, ?, ?, ?)\",\n\t\t\t\tuserId,\n\t\t\t\t\"123 Test St\",\n\t\t\t\t\"Testville\",\n\t\t\t\t\"CA\",\n\t\t\t\t\"90210\",\n\t\t\t\t\"US\",\n\t\t\t);\n\t\t\tresults.push(\"inserted address\");\n\n\t\t\t// Query 17: Complex join query\n\t\t\tconst orderDetails = await c.db.execute(`\n\t\t\t\tSELECT o.id, o.status, o.total, u.name as customer, COUNT(oi.id) as item_count\n\t\t\t\tFROM orders o\n\t\t\t\tJOIN users u ON o.user_id = u.id\n\t\t\t\tLEFT JOIN order_items oi ON oi.order_id = o.id\n\t\t\t\tGROUP BY o.id\n\t\t\t\tLIMIT 10\n\t\t\t`);\n\t\t\tresults.push(\n\t\t\t\t`fetched ${(orderDetails as unknown[]).length} order details`,\n\t\t\t);\n\n\t\t\t// Query 18: Update order status\n\t\t\tawait c.db.execute(\n\t\t\t\t\"UPDATE orders SET status = ? WHERE id = ?\",\n\t\t\t\t\"completed\",\n\t\t\t\torderId,\n\t\t\t);\n\t\t\tresults.push(\"updated order status\");\n\n\t\t\t// Query 19: Get schema version\n\t\t\tconst version = await c.db.execute(\n\t\t\t\t\"SELECT version FROM schema_version WHERE id = 1\",\n\t\t\t);\n\t\t\tresults.push(\n\t\t\t\t`schema version: ${(version[0] as { version: number }).version}`,\n\t\t\t);\n\n\t\t\t// Query 20: Count all tables\n\t\t\tconst tableCounts = await c.db.execute(`\n\t\t\t\tSELECT 'users' as tbl, COUNT(*) as cnt FROM users\n\t\t\t\tUNION ALL SELECT 'products', COUNT(*) FROM products\n\t\t\t\tUNION ALL SELECT 'orders', COUNT(*) FROM orders\n\t\t\t\tUNION ALL SELECT 'reviews', COUNT(*) FROM reviews\n\t\t\t\tUNION ALL SELECT 'categories', COUNT(*) FROM categories\n\t\t\t`);\n\t\t\tresults.push(\n\t\t\t\t`table counts: ${(tableCounts as unknown[]).length} tables checked`,\n\t\t\t);\n\n\t\t\treturn {\n\t\t\t\tqueriesRun: 20,\n\t\t\t\tresults,\n\t\t\t};\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nconst CHAT_LOG_CHUNK_BYTES = 4 * 1024;\nconst CHAT_LOG_INSERT_BATCH_SIZE = 50;\n\nfunction buildChatLogMessage(seq: number, targetBytes: number): string {\n\tconst prefix = `message-${seq}: `;\n\treturn prefix + \"x\".repeat(Math.max(0, targetBytes - prefix.length));\n}\n\nasync function seedChatLog(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\ttargetBytes: number,\n) {\n\tconst threadId = `chat-${crypto.randomUUID()}`;\n\tconst createdAtBase = Date.now();\n\tlet remainingBytes = targetBytes;\n\tlet rows = 0;\n\n\tawait database.execute(\"BEGIN\");\n\ttry {\n\t\twhile (remainingBytes > 0) {\n\t\t\tconst placeholders: string[] = [];\n\t\t\tconst args: unknown[] = [];\n\n\t\t\tfor (\n\t\t\t\tlet batchIndex = 0;\n\t\t\t\tbatchIndex < CHAT_LOG_INSERT_BATCH_SIZE && remainingBytes > 0;\n\t\t\t\tbatchIndex++\n\t\t\t) {\n\t\t\t\tconst contentBytes = Math.min(CHAT_LOG_CHUNK_BYTES, remainingBytes);\n\t\t\t\tconst seq = rows;\n\t\t\t\tconst role = seq % 2 === 0 ? \"user\" : \"assistant\";\n\n\t\t\t\tplaceholders.push(\"(?, ?, ?, ?, ?, ?, ?)\");\n\t\t\t\targs.push(\n\t\t\t\t\tthreadId,\n\t\t\t\t\tseq,\n\t\t\t\t\trole,\n\t\t\t\t\tbuildChatLogMessage(seq, contentBytes),\n\t\t\t\t\tcontentBytes,\n\t\t\t\t\tMath.ceil(contentBytes / 4),\n\t\t\t\t\tcreatedAtBase + seq,\n\t\t\t\t);\n\n\t\t\t\tremainingBytes -= contentBytes;\n\t\t\t\trows++;\n\t\t\t}\n\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO chat_log (thread_id, seq, role, content, content_bytes, token_estimate, created_at) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t...args,\n\t\t\t);\n\t\t}\n\n\t\tawait database.execute(\"COMMIT\");\n\t} catch (err) {\n\t\tawait database.execute(\"ROLLBACK\");\n\t\tthrow err;\n\t}\n\n\treturn { threadId, rows, totalBytes: targetBytes };\n}\n\nexport const testSqliteBench = actor({\n\toptions: {\n\t\tactionTimeout: 300_000,\n\t},\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS bench (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tkey TEXT NOT NULL,\n\t\t\t\tvalue TEXT NOT NULL,\n\t\t\t\tnum INTEGER NOT NULL DEFAULT 0,\n\t\t\t\tpayload BLOB,\n\t\t\t\tcreated_at INTEGER NOT NULL DEFAULT 0\n\t\t\t)`);\n\t\t\tawait database.execute(\"CREATE INDEX IF NOT EXISTS idx_bench_key ON bench(key)\");\n\t\t\tawait database.execute(\"CREATE INDEX IF NOT EXISTS idx_bench_num ON bench(num)\");\n\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS bench_json (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tdata TEXT NOT NULL DEFAULT '{}'\n\t\t\t)`);\n\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS bench_secondary (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tbench_id INTEGER NOT NULL,\n\t\t\t\tlabel TEXT NOT NULL,\n\t\t\t\tscore REAL NOT NULL DEFAULT 0,\n\t\t\t\tFOREIGN KEY (bench_id) REFERENCES bench(id)\n\t\t\t)`);\n\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS chat_log (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tthread_id TEXT NOT NULL,\n\t\t\t\tseq INTEGER NOT NULL,\n\t\t\t\trole TEXT NOT NULL,\n\t\t\t\tcontent TEXT NOT NULL,\n\t\t\t\tcontent_bytes INTEGER NOT NULL,\n\t\t\t\ttoken_estimate INTEGER NOT NULL,\n\t\t\t\tcreated_at INTEGER NOT NULL DEFAULT 0\n\t\t\t)`);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_chat_log_thread_seq ON chat_log(thread_id, seq DESC)\",\n\t\t\t);\n\t\t},\n\t}),\n\tactions: {\n\t\tnoop: (_c) => ({ ok: true }),\n\n\t\tgoToSleep: (c) => {\n\t\t\tc.sleep();\n\t\t\treturn { ok: true };\n\t\t},\n\n\t\tinsertSingle: async (c, n: number) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tfor (let i = 0; i < n; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`k-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\treturn { ms: performance.now() - t0, ops: n };\n\t\t},\n\n\t\tinsertTx: async (c, n: number) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < n; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`k-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: n };\n\t\t},\n\n\t\tinsertBatch: async (c, n: number) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tconst placeholders = Array.from({ length: n }, () => \"(?, ?, ?, ?)\").join(\", \");\n\t\t\tconst args: unknown[] = [];\n\t\t\tfor (let i = 0; i < n; i++) {\n\t\t\t\targs.push(`k-${i}`, `v-${i}`, i, Date.now());\n\t\t\t}\n\t\t\tawait c.db.execute(`INSERT INTO bench (key, value, num, created_at) VALUES ${placeholders}`, ...args);\n\t\t\treturn { ms: performance.now() - t0, ops: n };\n\t\t},\n\n\t\tpointRead: async (c, n: number) => {\n\t\t\tawait c.db.execute(\"INSERT INTO bench (key, value, num, created_at) VALUES ('pr', 'pr', 0, 0)\");\n\t\t\tconst rows = await c.db.execute(\"SELECT id FROM bench WHERE key = 'pr' LIMIT 1\");\n\t\t\tconst id = (rows[0] as { id: number }).id;\n\t\t\tconst t0 = performance.now();\n\t\t\tfor (let i = 0; i < n; i++) {\n\t\t\t\tawait c.db.execute(\"SELECT * FROM bench WHERE id = ?\", id);\n\t\t\t}\n\t\t\treturn { ms: performance.now() - t0, ops: n };\n\t\t},\n\n\t\tfullScan: async (c, seedRows: number) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < seedRows; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`scan-${i}`, `val-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\"SELECT * FROM bench\");\n\t\t\treturn { ms: performance.now() - t0, seedMs, rows: (rows as unknown[]).length };\n\t\t},\n\n\t\trangeScanIndexed: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 500; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`rs-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\"SELECT * FROM bench WHERE num BETWEEN 100 AND 300\");\n\t\t\treturn { ms: performance.now() - t0, seedMs, rows: (rows as unknown[]).length };\n\t\t},\n\n\t\trangeScanUnindexed: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 500; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`ru-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\"SELECT * FROM bench WHERE value BETWEEN 'v-100' AND 'v-300'\");\n\t\t\treturn { ms: performance.now() - t0, seedMs, rows: (rows as unknown[]).length };\n\t\t},\n\n\t\tbulkUpdate: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 200; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`bu-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"UPDATE bench SET value = 'updated', num = num + 1000 WHERE key LIKE 'bu-%'\");\n\t\t\treturn { ms: performance.now() - t0, seedMs, ops: 200 };\n\t\t},\n\n\t\tbulkDelete: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 200; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`bd-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"DELETE FROM bench WHERE key LIKE 'bd-%'\");\n\t\t\treturn { ms: performance.now() - t0, seedMs, ops: 200 };\n\t\t},\n\n\t\thotRowUpdates: async (c, n: number) => {\n\t\t\tawait c.db.execute(\"INSERT INTO bench (key, value, num, created_at) VALUES ('hot', 'v', 0, 0)\");\n\t\t\tconst rows = await c.db.execute(\"SELECT id FROM bench WHERE key = 'hot' LIMIT 1\");\n\t\t\tconst id = (rows[0] as { id: number }).id;\n\t\t\tconst t0 = performance.now();\n\t\t\tfor (let i = 0; i < n; i++) {\n\t\t\t\tawait c.db.execute(\"UPDATE bench SET num = ? WHERE id = ?\", i, id);\n\t\t\t}\n\t\t\treturn { ms: performance.now() - t0, ops: n };\n\t\t},\n\n\t\tvacuumAfterDelete: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 500; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`vac-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tawait c.db.execute(\"DELETE FROM bench WHERE key LIKE 'vac-%'\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"VACUUM\");\n\t\t\treturn { ms: performance.now() - t0, seedMs };\n\t\t},\n\n\t\tlargePayloadInsert: async (c, n: number) => {\n\t\t\tconst blob = \"x\".repeat(32 * 1024);\n\t\t\tconst t0 = performance.now();\n\t\t\tfor (let i = 0; i < n; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, payload, created_at) VALUES (?, ?, ?, ?, ?)\",\n\t\t\t\t\t`lp-${i}`, `v-${i}`, i, blob, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\treturn { ms: performance.now() - t0, ops: n };\n\t\t},\n\n\t\tmixedOltp: async (c) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\"oltp\", \"initial\", 0, Date.now(),\n\t\t\t);\n\t\t\tconst rows = await c.db.execute(\"SELECT * FROM bench WHERE key = 'oltp' LIMIT 1\");\n\t\t\tconst id = (rows[0] as { id: number }).id;\n\t\t\tawait c.db.execute(\"UPDATE bench SET value = 'updated', num = 1 WHERE id = ?\", id);\n\t\t\tawait c.db.execute(\"SELECT * FROM bench WHERE id = ?\", id);\n\t\t\treturn { ms: performance.now() - t0, ops: 4 };\n\t\t},\n\n\t\tjsonInsertAndQuery: async (c) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tfor (let i = 0; i < 50; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench_json (data) VALUES (?)\",\n\t\t\t\t\tJSON.stringify({ name: `item-${i}`, tags: [\"a\", \"b\"], score: Math.random() * 100 }),\n\t\t\t\t);\n\t\t\t}\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT id, json_extract(data, '$.name') as name, json_extract(data, '$.score') as score FROM bench_json ORDER BY json_extract(data, '$.score') DESC LIMIT 10\",\n\t\t\t);\n\t\t\treturn { ms: performance.now() - t0, ops: 51, rows: (rows as unknown[]).length };\n\t\t},\n\n\t\tjsonEachAgg: async (c) => {\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO bench_json (data) VALUES (?)\",\n\t\t\t\tJSON.stringify({ items: Array.from({ length: 100 }, (_, i) => ({ id: i, val: i * 10 })) }),\n\t\t\t);\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT SUM(json_extract(value, '$.val')) as total FROM bench_json, json_each(json_extract(data, '$.items')) LIMIT 1\",\n\t\t\t);\n\t\t\treturn { ms: performance.now() - t0, total: (rows[0] as { total: number }).total };\n\t\t},\n\n\t\tcomplexAggregation: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 200; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`grp-${i % 10}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT key, COUNT(*) as cnt, AVG(num) as avg_num, MIN(num) as min_num, MAX(num) as max_num FROM bench WHERE key LIKE 'grp-%' GROUP BY key ORDER BY cnt DESC\",\n\t\t\t);\n\t\t\treturn { ms: performance.now() - t0, seedMs, groups: (rows as unknown[]).length };\n\t\t},\n\n\t\tcomplexSubquery: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 200; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`sq-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT * FROM bench WHERE num > (SELECT AVG(num) FROM bench) ORDER BY num DESC LIMIT 50\",\n\t\t\t);\n\t\t\treturn { ms: performance.now() - t0, seedMs, rows: (rows as unknown[]).length };\n\t\t},\n\n\t\tcomplexJoin: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 200; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`j-${i}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench_secondary (bench_id, label, score) VALUES (?, ?, ?)\",\n\t\t\t\t\ti + 1, `label-${i}`, Math.random() * 100,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT b.key, b.num, s.label, s.score FROM bench b INNER JOIN bench_secondary s ON s.bench_id = b.id WHERE b.key LIKE 'j-%' ORDER BY s.score DESC LIMIT 200\",\n\t\t\t);\n\t\t\treturn { ms: performance.now() - t0, seedMs, rows: (rows as unknown[]).length };\n\t\t},\n\n\t\tcomplexCteWindow: async (c) => {\n\t\t\tconst t0Seed = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 200; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bench (key, value, num, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t`cte-${i % 10}`, `v-${i}`, i, Date.now(),\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst seedMs = performance.now() - t0Seed;\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(`\n\t\t\t\tWITH ranked AS (\n\t\t\t\t\tSELECT key, num, ROW_NUMBER() OVER (PARTITION BY key ORDER BY num DESC) as rn,\n\t\t\t\t\t AVG(num) OVER (PARTITION BY key) as avg_num\n\t\t\t\t\tFROM bench\n\t\t\t\t\tWHERE key LIKE 'cte-%'\n\t\t\t\t)\n\t\t\t\tSELECT * FROM ranked WHERE rn <= 3 ORDER BY key, rn\n\t\t\t`);\n\t\t\treturn { ms: performance.now() - t0, seedMs, rows: (rows as unknown[]).length };\n\t\t},\n\n\t\tmigrationTables: async (c, n: number) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < n; i++) {\n\t\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS mig_${i} (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tdata TEXT NOT NULL DEFAULT ''\n\t\t\t\t)`);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: n };\n\t\t},\n\n\t\tchatLogInsert: async (c, totalBytes: number) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tconst seeded = await seedChatLog(c.db, totalBytes);\n\t\t\treturn { ms: performance.now() - t0, ops: seeded.rows, bytes: seeded.totalBytes };\n\t\t},\n\n\t\tchatLogSelectLimit: async (c, totalBytes: number) => {\n\t\t\tconst seeded = await seedChatLog(c.db, totalBytes);\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT seq, role, substr(content, 1, 128) AS preview FROM chat_log ORDER BY created_at DESC LIMIT 100\",\n\t\t\t);\n\t\t\treturn {\n\t\t\t\tms: performance.now() - t0,\n\t\t\t\tops: (rows as unknown[]).length,\n\t\t\t\trows: (rows as unknown[]).length,\n\t\t\t\tbytes: seeded.totalBytes,\n\t\t\t};\n\t\t},\n\n\t\tchatLogSelectIndexed: async (c, totalBytes: number) => {\n\t\t\tconst seeded = await seedChatLog(c.db, totalBytes);\n\t\t\tconst lowerBound = Math.max(0, seeded.rows - 100);\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT seq, role, content_bytes FROM chat_log WHERE thread_id = ? AND seq >= ? ORDER BY seq DESC LIMIT 100\",\n\t\t\t\tseeded.threadId,\n\t\t\t\tlowerBound,\n\t\t\t);\n\t\t\treturn {\n\t\t\t\tms: performance.now() - t0,\n\t\t\t\tops: (rows as unknown[]).length,\n\t\t\t\trows: (rows as unknown[]).length,\n\t\t\t\tbytes: seeded.totalBytes,\n\t\t\t};\n\t\t},\n\n\t\tchatLogCount: async (c, totalBytes: number) => {\n\t\t\tconst seeded = await seedChatLog(c.db, totalBytes);\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT COUNT(*) AS count FROM chat_log WHERE thread_id = ?\",\n\t\t\t\tseeded.threadId,\n\t\t\t);\n\t\t\treturn {\n\t\t\t\tms: performance.now() - t0,\n\t\t\t\tops: 1,\n\t\t\t\tcount: (rows[0] as { count: number }).count,\n\t\t\t\tbytes: seeded.totalBytes,\n\t\t\t};\n\t\t},\n\n\t\tchatLogSum: async (c, totalBytes: number) => {\n\t\t\tconst seeded = await seedChatLog(c.db, totalBytes);\n\t\t\tconst t0 = performance.now();\n\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\"SELECT SUM(content_bytes) AS total_bytes FROM chat_log WHERE thread_id = ?\",\n\t\t\t\tseeded.threadId,\n\t\t\t);\n\t\t\treturn {\n\t\t\t\tms: performance.now() - t0,\n\t\t\t\tops: 1,\n\t\t\t\ttotalBytes: (rows[0] as { total_bytes: number | null }).total_bytes ?? 0,\n\t\t\t\tbytes: seeded.totalBytes,\n\t\t\t};\n\t\t},\n\n\t\tlargeTxInsert500KB: async (c) => {\n\t\t\tconst targetBytes = 500 * 1024;\n\t\t\tconst rowSize = 4 * 1024;\n\t\t\tconst rowCount = Math.ceil(targetBytes / rowSize);\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < rowCount; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO large_tx (payload) VALUES (randomblob(?))\",\n\t\t\t\t\trowSize,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize };\n\t\t},\n\n\t\tlargeTxInsert1MB: async (c) => {\n\t\t\tconst targetBytes = 1024 * 1024;\n\t\t\tconst rowSize = 4 * 1024;\n\t\t\tconst rowCount = Math.ceil(targetBytes / rowSize);\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < rowCount; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO large_tx (payload) VALUES (randomblob(?))\",\n\t\t\t\t\trowSize,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize };\n\t\t},\n\n\t\t// 1 MiB total, 4096 × 256 B rows. Max NAPI crossings.\n\t\tlargeTxInsert1MBTinyRows: async (c) => {\n\t\t\tconst targetBytes = 1024 * 1024;\n\t\t\tconst rowSize = 256;\n\t\t\tconst rowCount = Math.ceil(targetBytes / rowSize);\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < rowCount; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO large_tx (payload) VALUES (randomblob(?))\",\n\t\t\t\t\trowSize,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize };\n\t\t},\n\n\t\t// 1 MiB total, 256 × 4 KiB rows. Same shape as largeTxInsert1MB; kept as a sanity duplicate.\n\t\tlargeTxInsert1MBMediumRows: async (c) => {\n\t\t\tconst targetBytes = 1024 * 1024;\n\t\t\tconst rowSize = 4 * 1024;\n\t\t\tconst rowCount = Math.ceil(targetBytes / rowSize);\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < rowCount; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO large_tx (payload) VALUES (randomblob(?))\",\n\t\t\t\t\trowSize,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize };\n\t\t},\n\n\t\t// 1 MiB total, 1 × 1 MiB row. One NAPI crossing, exercises SQLite overflow-page chain.\n\t\tlargeTxInsert1MBOneRow: async (c) => {\n\t\t\tconst rowSize = 1024 * 1024;\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO large_tx (payload) VALUES (randomblob(?))\",\n\t\t\t\trowSize,\n\t\t\t);\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: 1, bytes: rowSize };\n\t\t},\n\n\t\tlargeTxInsert5MB: async (c) => {\n\t\t\tconst targetBytes = 5 * 1024 * 1024;\n\t\t\tconst rowSize = 4 * 1024;\n\t\t\tconst rowCount = Math.ceil(targetBytes / rowSize);\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < rowCount; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO large_tx (payload) VALUES (randomblob(?))\",\n\t\t\t\t\trowSize,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize };\n\t\t},\n\n\t\tlargeTxInsert10MB: async (c) => {\n\t\t\tconst targetBytes = 10 * 1024 * 1024;\n\t\t\tconst rowSize = 4 * 1024;\n\t\t\tconst rowCount = Math.ceil(targetBytes / rowSize);\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < rowCount; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO large_tx (payload) VALUES (randomblob(?))\",\n\t\t\t\t\trowSize,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize };\n\t\t},\n\n\t\tlargeTxInsert50MB: async (c) => {\n\t\t\tconst targetBytes = 50 * 1024 * 1024;\n\t\t\tconst rowSize = 4 * 1024;\n\t\t\tconst rowCount = Math.ceil(targetBytes / rowSize);\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS large_tx (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < rowCount; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO large_tx (payload) VALUES (randomblob(?))\",\n\t\t\t\t\trowSize,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: rowCount, bytes: rowCount * rowSize };\n\t\t},\n\n\t\t// Stress test: insert 1000 rows, delete them all, repeat 10 times.\n\t\t// Tests freelist reuse and space reclamation patterns.\n\t\tchurnInsertDelete: async (c) => {\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS churn (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\tconst t0 = performance.now();\n\t\t\tconst cycles = 10;\n\t\t\tconst perCycle = 1000;\n\t\t\tfor (let cycle = 0; cycle < cycles; cycle++) {\n\t\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\t\tfor (let i = 0; i < perCycle; i++) {\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"INSERT INTO churn (payload) VALUES (randomblob(1024))\",\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tawait c.db.execute(\"DELETE FROM churn\");\n\t\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\t}\n\t\t\treturn {\n\t\t\t\tms: performance.now() - t0,\n\t\t\t\tops: cycles * perCycle,\n\t\t\t\tcycles,\n\t\t\t};\n\t\t},\n\n\t\t// Interleave inserts, updates, deletes in same transaction. Tests how\n\t\t// the VFS handles mixed page dirtying patterns.\n\t\tmixedOltpLarge: async (c) => {\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS mixed_oltp (\n\t\t\t\tid INTEGER PRIMARY KEY,\n\t\t\t\tvalue INTEGER NOT NULL,\n\t\t\t\tdata BLOB NOT NULL\n\t\t\t)`);\n\t\t\tawait c.db.execute(\"DELETE FROM mixed_oltp\");\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 500; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO mixed_oltp (id, value, data) VALUES (?, ?, randomblob(1024))\",\n\t\t\t\t\ti,\n\t\t\t\t\ti * 2,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 500; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO mixed_oltp (id, value, data) VALUES (?, ?, randomblob(1024))\",\n\t\t\t\t\t500 + i,\n\t\t\t\t\ti * 3,\n\t\t\t\t);\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"UPDATE mixed_oltp SET value = value + 1 WHERE id = ?\",\n\t\t\t\t\ti,\n\t\t\t\t);\n\t\t\t\tif (i % 5 === 0) {\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"DELETE FROM mixed_oltp WHERE id = ?\",\n\t\t\t\t\t\ti - 50 >= 0 ? i - 50 : i,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: 500 * 2 + 100 };\n\t\t},\n\n\t\t// Growing aggregation: insert then SELECT SUM after each batch.\n\t\t// Tests cache invalidation and read-after-write patterns.\n\t\tgrowingAggregation: async (c) => {\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS agg_test (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tvalue INTEGER NOT NULL\n\t\t\t)`);\n\t\t\tawait c.db.execute(\"DELETE FROM agg_test\");\n\t\t\tconst t0 = performance.now();\n\t\t\tconst batches = 20;\n\t\t\tconst perBatch = 100;\n\t\t\tlet lastSum = 0;\n\t\t\tfor (let batch = 0; batch < batches; batch++) {\n\t\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\t\tfor (let i = 0; i < perBatch; i++) {\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"INSERT INTO agg_test (value) VALUES (?)\",\n\t\t\t\t\t\tbatch * perBatch + i,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\t\tconst rows = (await c.db.execute(\n\t\t\t\t\t\"SELECT SUM(value) AS s FROM agg_test\",\n\t\t\t\t)) as Array<{ s: number }>;\n\t\t\t\tlastSum = rows[0]?.s ?? 0;\n\t\t\t}\n\t\t\treturn {\n\t\t\t\tms: performance.now() - t0,\n\t\t\t\tops: batches * perBatch,\n\t\t\t\tbatches,\n\t\t\t\tlastSum,\n\t\t\t};\n\t\t},\n\n\t\t// Create index on already-populated table. Tests large rewrite patterns.\n\t\tindexCreationOnLargeTable: async (c) => {\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS idx_test (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tkey TEXT NOT NULL,\n\t\t\t\tvalue INTEGER NOT NULL\n\t\t\t)`);\n\t\t\tawait c.db.execute(\"DROP INDEX IF EXISTS idx_test_key\");\n\t\t\tawait c.db.execute(\"DELETE FROM idx_test\");\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 10000; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO idx_test (key, value) VALUES (?, ?)\",\n\t\t\t\t\t`key-${i % 1000}-${i}`,\n\t\t\t\t\ti,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"CREATE INDEX idx_test_key ON idx_test(key)\");\n\t\t\treturn { ms: performance.now() - t0, ops: 10000 };\n\t\t},\n\n\t\t// Update 1000 different rows in separate UPDATEs in one transaction.\n\t\t// Stresses B-tree navigation and page dirtying.\n\t\tbulkUpdate1000Rows: async (c) => {\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS bulk_update (\n\t\t\t\tid INTEGER PRIMARY KEY,\n\t\t\t\tvalue INTEGER NOT NULL\n\t\t\t)`);\n\t\t\tawait c.db.execute(\"DELETE FROM bulk_update\");\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 1000; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO bulk_update (id, value) VALUES (?, ?)\",\n\t\t\t\t\ti,\n\t\t\t\t\ti,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 1000; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"UPDATE bulk_update SET value = value + 1 WHERE id = ?\",\n\t\t\t\t\ti,\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: 1000 };\n\t\t},\n\n\t\t// Delete everything then re-insert. Tests truncate+regrow cycle.\n\t\ttruncateAndRegrow: async (c) => {\n\t\t\tawait c.db.execute(`CREATE TABLE IF NOT EXISTS regrow (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tpayload BLOB NOT NULL\n\t\t\t)`);\n\t\t\t// Seed\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 500; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO regrow (payload) VALUES (randomblob(1024))\",\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"DELETE FROM regrow\");\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 500; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO regrow (payload) VALUES (randomblob(1024))\",\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: 500 };\n\t\t},\n\n\t\t// Many small tables vs one large. Tests schema page growth.\n\t\tmanySmallTables: async (c) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\tfor (let i = 0; i < 50; i++) {\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t`CREATE TABLE IF NOT EXISTS small_t_${i} (id INTEGER PRIMARY KEY, value INTEGER)`,\n\t\t\t\t);\n\t\t\t\tfor (let j = 0; j < 10; j++) {\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t`INSERT INTO small_t_${i} (id, value) VALUES (?, ?)`,\n\t\t\t\t\t\tj,\n\t\t\t\t\t\ti * j,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\treturn { ms: performance.now() - t0, ops: 50 * 10, tables: 50 };\n\t\t},\n\t},\n});\n","import { randomBytes } from \"node:crypto\";\nimport { actor } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nconst DEFAULT_TARGET_BYTES = 50 * 1024 * 1024;\nconst DEFAULT_ROW_BYTES = 16 * 1024;\nconst DEFAULT_BATCH_ROWS = 8;\nconst DEFAULT_TRANSACTION_BYTES = 64 * 1024;\nconst READ_BATCH_ROWS = 64;\nconst REVERSE_PROBE_ROWS = 32 * 1024;\nconst PAYLOAD_TABLE = \"cold_start_payload\";\nconst REVERSE_PROBE_TABLE = \"cold_start_reverse_probe\";\n\ninterface WriteInput {\n\ttargetBytes?: number;\n\trowBytes?: number;\n\tbatchRows?: number;\n\ttransactionBytes?: number;\n}\n\ninterface PayloadRow {\n\tmin_id?: number | null;\n\tmax_id?: number | null;\n\trows: number;\n\tbytes: number;\n\texpected_bytes: number;\n}\n\ninterface PayloadValueRow {\n\tbytes: number;\n\texpected_bytes: number;\n}\n\nfunction positiveInteger(value: number | undefined, fallback: number, name: string) {\n\tconst resolved = value ?? fallback;\n\tif (!Number.isInteger(resolved) || resolved < 1) {\n\t\tthrow new Error(`${name} must be a positive integer`);\n\t}\n\treturn resolved;\n}\n\nfunction randomAsciiString(bytes: number): string {\n\treturn randomBytes(Math.ceil(bytes / 2)).toString(\"hex\").slice(0, bytes);\n}\n\nasync function readPayloads(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\tdirection: \"forward\" | \"backward\" = \"forward\",\n) {\n\tconst t0 = performance.now();\n\tconst [bounds] = (await database.execute(\n\t\t`\n\t\t\tSELECT\n\t\t\t\tMIN(id) AS min_id,\n\t\t\t\tMAX(id) AS max_id,\n\t\t\t\tCOUNT(*) AS rows,\n\t\t\t\t0 AS bytes,\n\t\t\t\t0 AS expected_bytes\n\t\t\tFROM ${PAYLOAD_TABLE}\n\t\t`,\n\t)) as PayloadRow[];\n\n\tif (!bounds) throw new Error(\"read query returned no rows\");\n\n\tlet rows = 0;\n\tlet bytes = 0;\n\tlet expectedBytes = 0;\n\tlet chunks = 0;\n\tconst minId = bounds.min_id ?? 0;\n\tconst maxId = bounds.max_id ?? 0;\n\n\tif (direction === \"backward\") {\n\t\tconst [probeBounds] = (await database.execute(\n\t\t\t`\n\t\t\t\tSELECT\n\t\t\t\t\tMIN(id) AS min_id,\n\t\t\t\t\tMAX(id) AS max_id,\n\t\t\t\t\tCOUNT(*) AS rows,\n\t\t\t\t\t0 AS bytes,\n\t\t\t\t\t0 AS expected_bytes\n\t\t\t\tFROM ${REVERSE_PROBE_TABLE}\n\t\t\t`,\n\t\t)) as PayloadRow[];\n\t\tif (!probeBounds) throw new Error(\"reverse probe query returned no rows\");\n\t\tconst probeMinId = probeBounds.min_id ?? 0;\n\t\tconst probeMaxId = probeBounds.max_id ?? 0;\n\n\t\tfor (\n\t\t\tlet upperId = probeMaxId;\n\t\t\tupperId >= probeMinId && upperId > 0;\n\t\t\tupperId -= READ_BATCH_ROWS\n\t\t) {\n\t\t\tconst lowerId = Math.max(probeMinId, upperId - READ_BATCH_ROWS + 1);\n\t\t\tconst chunkRows = (await database.execute(\n\t\t\t\t`\n\t\t\t\t\tSELECT\n\t\t\t\t\t\tmarker AS bytes,\n\t\t\t\t\t\tmarker AS expected_bytes\n\t\t\t\t\tFROM ${REVERSE_PROBE_TABLE}\n\t\t\t\t\tWHERE id BETWEEN ? AND ?\n\t\t\t\t\tORDER BY id DESC\n\t\t\t\t`,\n\t\t\t\tlowerId,\n\t\t\t\tupperId,\n\t\t\t)) as PayloadValueRow[];\n\n\t\t\tfor (const row of chunkRows) {\n\t\t\t\trows += 1;\n\t\t\t\tbytes += row.bytes;\n\t\t\t\texpectedBytes += row.expected_bytes;\n\t\t\t}\n\t\t\tchunks += 1;\n\t\t}\n\n\t\treturn {\n\t\t\tms: performance.now() - t0,\n\t\t\tops: rows,\n\t\t\trows,\n\t\t\tbytes,\n\t\t\texpectedBytes,\n\t\t\tchunks,\n\t\t\treadBatchRows: READ_BATCH_ROWS,\n\t\t\tdirection,\n\t\t};\n\t}\n\n\tfor (\n\t\tlet lowerId = minId;\n\t\tlowerId <= maxId;\n\t\tlowerId += READ_BATCH_ROWS\n\t) {\n\t\tconst upperId = lowerId + READ_BATCH_ROWS - 1;\n\t\tconst [chunk] = (await database.execute(\n\t\t\t`\n\t\t\t\tSELECT\n\t\t\t\t\tCOUNT(*) AS rows,\n\t\t\t\t\tCOALESCE(SUM(length(payload)), 0) AS bytes,\n\t\t\t\t\tCOALESCE(SUM(payload_bytes), 0) AS expected_bytes\n\t\t\t\tFROM ${PAYLOAD_TABLE}\n\t\t\t\tWHERE id BETWEEN ? AND ?\n\t\t\t`,\n\t\t\tlowerId,\n\t\t\tupperId,\n\t\t)) as PayloadRow[];\n\t\tif (!chunk) throw new Error(\"chunked read query returned no rows\");\n\n\t\trows += chunk.rows;\n\t\tbytes += chunk.bytes;\n\t\texpectedBytes += chunk.expected_bytes;\n\t\tchunks += 1;\n\t}\n\n\treturn {\n\t\tms: performance.now() - t0,\n\t\tops: rows,\n\t\trows,\n\t\tbytes,\n\t\texpectedBytes,\n\t\tchunks,\n\t\treadBatchRows: READ_BATCH_ROWS,\n\t};\n}\n\nexport const sqliteColdStartBench = actor({\n\toptions: {\n\t\tactionTimeout: 600_000,\n\t},\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS cold_start_payload (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tpayload TEXT NOT NULL,\n\t\t\t\t\tpayload_bytes INTEGER NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS cold_start_reverse_probe (\n\t\t\t\t\tid INTEGER PRIMARY KEY,\n\t\t\t\t\tmarker INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t},\n\t}),\n\tactions: {\n\t\treset: async (c) => {\n\t\t\tawait c.db.execute(`DELETE FROM ${PAYLOAD_TABLE}`);\n\t\t\tawait c.db.execute(`DELETE FROM ${REVERSE_PROBE_TABLE}`);\n\t\t\treturn { ok: true };\n\t\t},\n\n\t\twriteRandomStrings: async (c, input: WriteInput = {}) => {\n\t\t\tconst targetBytes = positiveInteger(\n\t\t\t\tinput.targetBytes,\n\t\t\t\tDEFAULT_TARGET_BYTES,\n\t\t\t\t\"targetBytes\",\n\t\t\t);\n\t\t\tconst rowBytes = positiveInteger(input.rowBytes, DEFAULT_ROW_BYTES, \"rowBytes\");\n\t\t\tconst batchRows = positiveInteger(\n\t\t\t\tinput.batchRows,\n\t\t\t\tDEFAULT_BATCH_ROWS,\n\t\t\t\t\"batchRows\",\n\t\t\t);\n\t\t\tconst transactionBytes = positiveInteger(\n\t\t\t\tinput.transactionBytes,\n\t\t\t\tDEFAULT_TRANSACTION_BYTES,\n\t\t\t\t\"transactionBytes\",\n\t\t\t);\n\t\t\tconst createdAt = Date.now();\n\t\t\tlet remainingBytes = targetBytes;\n\t\t\tlet rows = 0;\n\t\t\tlet transactions = 0;\n\t\t\tlet randomStringMs = 0;\n\t\t\tlet sqliteInsertMs = 0;\n\t\t\tlet commitMs = 0;\n\t\t\tlet inTransaction = false;\n\n\t\t\tconst wallT0 = performance.now();\n\t\t\ttry {\n\t\t\t\twhile (remainingBytes > 0) {\n\t\t\t\t\tlet transactionRemainingBytes = Math.min(\n\t\t\t\t\t\ttransactionBytes,\n\t\t\t\t\t\tremainingBytes,\n\t\t\t\t\t);\n\t\t\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\t\t\tinTransaction = true;\n\t\t\t\t\ttransactions += 1;\n\n\t\t\t\t\twhile (transactionRemainingBytes > 0) {\n\t\t\t\t\t\tconst placeholders: string[] = [];\n\t\t\t\t\t\tconst args: unknown[] = [];\n\t\t\t\t\t\tconst generateT0 = performance.now();\n\n\t\t\t\t\t\tfor (\n\t\t\t\t\t\t\tlet batchIndex = 0;\n\t\t\t\t\t\t\tbatchIndex < batchRows &&\n\t\t\t\t\t\t\ttransactionRemainingBytes > 0 &&\n\t\t\t\t\t\t\tremainingBytes > 0;\n\t\t\t\t\t\t\tbatchIndex += 1\n\t\t\t\t\t\t) {\n\t\t\t\t\t\t\tconst payloadBytes = Math.min(\n\t\t\t\t\t\t\t\trowBytes,\n\t\t\t\t\t\t\t\ttransactionRemainingBytes,\n\t\t\t\t\t\t\t\tremainingBytes,\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t\tplaceholders.push(\"(?, ?, ?)\");\n\t\t\t\t\t\t\targs.push(\n\t\t\t\t\t\t\t\trandomAsciiString(payloadBytes),\n\t\t\t\t\t\t\t\tpayloadBytes,\n\t\t\t\t\t\t\t\tcreatedAt + rows,\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t\ttransactionRemainingBytes -= payloadBytes;\n\t\t\t\t\t\t\tremainingBytes -= payloadBytes;\n\t\t\t\t\t\t\trows += 1;\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\trandomStringMs += performance.now() - generateT0;\n\t\t\t\t\t\tconst insertT0 = performance.now();\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`INSERT INTO ${PAYLOAD_TABLE} (payload, payload_bytes, created_at) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t\t\t\t...args,\n\t\t\t\t\t\t);\n\t\t\t\t\t\tsqliteInsertMs += performance.now() - insertT0;\n\t\t\t\t\t}\n\n\t\t\t\t\tconst commitT0 = performance.now();\n\t\t\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\t\t\tcommitMs += performance.now() - commitT0;\n\t\t\t\t\tinTransaction = false;\n\t\t\t\t}\n\n\t\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\t\tinTransaction = true;\n\t\t\t\tfor (\n\t\t\t\t\tlet lowerId = 1;\n\t\t\t\t\tlowerId <= REVERSE_PROBE_ROWS;\n\t\t\t\t\tlowerId += 256\n\t\t\t\t) {\n\t\t\t\t\tconst upperId = Math.min(REVERSE_PROBE_ROWS, lowerId + 255);\n\t\t\t\t\tconst placeholders: string[] = [];\n\t\t\t\t\tconst args: unknown[] = [];\n\t\t\t\t\tfor (let id = lowerId; id <= upperId; id += 1) {\n\t\t\t\t\t\tplaceholders.push(\"(?, ?)\");\n\t\t\t\t\t\targs.push(id, 1);\n\t\t\t\t\t}\n\t\t\t\t\tconst insertT0 = performance.now();\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t`INSERT INTO ${REVERSE_PROBE_TABLE} (id, marker) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t\t\t...args,\n\t\t\t\t\t);\n\t\t\t\t\tsqliteInsertMs += performance.now() - insertT0;\n\t\t\t\t}\n\t\t\t\tconst reverseCommitT0 = performance.now();\n\t\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\t\tcommitMs += performance.now() - reverseCommitT0;\n\t\t\t\tinTransaction = false;\n\n\t\t\t\treturn {\n\t\t\t\t\tms: sqliteInsertMs + commitMs,\n\t\t\t\t\twriteWallMs: performance.now() - wallT0,\n\t\t\t\t\trandomStringMs,\n\t\t\t\t\tsqliteInsertMs,\n\t\t\t\t\tcommitMs,\n\t\t\t\t\tops: rows,\n\t\t\t\t\trows,\n\t\t\t\t\ttransactions,\n\t\t\t\t\tbytes: targetBytes,\n\t\t\t\t\trowBytes,\n\t\t\t\t\tbatchRows,\n\t\t\t\t\ttransactionBytes,\n\t\t\t\t\treverseProbeRows: REVERSE_PROBE_ROWS,\n\t\t\t\t};\n\t\t\t} catch (err) {\n\t\t\t\tif (inTransaction) {\n\t\t\t\t\tawait c.db.execute(\"ROLLBACK\");\n\t\t\t\t}\n\t\t\t\tthrow err;\n\t\t\t}\n\t\t},\n\n\t\treadAll: async (c) => {\n\t\t\treturn readPayloads(c.db);\n\t\t},\n\n\t\treadAllReverse: async (c) => {\n\t\t\treturn readPayloads(c.db, \"backward\");\n\t\t},\n\n\t\twakeSqlite: async (c) => {\n\t\t\tconst t0 = performance.now();\n\t\t\tconst [row] = (await c.db.execute(\n\t\t\t\t`SELECT COUNT(*) AS rows FROM ${PAYLOAD_TABLE} WHERE id = -1`,\n\t\t\t)) as Array<{ rows: number }>;\n\t\t\treturn {\n\t\t\t\tms: performance.now() - t0,\n\t\t\t\trows: row?.rows ?? 0,\n\t\t\t};\n\t\t},\n\n\t\tgoToSleep: (c) => {\n\t\t\tc.sleep();\n\t\t\treturn { ok: true };\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nconst DEFAULT_ROW_BYTES = 2 * 1024;\nconst ORDER_BATCH_ROWS = 50;\nconst DOC_BATCH_ROWS = 75;\nconst LEDGER_BATCH_ROWS = 100;\nconst POINT_LOOKUP_OPS = 1_000;\nconst RANGE_CHUNK_ROWS = 512;\nconst SETUP_TRANSACTION_ROWS = 128;\nconst FEED_PAGE_ROWS = 100;\nconst CHAT_LOG_CHUNK_BYTES = 4 * 1024;\nconst CHAT_LOG_INSERT_BATCH_SIZE = 50;\nconst CHAT_THREAD_ID = \"rw-chat-main\";\nconst SQL_RUSH_MSGS_COUNT = 2_500;\nconst SQL_RUSH_TOOL_REFS_COUNT = 240;\nconst SQL_RUSH_EVENTS_COUNT = 700;\nconst SQL_RUSH_KV_COUNT = 40;\nconst SQL_RUSH_TOOLS_COUNT = 41;\nconst SQL_RUSH_META_COUNT = 12;\n\n// Keep this list in sync with the runner's workload catalog. The runner owns\n// the per-workload rationale and expected cache/VFS behavior so benchmark\n// result artifacts can preserve that intent over time.\nconst WORKLOADS = [\n\t\"small-rowid-point\",\n\t\"small-schema-read\",\n\t\"small-range-scan\",\n\t\"rowid-range-forward\",\n\t\"rowid-range-backward\",\n\t\"secondary-index-covering-range\",\n\t\"secondary-index-scattered-table\",\n\t\"aggregate-status\",\n\t\"aggregate-time-bucket\",\n\t\"aggregate-tenant-time-range\",\n\t\"parallel-read-aggregates\",\n\t\"parallel-read-write-transition\",\n\t\"feed-order-by-limit\",\n\t\"feed-pagination-adjacent\",\n\t\"join-order-items\",\n\t\"random-point-lookups\",\n\t\"hot-index-cold-table\",\n\t\"ledger-without-rowid-range\",\n\t\"chat-log-select-limit\",\n\t\"chat-log-select-indexed\",\n\t\"chat-log-count\",\n\t\"chat-log-sum\",\n\t\"chat-tool-read-fanout\",\n\t\"chat-tool-script\",\n\t\"write-batch-after-wake\",\n\t\"update-hot-partition\",\n\t\"delete-churn-range-read\",\n\t\"migration-create-indexes-large\",\n\t\"migration-create-indexes-skewed-large\",\n\t\"migration-table-rebuild-large\",\n\t\"migration-add-column-large\",\n\t\"migration-ddl-small\",\n] as const;\n\ntype WorkloadName = (typeof WORKLOADS)[number];\n\ninterface SetupInput {\n\tworkload: WorkloadName;\n\ttargetBytes?: number;\n\trowBytes?: number;\n}\n\ninterface RunInput {\n\tworkload: WorkloadName;\n\ttargetBytes?: number;\n}\n\ninterface CountRow {\n\trows: number;\n}\n\ninterface PageCountRow {\n\tpage_count: number;\n}\n\ninterface CacheSizeRow {\n\tcache_size: number;\n}\n\ninterface PageSizeRow {\n\tpage_size: number;\n}\n\ninterface BytesRow {\n\tbytes: number;\n\trows?: number;\n}\n\ninterface AggregateRow {\n\trows: number;\n\ttotal: number;\n}\n\nfunction positiveInteger(value: number | undefined, fallback: number, name: string) {\n\tconst resolved = value ?? fallback;\n\tif (!Number.isInteger(resolved) || resolved < 1) {\n\t\tthrow new Error(`${name} must be a positive integer`);\n\t}\n\treturn resolved;\n}\n\nfunction assertWorkload(workload: string): asserts workload is WorkloadName {\n\tif (!(WORKLOADS as readonly string[]).includes(workload)) {\n\t\tthrow new Error(`unknown SQLite benchmark workload: ${workload}`);\n\t}\n}\n\nfunction pseudoRandom(value: number) {\n\treturn Math.imul(value ^ 0x9e3779b9, 0x85ebca6b) >>> 0;\n}\n\nfunction paddedHex(value: number) {\n\treturn pseudoRandom(value).toString(16).padStart(8, \"0\");\n}\n\nfunction payload(prefix: string, bytes: number) {\n\treturn prefix + \"x\".repeat(Math.max(0, bytes - prefix.length));\n}\n\nfunction typedRows(rows: unknown[]): T[] {\n\treturn rows as T[];\n}\n\nasync function queryPageCount(database: {\n\texecute: (sql: string, ...args: unknown[]) => Promise;\n}) {\n\tconst [row] = typedRows(await database.execute(\"PRAGMA page_count\"));\n\treturn row?.page_count ?? 0;\n}\n\nasync function resetCommerce(database: {\n\texecute: (sql: string, ...args: unknown[]) => Promise;\n}) {\n\tawait database.execute(\"DELETE FROM rw_order_items\");\n\tawait database.execute(\"DELETE FROM rw_orders\");\n\tawait database.execute(\"DELETE FROM rw_customers\");\n\tawait database.execute(\"DELETE FROM rw_events\");\n}\n\nasync function resetDocs(database: {\n\texecute: (sql: string, ...args: unknown[]) => Promise;\n}) {\n\tawait database.execute(\"DELETE FROM rw_docs\");\n}\n\nasync function resetLedger(database: {\n\texecute: (sql: string, ...args: unknown[]) => Promise;\n}) {\n\tawait database.execute(\"DELETE FROM rw_ledger\");\n}\n\nasync function resetChatLog(database: {\n\texecute: (sql: string, ...args: unknown[]) => Promise;\n}) {\n\tawait database.execute(\"DELETE FROM rw_chat_log\");\n}\n\nasync function resetSqlRush(database: {\n\texecute: (sql: string, ...args: unknown[]) => Promise;\n}) {\n\tawait database.execute(\"DELETE FROM tool_refs\");\n\tawait database.execute(\"DELETE FROM msgs\");\n\tawait database.execute(\"DELETE FROM events\");\n\tawait database.execute(\"DELETE FROM kv\");\n\tawait database.execute(\"DELETE FROM tools\");\n\tawait database.execute(\"DELETE FROM meta\");\n}\n\nasync function resetMigration(database: {\n\texecute: (sql: string, ...args: unknown[]) => Promise;\n}) {\n\tawait database.execute(\"DROP INDEX IF EXISTS idx_rw_migration_source_account\");\n\tawait database.execute(\"DROP INDEX IF EXISTS idx_rw_migration_source_created\");\n\tawait database.execute(\"DROP INDEX IF EXISTS idx_rw_migration_source_status_total\");\n\tawait database.execute(\"DROP INDEX IF EXISTS idx_rw_migration_source_skew_account\");\n\tawait database.execute(\"DROP INDEX IF EXISTS idx_rw_migration_source_skew_status\");\n\tawait database.execute(\"DROP TABLE IF EXISTS rw_migration_source_rebuilt\");\n\tawait database.execute(\"DROP TABLE IF EXISTS rw_migration_source\");\n\tawait database.execute(\"DROP TABLE IF EXISTS rw_migration_audit\");\n\tawait database.execute(\"DROP TABLE IF EXISTS rw_migration_empty\");\n}\n\nasync function withTransaction(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\tfn: () => Promise,\n) {\n\tlet inTransaction = false;\n\tawait database.execute(\"BEGIN\");\n\tinTransaction = true;\n\ttry {\n\t\tawait fn();\n\t\tawait database.execute(\"COMMIT\");\n\t\tinTransaction = false;\n\t} catch (err) {\n\t\tif (inTransaction) {\n\t\t\tawait database.execute(\"ROLLBACK\").catch(() => undefined);\n\t\t}\n\t\tthrow err;\n\t}\n}\n\nasync function seedCommerce(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\ttargetBytes: number,\n\trowBytes: number,\n) {\n\tawait resetCommerce(database);\n\tconst rows = Math.max(1, Math.ceil(targetBytes / rowBytes));\n\tconst customerCount = Math.max(32, Math.ceil(rows / 16));\n\tconst startedAt = performance.now();\n\n\tawait withTransaction(database, async () => {\n\t\tfor (let offset = 0; offset < customerCount; offset += ORDER_BATCH_ROWS) {\n\t\t\tconst placeholders: string[] = [];\n\t\t\tconst args: unknown[] = [];\n\t\t\tconst batchEnd = Math.min(customerCount, offset + ORDER_BATCH_ROWS);\n\t\t\tfor (let i = offset; i < batchEnd; i += 1) {\n\t\t\t\tplaceholders.push(\"(?, ?, ?, ?, ?)\");\n\t\t\t\targs.push(\n\t\t\t\t\ti + 1,\n\t\t\t\t\t`acct-${i % 64}`,\n\t\t\t\t\t`user-${paddedHex(i)}@example.test`,\n\t\t\t\t\t[\"free\", \"pro\", \"team\", \"enterprise\"][i % 4],\n\t\t\t\t\t[\"iad\", \"sfo\", \"fra\", \"sin\"][i % 4],\n\t\t\t\t);\n\t\t\t}\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO rw_customers (id, account_id, email, plan, region) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t...args,\n\t\t\t);\n\t\t}\n\t});\n\n\tfor (let txStart = 0; txStart < rows; txStart += SETUP_TRANSACTION_ROWS) {\n\t\tconst txEnd = Math.min(rows, txStart + SETUP_TRANSACTION_ROWS);\n\t\tawait withTransaction(database, async () => {\n\t\t\tfor (let offset = txStart; offset < txEnd; offset += ORDER_BATCH_ROWS) {\n\t\t\t\tconst orderPlaceholders: string[] = [];\n\t\t\t\tconst orderArgs: unknown[] = [];\n\t\t\t\tconst itemPlaceholders: string[] = [];\n\t\t\t\tconst itemArgs: unknown[] = [];\n\t\t\t\tconst eventPlaceholders: string[] = [];\n\t\t\t\tconst eventArgs: unknown[] = [];\n\t\t\t\tconst batchEnd = Math.min(txEnd, offset + ORDER_BATCH_ROWS);\n\n\t\t\t\tfor (let i = offset; i < batchEnd; i += 1) {\n\t\t\t\t\tconst id = i + 1;\n\t\t\t\t\tconst customerId = (pseudoRandom(i) % customerCount) + 1;\n\t\t\t\t\tconst createdAt = 1_700_000_000_000 + i * 1000;\n\t\t\t\t\tconst status = [\"pending\", \"paid\", \"shipped\", \"refunded\"][i % 4];\n\t\t\t\t\tconst totalCents = 500 + (pseudoRandom(i + 17) % 25_000);\n\t\t\t\t\tconst note = payload(`order-${id}-${status}:`, rowBytes);\n\n\t\t\t\t\torderPlaceholders.push(\"(?, ?, ?, ?, ?, ?, ?)\");\n\t\t\t\t\torderArgs.push(\n\t\t\t\t\t\tid,\n\t\t\t\t\t\tcustomerId,\n\t\t\t\t\t\tcreatedAt,\n\t\t\t\t\t\tstatus,\n\t\t\t\t\t\ttotalCents,\n\t\t\t\t\t\ti % 128,\n\t\t\t\t\t\tnote,\n\t\t\t\t\t);\n\n\t\t\t\t\tfor (let item = 0; item < 2; item += 1) {\n\t\t\t\t\t\titemPlaceholders.push(\"(?, ?, ?, ?, ?)\");\n\t\t\t\t\t\titemArgs.push(\n\t\t\t\t\t\t\tid,\n\t\t\t\t\t\t\t`sku-${paddedHex(i + item).slice(0, 6)}`,\n\t\t\t\t\t\t\t1 + ((i + item) % 5),\n\t\t\t\t\t\t\t100 + (pseudoRandom(i + item + 31) % 5000),\n\t\t\t\t\t\t\titem,\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\n\t\t\t\t\teventPlaceholders.push(\"(?, ?, ?, ?, ?)\");\n\t\t\t\t\teventArgs.push(\n\t\t\t\t\t\t`acct-${customerId % 64}`,\n\t\t\t\t\t\t[\"click\", \"purchase\", \"refund\", \"shipment\"][i % 4],\n\t\t\t\t\t\tcreatedAt,\n\t\t\t\t\t\t`order:${id}`,\n\t\t\t\t\t\tpayload(`event-${id}:`, Math.min(rowBytes, 512)),\n\t\t\t\t\t);\n\t\t\t\t}\n\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`INSERT INTO rw_orders (id, customer_id, created_at, status, total_cents, shard, note) VALUES ${orderPlaceholders.join(\", \")}`,\n\t\t\t\t\t...orderArgs,\n\t\t\t\t);\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`INSERT INTO rw_order_items (order_id, sku, quantity, price_cents, line_no) VALUES ${itemPlaceholders.join(\", \")}`,\n\t\t\t\t\t...itemArgs,\n\t\t\t\t);\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`INSERT INTO rw_events (account_id, event_type, created_at, entity_key, properties) VALUES ${eventPlaceholders.join(\", \")}`,\n\t\t\t\t\t...eventArgs,\n\t\t\t\t);\n\t\t\t}\n\t\t});\n\t}\n\n\treturn {\n\t\trows,\n\t\ttargetBytes,\n\t\trowBytes,\n\t\tsetupMs: performance.now() - startedAt,\n\t\tpageCount: await queryPageCount(database),\n\t};\n}\n\nasync function seedDocs(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\ttargetBytes: number,\n\trowBytes: number,\n) {\n\tawait resetDocs(database);\n\tconst rows = Math.max(1, Math.ceil(targetBytes / rowBytes));\n\tconst startedAt = performance.now();\n\n\tfor (let txStart = 0; txStart < rows; txStart += SETUP_TRANSACTION_ROWS) {\n\t\tconst txEnd = Math.min(rows, txStart + SETUP_TRANSACTION_ROWS);\n\t\tawait withTransaction(database, async () => {\n\t\t\tfor (let offset = txStart; offset < txEnd; offset += DOC_BATCH_ROWS) {\n\t\t\t\tconst placeholders: string[] = [];\n\t\t\t\tconst args: unknown[] = [];\n\t\t\t\tconst batchEnd = Math.min(txEnd, offset + DOC_BATCH_ROWS);\n\t\t\t\tfor (let i = offset; i < batchEnd; i += 1) {\n\t\t\t\t\tconst rank = pseudoRandom(i);\n\t\t\t\t\tconst body = payload(`doc-${i}-${rank}:`, rowBytes);\n\t\t\t\t\tplaceholders.push(\"(?, ?, ?, ?, ?)\");\n\t\t\t\t\targs.push(\n\t\t\t\t\t\t`doc-${paddedHex(i)}`,\n\t\t\t\t\t\trank,\n\t\t\t\t\t\t`tenant-${rank % 128}`,\n\t\t\t\t\t\tbody,\n\t\t\t\t\t\trowBytes,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`INSERT INTO rw_docs (external_key, row_rank, tenant_id, body, body_bytes) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t\t...args,\n\t\t\t\t);\n\t\t\t}\n\t\t});\n\t}\n\n\treturn {\n\t\trows,\n\t\ttargetBytes,\n\t\trowBytes,\n\t\tsetupMs: performance.now() - startedAt,\n\t\tpageCount: await queryPageCount(database),\n\t};\n}\n\nasync function seedLedger(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\ttargetBytes: number,\n\trowBytes: number,\n) {\n\tawait resetLedger(database);\n\tconst rows = Math.max(1, Math.ceil(targetBytes / rowBytes));\n\tconst startedAt = performance.now();\n\n\tfor (let txStart = 0; txStart < rows; txStart += SETUP_TRANSACTION_ROWS) {\n\t\tconst txEnd = Math.min(rows, txStart + SETUP_TRANSACTION_ROWS);\n\t\tawait withTransaction(database, async () => {\n\t\t\tfor (let offset = txStart; offset < txEnd; offset += LEDGER_BATCH_ROWS) {\n\t\t\t\tconst placeholders: string[] = [];\n\t\t\t\tconst args: unknown[] = [];\n\t\t\t\tconst batchEnd = Math.min(txEnd, offset + LEDGER_BATCH_ROWS);\n\t\t\t\tfor (let i = offset; i < batchEnd; i += 1) {\n\t\t\t\t\tconst accountId = `acct-${String(i % 256).padStart(4, \"0\")}`;\n\t\t\t\t\tconst entryId = Math.floor(i / 256) + 1;\n\t\t\t\t\tplaceholders.push(\"(?, ?, ?, ?, ?)\");\n\t\t\t\t\targs.push(\n\t\t\t\t\t\taccountId,\n\t\t\t\t\t\tentryId,\n\t\t\t\t\t\t(i % 2 === 0 ? 1 : -1) * (100 + (i % 10_000)),\n\t\t\t\t\t\t1_700_000_000_000 + i * 1000,\n\t\t\t\t\t\tpayload(`ledger-${accountId}-${entryId}:`, Math.min(rowBytes, 512)),\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`INSERT INTO rw_ledger (account_id, entry_id, amount_cents, created_at, memo) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t\t...args,\n\t\t\t\t);\n\t\t\t}\n\t\t});\n\t}\n\n\treturn {\n\t\trows,\n\t\ttargetBytes,\n\t\trowBytes,\n\t\tsetupMs: performance.now() - startedAt,\n\t\tpageCount: await queryPageCount(database),\n\t};\n}\n\nfunction buildChatLogMessage(seq: number, targetBytes: number): string {\n\tconst prefix = `message-${seq}: `;\n\treturn prefix + \"x\".repeat(Math.max(0, targetBytes - prefix.length));\n}\n\nasync function seedChatLog(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\ttargetBytes: number,\n) {\n\tawait resetChatLog(database);\n\tconst createdAtBase = 1_700_000_000_000;\n\tlet remainingBytes = targetBytes;\n\tlet rows = 0;\n\tconst startedAt = performance.now();\n\n\tawait withTransaction(database, async () => {\n\t\twhile (remainingBytes > 0) {\n\t\t\tconst placeholders: string[] = [];\n\t\t\tconst args: unknown[] = [];\n\n\t\t\tfor (\n\t\t\t\tlet batchIndex = 0;\n\t\t\t\tbatchIndex < CHAT_LOG_INSERT_BATCH_SIZE && remainingBytes > 0;\n\t\t\t\tbatchIndex += 1\n\t\t\t) {\n\t\t\t\tconst contentBytes = Math.min(CHAT_LOG_CHUNK_BYTES, remainingBytes);\n\t\t\t\tconst seq = rows;\n\t\t\t\tconst role = seq % 2 === 0 ? \"user\" : \"assistant\";\n\n\t\t\t\tplaceholders.push(\"(?, ?, ?, ?, ?, ?, ?)\");\n\t\t\t\targs.push(\n\t\t\t\t\tCHAT_THREAD_ID,\n\t\t\t\t\tseq,\n\t\t\t\t\trole,\n\t\t\t\t\tbuildChatLogMessage(seq, contentBytes),\n\t\t\t\t\tcontentBytes,\n\t\t\t\t\tMath.ceil(contentBytes / 4),\n\t\t\t\t\tcreatedAtBase + seq,\n\t\t\t\t);\n\n\t\t\t\tremainingBytes -= contentBytes;\n\t\t\t\trows += 1;\n\t\t\t}\n\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO rw_chat_log (thread_id, seq, role, content, content_bytes, token_estimate, created_at) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t...args,\n\t\t\t);\n\t\t}\n\t});\n\n\treturn {\n\t\trows,\n\t\ttargetBytes,\n\t\trowBytes: CHAT_LOG_CHUNK_BYTES,\n\t\tsetupMs: performance.now() - startedAt,\n\t\tpageCount: await queryPageCount(database),\n\t};\n}\n\nasync function batchInsert(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\tsql: string,\n\trows: unknown[][],\n\tbatchSize: number,\n) {\n\tif (rows.length === 0) return;\n\tconst colsPerRow = rows[0]?.length ?? 0;\n\tif (colsPerRow === 0) return;\n\tconst placeholder = `(${\"?,\".repeat(colsPerRow).slice(0, -1)})`;\n\tfor (let i = 0; i < rows.length; i += batchSize) {\n\t\tconst chunk = rows.slice(i, i + batchSize);\n\t\tconst values = new Array(chunk.length).fill(placeholder).join(\",\");\n\t\tconst args: unknown[] = [];\n\t\tfor (const row of chunk) args.push(...row);\n\t\tawait database.execute(`${sql} VALUES ${values}`, ...args);\n\t}\n}\n\nasync function seedSqlRush(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\ttargetBytes: number,\n) {\n\tawait resetSqlRush(database);\n\tconst now = 1_700_000_000_000;\n\tconst startedAt = performance.now();\n\n\tawait withTransaction(database, async () => {\n\t\tconst msgsRows: unknown[][] = [];\n\t\tfor (let i = 0; i < SQL_RUSH_MSGS_COUNT; i += 1) {\n\t\t\tmsgsRows.push([\n\t\t\t\ti === 0 ? null : i,\n\t\t\t\ti % 3 === 0 ? \"user\" : \"assistant\",\n\t\t\t\tpayload(\"msg:\", 512),\n\t\t\t\t0,\n\t\t\t\tnow - (SQL_RUSH_MSGS_COUNT - i) * 1000,\n\t\t\t]);\n\t\t}\n\t\tawait batchInsert(\n\t\t\tdatabase,\n\t\t\t\"INSERT INTO msgs (parent, role, content, cancelled, created_at)\",\n\t\t\tmsgsRows,\n\t\t\t50,\n\t\t);\n\n\t\tconst toolRefsRows: unknown[][] = [];\n\t\tfor (let i = 0; i < SQL_RUSH_TOOL_REFS_COUNT; i += 1) {\n\t\t\ttoolRefsRows.push([\n\t\t\t\ti + 1,\n\t\t\t\t`tool_${i % 20}`,\n\t\t\t\t`call_${i}`,\n\t\t\t\ti % 5 === 0 ? \"pending\" : \"done\",\n\t\t\t]);\n\t\t}\n\t\tawait batchInsert(\n\t\t\tdatabase,\n\t\t\t\"INSERT INTO tool_refs (msg_id, tool_name, tool_call_id, status)\",\n\t\t\ttoolRefsRows,\n\t\t\t100,\n\t\t);\n\n\t\tconst eventsRows: unknown[][] = [];\n\t\tfor (let i = 0; i < SQL_RUSH_EVENTS_COUNT; i += 1) {\n\t\t\teventsRows.push([\n\t\t\t\ti + 1,\n\t\t\t\t`event_${i % 8}`,\n\t\t\t\tpayload(\"event:\", 256),\n\t\t\t\tnow - (SQL_RUSH_EVENTS_COUNT - i) * 100,\n\t\t\t]);\n\t\t}\n\t\tawait batchInsert(\n\t\t\tdatabase,\n\t\t\t\"INSERT INTO events (seq, event_type, payload, created_at)\",\n\t\t\teventsRows,\n\t\t\t100,\n\t\t);\n\n\t\tconst kvRows: unknown[][] = [];\n\t\tfor (let i = 0; i < SQL_RUSH_KV_COUNT; i += 1) {\n\t\t\tkvRows.push([`kv_${i}`, payload(\"kv:\", 128), now]);\n\t\t}\n\t\tawait batchInsert(database, \"INSERT INTO kv (key, value, updated_at)\", kvRows, 40);\n\n\t\tconst toolsRows: unknown[][] = [];\n\t\tfor (let i = 0; i < SQL_RUSH_TOOLS_COUNT; i += 1) {\n\t\t\ttoolsRows.push([\"exec-1\", `tool_${i}`, payload(\"tool:\", 1024), now]);\n\t\t}\n\t\tawait batchInsert(\n\t\t\tdatabase,\n\t\t\t\"INSERT INTO tools (executor_id, name, spec, updated_at)\",\n\t\t\ttoolsRows,\n\t\t\t41,\n\t\t);\n\n\t\tconst metaRows: unknown[][] = [];\n\t\tfor (let i = 0; i < SQL_RUSH_META_COUNT; i += 1) {\n\t\t\tmetaRows.push([`key_${i}`, payload(\"meta:\", 64)]);\n\t\t}\n\t\tawait batchInsert(database, \"INSERT INTO meta (key, value)\", metaRows, 12);\n\t});\n\n\treturn {\n\t\trows:\n\t\t\tSQL_RUSH_MSGS_COUNT +\n\t\t\tSQL_RUSH_TOOL_REFS_COUNT +\n\t\t\tSQL_RUSH_EVENTS_COUNT +\n\t\t\tSQL_RUSH_KV_COUNT +\n\t\t\tSQL_RUSH_TOOLS_COUNT +\n\t\t\tSQL_RUSH_META_COUNT,\n\t\ttargetBytes,\n\t\trowBytes: 0,\n\t\tsetupMs: performance.now() - startedAt,\n\t\tpageCount: await queryPageCount(database),\n\t};\n}\n\nasync function seedMigrationSource(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\ttargetBytes: number,\n\trowBytes: number,\n\tskewed = false,\n) {\n\tawait resetMigration(database);\n\tawait database.execute(`CREATE TABLE rw_migration_source (\n\t\tid INTEGER PRIMARY KEY,\n\t\taccount_id TEXT NOT NULL,\n\t\tstatus TEXT NOT NULL,\n\t\tcreated_at INTEGER NOT NULL,\n\t\ttotal_cents INTEGER NOT NULL,\n\t\tbody TEXT NOT NULL\n\t)`);\n\n\tconst rows = Math.max(1, Math.ceil(targetBytes / rowBytes));\n\tconst startedAt = performance.now();\n\n\tfor (let txStart = 0; txStart < rows; txStart += SETUP_TRANSACTION_ROWS) {\n\t\tconst txEnd = Math.min(rows, txStart + SETUP_TRANSACTION_ROWS);\n\t\tawait withTransaction(database, async () => {\n\t\t\tfor (let offset = txStart; offset < txEnd; offset += ORDER_BATCH_ROWS) {\n\t\t\t\tconst placeholders: string[] = [];\n\t\t\t\tconst args: unknown[] = [];\n\t\t\t\tconst batchEnd = Math.min(txEnd, offset + ORDER_BATCH_ROWS);\n\t\t\t\tfor (let i = offset; i < batchEnd; i += 1) {\n\t\t\t\t\tconst accountId = skewed\n\t\t\t\t\t\t? `acct-${i % 10 === 0 ? i % 512 : i % 8}`\n\t\t\t\t\t\t: `acct-${pseudoRandom(i) % 512}`;\n\t\t\t\t\tconst status = skewed\n\t\t\t\t\t\t? i % 20 === 0\n\t\t\t\t\t\t\t? \"failed\"\n\t\t\t\t\t\t\t: \"open\"\n\t\t\t\t\t\t: [\"open\", \"closed\", \"failed\", \"pending\"][i % 4];\n\t\t\t\t\tplaceholders.push(\"(?, ?, ?, ?, ?, ?)\");\n\t\t\t\t\targs.push(\n\t\t\t\t\t\ti + 1,\n\t\t\t\t\t\taccountId,\n\t\t\t\t\t\tstatus,\n\t\t\t\t\t\t1_700_000_000_000 + i * 1000,\n\t\t\t\t\t\t100 + (pseudoRandom(i + 41) % 50_000),\n\t\t\t\t\t\tpayload(`migration-${i}:`, rowBytes),\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`INSERT INTO rw_migration_source (id, account_id, status, created_at, total_cents, body) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t\t...args,\n\t\t\t\t);\n\t\t\t}\n\t\t});\n\t}\n\n\treturn {\n\t\trows,\n\t\ttargetBytes,\n\t\trowBytes,\n\t\tsetupMs: performance.now() - startedAt,\n\t\tpageCount: await queryPageCount(database),\n\t};\n}\n\nasync function readRowidRange(\n\tdatabase: {\n\t\texecute: (sql: string, ...args: unknown[]) => Promise;\n\t},\n\tdirection: \"forward\" | \"backward\",\n) {\n\tconst [count] = typedRows(\n\t\tawait database.execute(\"SELECT COUNT(*) AS rows FROM rw_orders\"),\n\t);\n\tconst rows = count?.rows ?? 0;\n\tlet bytes = 0;\n\tlet scannedRows = 0;\n\n\tif (direction === \"backward\") {\n\t\tfor (let upper = rows; upper > 0; upper -= RANGE_CHUNK_ROWS) {\n\t\t\tconst lower = Math.max(1, upper - RANGE_CHUNK_ROWS + 1);\n\t\t\tconst chunk = typedRows(\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`SELECT length(note) AS bytes FROM rw_orders WHERE id BETWEEN ? AND ? ORDER BY id DESC`,\n\t\t\t\t\tlower,\n\t\t\t\t\tupper,\n\t\t\t\t),\n\t\t\t);\n\t\t\tfor (const row of chunk) {\n\t\t\t\tbytes += row.bytes;\n\t\t\t\tscannedRows += 1;\n\t\t\t}\n\t\t}\n\t\treturn { rows: scannedRows, bytes };\n\t}\n\n\tfor (let lower = 1; lower <= rows; lower += RANGE_CHUNK_ROWS) {\n\t\tconst upper = lower + RANGE_CHUNK_ROWS - 1;\n\t\tconst [chunk] = typedRows<{ rows: number; bytes: number }>(\n\t\t\tawait database.execute(\n\t\t\t\t`SELECT COUNT(*) AS rows, COALESCE(SUM(length(note)), 0) AS bytes FROM rw_orders WHERE id BETWEEN ? AND ?`,\n\t\t\t\tlower,\n\t\t\t\tupper,\n\t\t\t),\n\t\t);\n\t\tbytes += chunk?.bytes ?? 0;\n\t\tscannedRows += chunk?.rows ?? 0;\n\t}\n\n\treturn { rows: scannedRows, bytes };\n}\n\nexport const sqliteRealworldBench = actor({\n\toptions: {\n\t\tactionTimeout: 1_200_000,\n\t\tsleepGracePeriod: 30_000,\n\t},\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS rw_customers (\n\t\t\t\tid INTEGER PRIMARY KEY,\n\t\t\t\taccount_id TEXT NOT NULL,\n\t\t\t\temail TEXT NOT NULL,\n\t\t\t\tplan TEXT NOT NULL,\n\t\t\t\tregion TEXT NOT NULL\n\t\t\t)`);\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS rw_orders (\n\t\t\t\tid INTEGER PRIMARY KEY,\n\t\t\t\tcustomer_id INTEGER NOT NULL,\n\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\tstatus TEXT NOT NULL,\n\t\t\t\ttotal_cents INTEGER NOT NULL,\n\t\t\t\tshard INTEGER NOT NULL,\n\t\t\t\tnote TEXT NOT NULL\n\t\t\t)`);\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS rw_order_items (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\torder_id INTEGER NOT NULL,\n\t\t\t\tsku TEXT NOT NULL,\n\t\t\t\tquantity INTEGER NOT NULL,\n\t\t\t\tprice_cents INTEGER NOT NULL,\n\t\t\t\tline_no INTEGER NOT NULL\n\t\t\t)`);\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS rw_events (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\taccount_id TEXT NOT NULL,\n\t\t\t\tevent_type TEXT NOT NULL,\n\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\tentity_key TEXT NOT NULL,\n\t\t\t\tproperties TEXT NOT NULL\n\t\t\t)`);\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS rw_docs (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\texternal_key TEXT NOT NULL UNIQUE,\n\t\t\t\trow_rank INTEGER NOT NULL,\n\t\t\t\ttenant_id TEXT NOT NULL,\n\t\t\t\tbody TEXT NOT NULL,\n\t\t\t\tbody_bytes INTEGER NOT NULL\n\t\t\t)`);\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS rw_ledger (\n\t\t\t\taccount_id TEXT NOT NULL,\n\t\t\t\tentry_id INTEGER NOT NULL,\n\t\t\t\tamount_cents INTEGER NOT NULL,\n\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\tmemo TEXT NOT NULL,\n\t\t\t\tPRIMARY KEY (account_id, entry_id)\n\t\t\t) WITHOUT ROWID`);\n\t\t\tawait database.execute(`CREATE TABLE IF NOT EXISTS rw_chat_log (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tthread_id TEXT NOT NULL,\n\t\t\t\tseq INTEGER NOT NULL,\n\t\t\t\trole TEXT NOT NULL,\n\t\t\t\tcontent TEXT NOT NULL,\n\t\t\t\tcontent_bytes INTEGER NOT NULL,\n\t\t\t\ttoken_estimate INTEGER NOT NULL,\n\t\t\t\tcreated_at INTEGER NOT NULL DEFAULT 0\n\t\t\t)`);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE TABLE IF NOT EXISTS msgs (id INTEGER PRIMARY KEY AUTOINCREMENT, parent INTEGER, role TEXT NOT NULL, content TEXT NOT NULL, cancelled INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE TABLE IF NOT EXISTS tool_refs (id INTEGER PRIMARY KEY AUTOINCREMENT, msg_id INTEGER NOT NULL, tool_name TEXT NOT NULL, tool_call_id TEXT NOT NULL, status TEXT NOT NULL)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE TABLE IF NOT EXISTS events (seq INTEGER PRIMARY KEY, event_type TEXT NOT NULL, payload TEXT NOT NULL, created_at INTEGER NOT NULL)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE TABLE IF NOT EXISTS kv (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at INTEGER NOT NULL)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE TABLE IF NOT EXISTS tools (id INTEGER PRIMARY KEY AUTOINCREMENT, executor_id TEXT NOT NULL, name TEXT NOT NULL, spec TEXT NOT NULL, updated_at INTEGER NOT NULL)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_rw_orders_customer_created ON rw_orders(customer_id, created_at DESC)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_rw_orders_status_created ON rw_orders(status, created_at)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_rw_orders_created ON rw_orders(created_at DESC)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_rw_order_items_order ON rw_order_items(order_id)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_rw_events_account_created ON rw_events(account_id, created_at)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_rw_docs_external_rank ON rw_docs(external_key, row_rank)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_rw_docs_tenant_rank ON rw_docs(tenant_id, row_rank)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_rw_chat_log_thread_seq ON rw_chat_log(thread_id, seq DESC)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_msgs_parent_role_cancelled_created_at ON msgs (parent, role, cancelled, created_at)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_tool_refs_msg_id ON tool_refs (msg_id)\",\n\t\t\t);\n\t\t},\n\t}),\n\tactions: {\n\t\tinspectCacheConfig: async (c) => {\n\t\t\tconst [cacheSize] = typedRows(\n\t\t\t\tawait c.db.execute(\"PRAGMA cache_size\"),\n\t\t\t);\n\t\t\tconst [pageSize] = typedRows(\n\t\t\t\tawait c.db.execute(\"PRAGMA page_size\"),\n\t\t\t);\n\t\t\treturn {\n\t\t\t\tsqliteCacheSizePragma: cacheSize?.cache_size ?? null,\n\t\t\t\tsqlitePageSize: pageSize?.page_size ?? null,\n\t\t\t\tpageCount: await queryPageCount(c.db),\n\t\t\t};\n\t\t},\n\n\t\tsetupWorkload: async (c, input: SetupInput) => {\n\t\t\tassertWorkload(input.workload);\n\t\t\tconst rowBytes = positiveInteger(input.rowBytes, DEFAULT_ROW_BYTES, \"rowBytes\");\n\t\t\tif (input.workload === \"migration-ddl-small\") {\n\t\t\t\tawait resetMigration(c.db);\n\t\t\t\treturn {\n\t\t\t\t\trows: 0,\n\t\t\t\t\ttargetBytes: 0,\n\t\t\t\t\trowBytes,\n\t\t\t\t\tsetupMs: 0,\n\t\t\t\t\tpageCount: await queryPageCount(c.db),\n\t\t\t\t};\n\t\t\t}\n\t\t\tconst targetBytes = positiveInteger(\n\t\t\t\tinput.targetBytes,\n\t\t\t\t8 * 1024 * 1024,\n\t\t\t\t\"targetBytes\",\n\t\t\t);\n\n\t\t\tswitch (input.workload) {\n\t\t\t\tcase \"small-rowid-point\":\n\t\t\t\tcase \"small-schema-read\":\n\t\t\t\tcase \"small-range-scan\":\n\t\t\t\tcase \"rowid-range-forward\":\n\t\t\t\tcase \"rowid-range-backward\":\n\t\t\t\tcase \"aggregate-status\":\n\t\t\t\tcase \"aggregate-time-bucket\":\n\t\t\t\tcase \"aggregate-tenant-time-range\":\n\t\t\t\tcase \"parallel-read-aggregates\":\n\t\t\t\tcase \"parallel-read-write-transition\":\n\t\t\t\tcase \"feed-order-by-limit\":\n\t\t\t\tcase \"feed-pagination-adjacent\":\n\t\t\t\tcase \"join-order-items\":\n\t\t\t\tcase \"random-point-lookups\":\n\t\t\t\tcase \"write-batch-after-wake\":\n\t\t\t\tcase \"update-hot-partition\":\n\t\t\t\tcase \"delete-churn-range-read\":\n\t\t\t\t\treturn seedCommerce(c.db, targetBytes, rowBytes);\n\t\t\t\tcase \"secondary-index-covering-range\":\n\t\t\t\tcase \"secondary-index-scattered-table\":\n\t\t\t\tcase \"hot-index-cold-table\":\n\t\t\t\t\treturn seedDocs(c.db, targetBytes, rowBytes);\n\t\t\t\tcase \"ledger-without-rowid-range\":\n\t\t\t\t\treturn seedLedger(c.db, targetBytes, rowBytes);\n\t\t\t\tcase \"chat-log-select-limit\":\n\t\t\t\tcase \"chat-log-select-indexed\":\n\t\t\t\tcase \"chat-log-count\":\n\t\t\t\tcase \"chat-log-sum\":\n\t\t\t\tcase \"chat-tool-read-fanout\":\n\t\t\t\t\treturn seedChatLog(c.db, targetBytes);\n\t\t\t\tcase \"chat-tool-script\":\n\t\t\t\t\treturn seedSqlRush(c.db, targetBytes);\n\t\t\t\tcase \"migration-create-indexes-large\":\n\t\t\t\t\treturn seedMigrationSource(c.db, targetBytes, rowBytes);\n\t\t\t\tcase \"migration-create-indexes-skewed-large\":\n\t\t\t\t\treturn seedMigrationSource(c.db, targetBytes, rowBytes, true);\n\t\t\t\tcase \"migration-table-rebuild-large\":\n\t\t\t\tcase \"migration-add-column-large\":\n\t\t\t\t\treturn seedMigrationSource(c.db, targetBytes, rowBytes);\n\t\t\t}\n\t\t},\n\n\t\trunWorkload: async (c, input: RunInput) => {\n\t\t\tassertWorkload(input.workload);\n\t\t\tconst t0 = performance.now();\n\t\t\tlet details: Record;\n\n\t\t\tswitch (input.workload) {\n\t\t\t\tcase \"small-rowid-point\": {\n\t\t\t\t\tlet bytes = 0;\n\t\t\t\t\tfor (let i = 0; i < 50; i += 1) {\n\t\t\t\t\t\tconst id = (i % 16) + 1;\n\t\t\t\t\t\tconst [row] = typedRows(\n\t\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\t\"SELECT length(note) AS bytes FROM rw_orders WHERE id = ?\",\n\t\t\t\t\t\t\t\tid,\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t);\n\t\t\t\t\t\tbytes += row?.bytes ?? 0;\n\t\t\t\t\t}\n\t\t\t\t\tdetails = { ops: 50, bytes };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"small-schema-read\": {\n\t\t\t\t\tconst tables = await c.db.execute(\n\t\t\t\t\t\t\"SELECT name, type FROM sqlite_master WHERE type IN ('table', 'index') ORDER BY name\",\n\t\t\t\t\t);\n\t\t\t\t\tconst columns = await c.db.execute(\"PRAGMA table_info(rw_orders)\");\n\t\t\t\t\tconst [count] = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\"SELECT COUNT(*) AS rows FROM rw_orders\"),\n\t\t\t\t\t);\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tobjects: tables.length,\n\t\t\t\t\t\tcolumns: columns.length,\n\t\t\t\t\t\trows: count?.rows ?? 0,\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"small-range-scan\":\n\t\t\t\tcase \"rowid-range-forward\": {\n\t\t\t\t\tdetails = await readRowidRange(c.db, \"forward\");\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"rowid-range-backward\": {\n\t\t\t\t\tdetails = await readRowidRange(c.db, \"backward\");\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"secondary-index-covering-range\": {\n\t\t\t\t\tconst rows = typedRows<{ external_key: string; row_rank: number }>(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT external_key, row_rank FROM rw_docs\n\t\t\t\t\t\tWHERE external_key BETWEEN 'doc-00000000' AND 'doc-ffffffff'\n\t\t\t\t\t\tORDER BY external_key`,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tlet checksum = 0;\n\t\t\t\t\tfor (const row of rows) checksum = (checksum + row.row_rank) >>> 0;\n\t\t\t\t\tdetails = { rows: rows.length, checksum };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"secondary-index-scattered-table\": {\n\t\t\t\t\tconst rows = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT body_bytes AS bytes FROM rw_docs\n\t\t\t\t\t\tWHERE external_key BETWEEN 'doc-00000000' AND 'doc-ffffffff'\n\t\t\t\t\t\tORDER BY external_key`,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tlet bytes = 0;\n\t\t\t\t\tfor (const row of rows) bytes += row.bytes;\n\t\t\t\t\tdetails = { rows: rows.length, bytes };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"aggregate-status\": {\n\t\t\t\t\tconst rows = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT status, COUNT(*) AS rows, SUM(total_cents) AS total\n\t\t\t\t\t\tFROM rw_orders\n\t\t\t\t\t\tGROUP BY status\n\t\t\t\t\t\tORDER BY status`,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tgroups: rows.length,\n\t\t\t\t\t\trows: rows.reduce((sum, row) => sum + row.rows, 0),\n\t\t\t\t\t\ttotal: rows.reduce((sum, row) => sum + row.total, 0),\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"aggregate-time-bucket\": {\n\t\t\t\t\tconst rows = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT (created_at / 300000) AS bucket, COUNT(*) AS rows, SUM(total_cents) AS total\n\t\t\t\t\t\tFROM rw_orders\n\t\t\t\t\t\tGROUP BY bucket\n\t\t\t\t\t\tORDER BY bucket`,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tbuckets: rows.length,\n\t\t\t\t\t\trows: rows.reduce((sum, row) => sum + row.rows, 0),\n\t\t\t\t\t\ttotal: rows.reduce((sum, row) => sum + row.total, 0),\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"aggregate-tenant-time-range\": {\n\t\t\t\t\tconst rows = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT e.event_type, COUNT(*) AS rows, SUM(o.total_cents) AS total\n\t\t\t\t\t\tFROM rw_events e\n\t\t\t\t\t\tJOIN rw_orders o ON o.id = CAST(substr(e.entity_key, 7) AS INTEGER)\n\t\t\t\t\t\tWHERE e.account_id = ? AND e.created_at BETWEEN ? AND ?\n\t\t\t\t\t\tGROUP BY e.event_type\n\t\t\t\t\t\tORDER BY e.event_type`,\n\t\t\t\t\t\t\t\"acct-7\",\n\t\t\t\t\t\t\t1_700_000_000_000,\n\t\t\t\t\t\t\t1_700_000_000_000 + 86_400_000,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tgroups: rows.length,\n\t\t\t\t\t\trows: rows.reduce((sum, row) => sum + row.rows, 0),\n\t\t\t\t\t\ttotal: rows.reduce((sum, row) => sum + row.total, 0),\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"parallel-read-aggregates\": {\n\t\t\t\t\tconst [\n\t\t\t\t\t\tstatusRows,\n\t\t\t\t\t\tbucketRows,\n\t\t\t\t\t\ttenantRows,\n\t\t\t\t\t\tjoinRows,\n\t\t\t\t\t] = await Promise.all([\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t`SELECT status, COUNT(*) AS rows, SUM(total_cents) AS total\n\t\t\t\t\t\tFROM rw_orders\n\t\t\t\t\t\tGROUP BY status\n\t\t\t\t\t\tORDER BY status`,\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t`SELECT (created_at / 300000) AS bucket, COUNT(*) AS rows, SUM(total_cents) AS total\n\t\t\t\t\t\tFROM rw_orders\n\t\t\t\t\t\tGROUP BY bucket\n\t\t\t\t\t\tORDER BY bucket`,\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t`SELECT e.event_type, COUNT(*) AS rows, SUM(o.total_cents) AS total\n\t\t\t\t\t\tFROM rw_events e\n\t\t\t\t\t\tJOIN rw_orders o ON o.id = CAST(substr(e.entity_key, 7) AS INTEGER)\n\t\t\t\t\t\tWHERE e.account_id = ? AND e.created_at BETWEEN ? AND ?\n\t\t\t\t\t\tGROUP BY e.event_type\n\t\t\t\t\t\tORDER BY e.event_type`,\n\t\t\t\t\t\t\t\"acct-7\",\n\t\t\t\t\t\t\t1_700_000_000_000,\n\t\t\t\t\t\t\t1_700_000_000_000 + 86_400_000,\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t`SELECT o.status, COUNT(*) AS rows, SUM(oi.quantity * oi.price_cents) AS total\n\t\t\t\t\t\tFROM rw_orders o\n\t\t\t\t\t\tJOIN rw_order_items oi ON oi.order_id = o.id\n\t\t\t\t\t\tGROUP BY o.status\n\t\t\t\t\t\tORDER BY o.status`,\n\t\t\t\t\t\t),\n\t\t\t\t\t]);\n\t\t\t\t\tconst aggregates = [\n\t\t\t\t\t\t...typedRows(statusRows),\n\t\t\t\t\t\t...typedRows(bucketRows),\n\t\t\t\t\t\t...typedRows(tenantRows),\n\t\t\t\t\t\t...typedRows(joinRows),\n\t\t\t\t\t];\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tops: 4,\n\t\t\t\t\t\tgroups: aggregates.length,\n\t\t\t\t\t\trows: aggregates.reduce((sum, row) => sum + row.rows, 0),\n\t\t\t\t\t\ttotal: aggregates.reduce((sum, row) => sum + row.total, 0),\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"parallel-read-write-transition\": {\n\t\t\t\t\tconst readStatus = c.db.execute(\n\t\t\t\t\t\t`SELECT status, COUNT(*) AS rows, SUM(total_cents) AS total\n\t\t\t\t\t\tFROM rw_orders\n\t\t\t\t\t\tGROUP BY status\n\t\t\t\t\t\tORDER BY status`,\n\t\t\t\t\t);\n\t\t\t\t\tconst readJoin = c.db.execute(\n\t\t\t\t\t\t`SELECT o.status, COUNT(*) AS rows, SUM(oi.quantity * oi.price_cents) AS total\n\t\t\t\t\t\tFROM rw_orders o\n\t\t\t\t\t\tJOIN rw_order_items oi ON oi.order_id = o.id\n\t\t\t\t\t\tGROUP BY o.status\n\t\t\t\t\t\tORDER BY o.status`,\n\t\t\t\t\t);\n\t\t\t\t\tconst writeHotShard = c.db.execute(\n\t\t\t\t\t\t\"UPDATE rw_orders SET total_cents = total_cents + 1 WHERE shard BETWEEN 0 AND 7\",\n\t\t\t\t\t);\n\t\t\t\t\tconst readAfterWrite = c.db.execute(\n\t\t\t\t\t\t\"SELECT COUNT(*) AS rows FROM rw_orders WHERE shard BETWEEN 0 AND 7\",\n\t\t\t\t\t);\n\t\t\t\t\tconst [statusRows, joinRows, , shardRows] = await Promise.all([\n\t\t\t\t\t\treadStatus,\n\t\t\t\t\t\treadJoin,\n\t\t\t\t\t\twriteHotShard,\n\t\t\t\t\t\treadAfterWrite,\n\t\t\t\t\t]);\n\t\t\t\t\tconst aggregates = [\n\t\t\t\t\t\t...typedRows(statusRows),\n\t\t\t\t\t\t...typedRows(joinRows),\n\t\t\t\t\t];\n\t\t\t\t\tconst [shardCount] = typedRows(shardRows);\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tops: 4,\n\t\t\t\t\t\treadOps: 3,\n\t\t\t\t\t\twriteOps: 1,\n\t\t\t\t\t\tgroups: aggregates.length,\n\t\t\t\t\t\trows:\n\t\t\t\t\t\t\taggregates.reduce((sum, row) => sum + row.rows, 0) +\n\t\t\t\t\t\t\t(shardCount?.rows ?? 0),\n\t\t\t\t\t\ttotal: aggregates.reduce((sum, row) => sum + row.total, 0),\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"feed-order-by-limit\": {\n\t\t\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\t\t`SELECT id, customer_id, created_at, status, total_cents\n\t\t\t\t\t\tFROM rw_orders\n\t\t\t\t\t\tWHERE created_at >= ?\n\t\t\t\t\t\tORDER BY created_at DESC\n\t\t\t\t\t\tLIMIT 1000`,\n\t\t\t\t\t\t1_700_000_000_000,\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { rows: rows.length };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"feed-pagination-adjacent\": {\n\t\t\t\t\tconst firstPage = typedRows<{ created_at: number }>(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT created_at\n\t\t\t\t\t\tFROM rw_orders\n\t\t\t\t\t\tWHERE created_at >= ?\n\t\t\t\t\t\tORDER BY created_at DESC\n\t\t\t\t\t\tLIMIT ?`,\n\t\t\t\t\t\t\t1_700_000_000_000,\n\t\t\t\t\t\t\tFEED_PAGE_ROWS,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tconst cursor = firstPage.at(-1)?.created_at ?? 1_700_000_000_000;\n\t\t\t\t\tconst secondPage = await c.db.execute(\n\t\t\t\t\t\t`SELECT id, customer_id, created_at, status, total_cents\n\t\t\t\t\t\tFROM rw_orders\n\t\t\t\t\t\tWHERE created_at < ?\n\t\t\t\t\t\tORDER BY created_at DESC\n\t\t\t\t\t\tLIMIT ?`,\n\t\t\t\t\t\tcursor,\n\t\t\t\t\t\tFEED_PAGE_ROWS,\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { firstPageRows: firstPage.length, rows: secondPage.length };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"join-order-items\": {\n\t\t\t\t\tconst rows = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT o.status, COUNT(*) AS rows, SUM(oi.quantity * oi.price_cents) AS total\n\t\t\t\t\t\tFROM rw_orders o\n\t\t\t\t\t\tJOIN rw_order_items oi ON oi.order_id = o.id\n\t\t\t\t\t\tGROUP BY o.status\n\t\t\t\t\t\tORDER BY o.status`,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tgroups: rows.length,\n\t\t\t\t\t\trows: rows.reduce((sum, row) => sum + row.rows, 0),\n\t\t\t\t\t\ttotal: rows.reduce((sum, row) => sum + row.total, 0),\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"random-point-lookups\": {\n\t\t\t\t\tconst [count] = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\"SELECT COUNT(*) AS rows FROM rw_orders\"),\n\t\t\t\t\t);\n\t\t\t\t\tconst rows = Math.max(1, count?.rows ?? 1);\n\t\t\t\t\tlet bytes = 0;\n\t\t\t\t\tfor (let i = 0; i < POINT_LOOKUP_OPS; i += 1) {\n\t\t\t\t\t\tconst id = (pseudoRandom(i) % rows) + 1;\n\t\t\t\t\t\tconst [row] = typedRows(\n\t\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\t\"SELECT length(note) AS bytes FROM rw_orders WHERE id = ?\",\n\t\t\t\t\t\t\t\tid,\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t);\n\t\t\t\t\t\tbytes += row?.bytes ?? 0;\n\t\t\t\t\t}\n\t\t\t\t\tdetails = { ops: POINT_LOOKUP_OPS, bytes };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"hot-index-cold-table\": {\n\t\t\t\t\tconst indexRows = typedRows<{ id: number }>(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT id\n\t\t\t\t\t\tFROM rw_docs\n\t\t\t\t\t\tWHERE tenant_id = ?\n\t\t\t\t\t\tORDER BY row_rank\n\t\t\t\t\t\tLIMIT 1000`,\n\t\t\t\t\t\t\t\"tenant-7\",\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tlet bytes = 0;\n\t\t\t\t\tfor (const row of indexRows) {\n\t\t\t\t\t\tconst [doc] = typedRows(\n\t\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\t\"SELECT body_bytes AS bytes FROM rw_docs WHERE id = ?\",\n\t\t\t\t\t\t\t\trow.id,\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t);\n\t\t\t\t\t\tbytes += doc?.bytes ?? 0;\n\t\t\t\t\t}\n\t\t\t\t\tdetails = { rows: indexRows.length, bytes };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"ledger-without-rowid-range\": {\n\t\t\t\t\tconst rows = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`SELECT account_id, entry_id, amount_cents, length(memo) AS bytes\n\t\t\t\t\t\tFROM rw_ledger\n\t\t\t\t\t\tWHERE account_id BETWEEN 'acct-0040' AND 'acct-0180'\n\t\t\t\t\t\tORDER BY account_id, entry_id`,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tlet bytes = 0;\n\t\t\t\t\tfor (const row of rows) bytes += row.bytes;\n\t\t\t\t\tdetails = { rows: rows.length, bytes };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"chat-log-select-limit\": {\n\t\t\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\t\t\"SELECT seq, role, substr(content, 1, 128) AS preview FROM rw_chat_log ORDER BY created_at DESC LIMIT 100\",\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { rows: rows.length };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"chat-log-select-indexed\": {\n\t\t\t\t\tconst expectedRows = Math.max(\n\t\t\t\t\t\t1,\n\t\t\t\t\t\tMath.ceil(\n\t\t\t\t\t\t\tpositiveInteger(input.targetBytes, CHAT_LOG_CHUNK_BYTES, \"targetBytes\") /\n\t\t\t\t\t\t\t\tCHAT_LOG_CHUNK_BYTES,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tconst lowerBound = Math.max(0, expectedRows - 100);\n\t\t\t\t\tconst rows = await c.db.execute(\n\t\t\t\t\t\t\"SELECT seq, role, content_bytes FROM rw_chat_log WHERE thread_id = ? AND seq >= ? ORDER BY seq DESC LIMIT 100\",\n\t\t\t\t\t\tCHAT_THREAD_ID,\n\t\t\t\t\t\tlowerBound,\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { rows: rows.length };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"chat-log-count\": {\n\t\t\t\t\tconst [row] = typedRows<{ count: number }>(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\"SELECT COUNT(*) AS count FROM rw_chat_log WHERE thread_id = ?\",\n\t\t\t\t\t\t\tCHAT_THREAD_ID,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { ops: 1, rows: row?.count ?? 0 };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"chat-log-sum\": {\n\t\t\t\t\tconst [row] = typedRows<{ total_bytes: number | null }>(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\"SELECT SUM(content_bytes) AS total_bytes FROM rw_chat_log WHERE thread_id = ?\",\n\t\t\t\t\t\t\tCHAT_THREAD_ID,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { ops: 1, bytes: row?.total_bytes ?? 0 };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"chat-tool-read-fanout\": {\n\t\t\t\t\tconst expectedRows = Math.max(\n\t\t\t\t\t\t1,\n\t\t\t\t\t\tMath.ceil(\n\t\t\t\t\t\t\tpositiveInteger(input.targetBytes, CHAT_LOG_CHUNK_BYTES, \"targetBytes\") /\n\t\t\t\t\t\t\t\tCHAT_LOG_CHUNK_BYTES,\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tconst lowerBound = Math.max(0, expectedRows - 100);\n\t\t\t\t\tconst [limitRows, indexedRows, countRows, sumRows] = await Promise.all([\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT seq, role, substr(content, 1, 128) AS preview FROM rw_chat_log ORDER BY created_at DESC LIMIT 100\",\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT seq, role, content_bytes FROM rw_chat_log WHERE thread_id = ? AND seq >= ? ORDER BY seq DESC LIMIT 100\",\n\t\t\t\t\t\t\tCHAT_THREAD_ID,\n\t\t\t\t\t\t\tlowerBound,\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT COUNT(*) AS count FROM rw_chat_log WHERE thread_id = ?\",\n\t\t\t\t\t\t\tCHAT_THREAD_ID,\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT SUM(content_bytes) AS total_bytes FROM rw_chat_log WHERE thread_id = ?\",\n\t\t\t\t\t\t\tCHAT_THREAD_ID,\n\t\t\t\t\t\t),\n\t\t\t\t\t]);\n\t\t\t\t\tconst [countRow] = typedRows<{ count: number }>(countRows);\n\t\t\t\t\tconst [sumRow] = typedRows<{ total_bytes: number | null }>(sumRows);\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tops: 4,\n\t\t\t\t\t\tlimitRows: limitRows.length,\n\t\t\t\t\t\tindexedRows: indexedRows.length,\n\t\t\t\t\t\trows: countRow?.count ?? 0,\n\t\t\t\t\t\tbytes: sumRow?.total_bytes ?? 0,\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"chat-tool-script\": {\n\t\t\t\t\tconst [\n\t\t\t\t\t\tmsgsRows,\n\t\t\t\t\t\ttoolRefsRows,\n\t\t\t\t\t\teventsRows,\n\t\t\t\t\t\tkvRows,\n\t\t\t\t\t\ttoolsRows,\n\t\t\t\t\t\tmetaRows,\n\t\t\t\t\t\tunresolvedRows,\n\t\t\t\t\t] = await Promise.all([\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT id, role, length(content) AS bytes FROM msgs WHERE parent IS NOT NULL AND role = ? AND cancelled = 0 ORDER BY created_at DESC LIMIT 50\",\n\t\t\t\t\t\t\t\"assistant\",\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT id, tool_name, status FROM tool_refs WHERE status = ? ORDER BY id DESC LIMIT 50\",\n\t\t\t\t\t\t\t\"pending\",\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT seq, event_type, length(payload) AS bytes FROM events WHERE seq > ? ORDER BY seq ASC LIMIT 100\",\n\t\t\t\t\t\t\t600,\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT key, length(value) AS bytes FROM kv ORDER BY updated_at DESC LIMIT 20\",\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\n\t\t\t\t\t\t\t\"SELECT id, name, length(spec) AS bytes FROM tools WHERE executor_id = ? ORDER BY updated_at DESC\",\n\t\t\t\t\t\t\t\"exec-1\",\n\t\t\t\t\t\t),\n\t\t\t\t\t\tc.db.execute(\"SELECT key, length(value) AS bytes FROM meta\"),\n\t\t\t\t\t\tc.db.execute(`SELECT m.id, m.role, count(tr.id) AS pending_refs\n\t\t\t\t\t\t\tFROM msgs m\n\t\t\t\t\t\t\tLEFT JOIN tool_refs tr ON tr.msg_id = m.id AND tr.status = 'pending'\n\t\t\t\t\t\t\tWHERE m.role = 'assistant' AND m.cancelled = 0\n\t\t\t\t\t\t\tGROUP BY m.id\n\t\t\t\t\t\t\tORDER BY m.created_at DESC\n\t\t\t\t\t\t\tLIMIT 100`),\n\t\t\t\t\t]);\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\tops: 7,\n\t\t\t\t\t\tmsgsRows: msgsRows.length,\n\t\t\t\t\t\ttoolRefsRows: toolRefsRows.length,\n\t\t\t\t\t\teventsRows: eventsRows.length,\n\t\t\t\t\t\tkvRows: kvRows.length,\n\t\t\t\t\t\ttoolsRows: toolsRows.length,\n\t\t\t\t\t\tmetaRows: metaRows.length,\n\t\t\t\t\t\tunresolvedRows: unresolvedRows.length,\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"write-batch-after-wake\": {\n\t\t\t\t\tconst [count] = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\"SELECT COUNT(*) AS rows FROM rw_orders\"),\n\t\t\t\t\t);\n\t\t\t\t\tconst startId = (count?.rows ?? 0) + 1;\n\t\t\t\t\tawait c.db.execute(\"BEGIN\");\n\t\t\t\t\tfor (let offset = 0; offset < 1000; offset += ORDER_BATCH_ROWS) {\n\t\t\t\t\t\tconst placeholders: string[] = [];\n\t\t\t\t\t\tconst args: unknown[] = [];\n\t\t\t\t\t\tfor (let i = offset; i < offset + ORDER_BATCH_ROWS; i += 1) {\n\t\t\t\t\t\t\tconst id = startId + i;\n\t\t\t\t\t\t\tplaceholders.push(\"(?, ?, ?, ?, ?, ?, ?)\");\n\t\t\t\t\t\t\targs.push(\n\t\t\t\t\t\t\t\tid,\n\t\t\t\t\t\t\t\t(i % 128) + 1,\n\t\t\t\t\t\t\t\t1_800_000_000_000 + i,\n\t\t\t\t\t\t\t\t\"pending\",\n\t\t\t\t\t\t\t\t1000 + i,\n\t\t\t\t\t\t\t\ti % 128,\n\t\t\t\t\t\t\t\tpayload(`wake-insert-${id}:`, DEFAULT_ROW_BYTES),\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t}\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t`INSERT INTO rw_orders (id, customer_id, created_at, status, total_cents, shard, note) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t\t\t\t...args,\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\t\t\t\t\tawait c.db.execute(\"COMMIT\");\n\t\t\t\t\tdetails = { rows: 1000 };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"update-hot-partition\": {\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"UPDATE rw_orders SET total_cents = total_cents + 1 WHERE shard BETWEEN 0 AND 15\",\n\t\t\t\t\t);\n\t\t\t\t\tconst [count] = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\"SELECT COUNT(*) AS rows FROM rw_orders WHERE shard BETWEEN 0 AND 15\",\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { rows: count?.rows ?? 0 };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"delete-churn-range-read\": {\n\t\t\t\t\tawait c.db.execute(\"DELETE FROM rw_orders WHERE shard BETWEEN 0 AND 15\");\n\t\t\t\t\tconst result = await readRowidRange(c.db, \"forward\");\n\t\t\t\t\tdetails = {\n\t\t\t\t\t\t...result,\n\t\t\t\t\t\tdeletedShardCount: 16,\n\t\t\t\t\t};\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"migration-create-indexes-large\": {\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"CREATE INDEX idx_rw_migration_source_account ON rw_migration_source(account_id)\",\n\t\t\t\t\t);\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"CREATE INDEX idx_rw_migration_source_created ON rw_migration_source(created_at)\",\n\t\t\t\t\t);\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"CREATE INDEX idx_rw_migration_source_status_total ON rw_migration_source(status, total_cents)\",\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { indexes: 3 };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"migration-create-indexes-skewed-large\": {\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"CREATE INDEX idx_rw_migration_source_skew_account ON rw_migration_source(account_id, created_at)\",\n\t\t\t\t\t);\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"CREATE INDEX idx_rw_migration_source_skew_status ON rw_migration_source(status, total_cents)\",\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { indexes: 2, skewed: true };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"migration-table-rebuild-large\": {\n\t\t\t\t\tawait c.db.execute(`CREATE TABLE rw_migration_source_rebuilt (\n\t\t\t\t\t\tid INTEGER PRIMARY KEY,\n\t\t\t\t\t\taccount_id TEXT NOT NULL,\n\t\t\t\t\t\tstatus TEXT NOT NULL,\n\t\t\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\t\t\ttotal_cents INTEGER NOT NULL,\n\t\t\t\t\t\tbody TEXT NOT NULL,\n\t\t\t\t\t\tarchived_at INTEGER\n\t\t\t\t\t)`);\n\t\t\t\t\tawait c.db.execute(`INSERT INTO rw_migration_source_rebuilt (\n\t\t\t\t\t\tid, account_id, status, created_at, total_cents, body, archived_at\n\t\t\t\t\t)\n\t\t\t\t\tSELECT id, account_id, status, created_at, total_cents, body, NULL\n\t\t\t\t\tFROM rw_migration_source`);\n\t\t\t\t\tawait c.db.execute(\"DROP TABLE rw_migration_source\");\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"ALTER TABLE rw_migration_source_rebuilt RENAME TO rw_migration_source\",\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { rebuilt: true };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"migration-add-column-large\": {\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"ALTER TABLE rw_migration_source ADD COLUMN archived_at INTEGER\",\n\t\t\t\t\t);\n\t\t\t\t\tdetails = { alters: 1, rewritesRows: false };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase \"migration-ddl-small\": {\n\t\t\t\t\tawait c.db.execute(`CREATE TABLE rw_migration_empty (\n\t\t\t\t\t\tid INTEGER PRIMARY KEY,\n\t\t\t\t\t\ttenant_id TEXT NOT NULL,\n\t\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t\t)`);\n\t\t\t\t\tawait c.db.execute(\"ALTER TABLE rw_migration_empty ADD COLUMN status TEXT\");\n\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\"CREATE INDEX idx_rw_migration_empty_tenant_created ON rw_migration_empty(tenant_id, created_at)\",\n\t\t\t\t\t);\n\t\t\t\t\tawait c.db.execute(`CREATE TABLE rw_migration_audit (\n\t\t\t\t\t\tid INTEGER PRIMARY KEY,\n\t\t\t\t\t\tmigration_name TEXT NOT NULL,\n\t\t\t\t\t\tapplied_at INTEGER NOT NULL\n\t\t\t\t\t)`);\n\t\t\t\t\tdetails = { tables: 2, indexes: 1, alters: 1 };\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tconst ms = performance.now() - t0;\n\t\t\treturn {\n\t\t\t\tms,\n\t\t\t\tworkload: input.workload,\n\t\t\t\t...details,\n\t\t\t\tpageCount: await queryPageCount(c.db),\n\t\t\t};\n\t\t},\n\n\t\tgoToSleep: (c) => {\n\t\t\tc.sleep();\n\t\t\treturn { ok: true };\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\ntype WorkloadMode =\n\t| \"balanced\"\n\t| \"hot\"\n\t| \"transactions\"\n\t| \"payloads\"\n\t| \"edge\"\n\t| \"fragmentation\"\n\t| \"schema\"\n\t| \"index\"\n\t| \"relational\"\n\t| \"constraints\"\n\t| \"savepoints\"\n\t| \"pragma\"\n\t| \"prepared\"\n\t| \"growth\"\n\t| \"readwrite\"\n\t| \"truncate\"\n\t| \"boundary-keys\"\n\t| \"shadow\"\n\t| \"actual-nul\"\n\t| \"nasty-script\"\n\t| \"nasty\"\n\t| \"kitchen-sink\";\n\ninterface RunPhaseInput {\n\tseed: string;\n\tphase: number;\n\titerations: number;\n\tmode?: WorkloadMode;\n\tmaxPayloadBytes?: number;\n\tgrowthTargetBytes?: number;\n\tkeySpace?: number;\n}\n\ninterface ValidationSummary {\n\ttotalEvents: number;\n\tactiveRows: number;\n\texpectedRows: number;\n\tmissingRows: number;\n\textraRows: number;\n\tmismatchedRows: number;\n\tduplicateKeys: number;\n\tactualVersionSum: number;\n\texpectedVersionSum: number;\n\tactualPayloadChecksumSum: number;\n\texpectedPayloadChecksumSum: number;\n\taccountCount: number;\n\taccountBalanceSum: number;\n\texpectedAccountBalanceSum: number;\n\taccountBalanceMismatch: number;\n\tintegrityCheck: string;\n\tquickCheck: string;\n\tedgeRows: number;\n\tedgeExpectedRows: number;\n\tedgeMismatches: number;\n\tindexRows: number;\n\tindexMismatches: number;\n\trelationalOrders: number;\n\trelationalMismatches: number;\n\tconstraintAttempts: number;\n\tconstraintLeaks: number;\n\tsavepointRows: number;\n\tsavepointMismatches: number;\n\tidempotentOps: number;\n\tidempotentMismatches: number;\n\tschemaObjects: number;\n\tschemaMissingObjects: number;\n\tprobeRows: number;\n\tprobeMismatches: number;\n\tpreparedRows: number;\n\tpreparedMismatches: number;\n\tshadowRows: number;\n\tshadowMismatches: number;\n}\n\ninterface ItemMismatchDebugRow {\n\titemKey: string;\n\tactualValue: string | null;\n\texpectedValue: string | null;\n\tactualVersion: number | null;\n\texpectedVersion: number | null;\n\tactualUpdateCount: number | null;\n\texpectedUpdateCount: number | null;\n\tactualPayloadChecksum: number | null;\n\texpectedPayloadChecksum: number | null;\n\tactualPayloadBytes: number | null;\n\texpectedPayloadBytes: number | null;\n}\n\ninterface ItemEventDebugRow {\n\tseq: number;\n\tphase: number;\n\tlocalIndex: number;\n\tkind: string;\n\tpresent: number;\n\tvalue: string | null;\n\tversion: number;\n\tupdateCount: number;\n\tpayloadChecksum: number;\n\tpayloadBytes: number;\n\tapplied: number;\n}\n\ninterface PhaseResult {\n\tseed: string;\n\tphase: number;\n\tmode: WorkloadMode;\n\titerations: number;\n\tops: Record;\n\tvalidation: ValidationSummary;\n}\n\ninterface ItemRow {\n\titem_key: string;\n\tvalue: string;\n\tversion: number;\n\tupdate_count: number;\n\tpayload?: string;\n\tpayload_checksum: number;\n\tpayload_bytes: number;\n}\n\nconst ACCOUNT_COUNT = 8;\nconst ACCOUNT_INITIAL_BALANCE = 100_000;\nconst DEFAULT_KEY_SPACE = 64;\nconst DEFAULT_MAX_PAYLOAD_BYTES = 8 * 1024;\nconst DEFAULT_GROWTH_TARGET_BYTES = 1024 * 1024;\nconst LARGE_WRITE_CHUNK_BYTES = 96 * 1024;\nconst PAGE_BOUNDARY_SIZES = [\n\t1,\n\t4095,\n\t4096,\n\t4097,\n\t8191,\n\t8192,\n\t8193,\n\t32768,\n\t65535,\n\t65536,\n\t98304,\n\t131072,\n];\n\nfunction hashSeed(input: string): number {\n\tlet hash = 2166136261;\n\tfor (let i = 0; i < input.length; i += 1) {\n\t\thash ^= input.charCodeAt(i);\n\t\thash = Math.imul(hash, 16777619);\n\t}\n\treturn hash >>> 0;\n}\n\nfunction makeRng(seed: string): () => number {\n\tlet state = hashSeed(seed) || 0x9e3779b9;\n\treturn () => {\n\t\tstate = (state + 0x6d2b79f5) >>> 0;\n\t\tlet t = state;\n\t\tt = Math.imul(t ^ (t >>> 15), t | 1);\n\t\tt ^= t + Math.imul(t ^ (t >>> 7), t | 61);\n\t\treturn ((t ^ (t >>> 14)) >>> 0) / 4294967296;\n\t};\n}\n\nfunction intBetween(rng: () => number, min: number, max: number): number {\n\treturn min + Math.floor(rng() * (max - min + 1));\n}\n\nfunction checksum(input: string): number {\n\tlet hash = 2166136261;\n\tfor (let i = 0; i < input.length; i += 1) {\n\t\thash ^= input.charCodeAt(i);\n\t\thash = Math.imul(hash, 16777619);\n\t}\n\treturn hash >>> 0;\n}\n\nfunction payloadFor(\n\tseed: string,\n\tphase: number,\n\tindex: number,\n\tbytes: number,\n): string {\n\tconst prefix = `${seed}:${phase}:${index}:`;\n\tif (bytes <= prefix.length) return prefix.slice(0, bytes);\n\treturn prefix + \"x\".repeat(bytes - prefix.length);\n}\n\nasync function queryOne(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tsql: string,\n\t...args: unknown[]\n): Promise {\n\tconst rows = await database.execute(sql, ...args);\n\treturn rows[0] as T | undefined;\n}\n\nasync function transaction(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tfn: () => Promise,\n): Promise {\n\tawait database.execute(\"BEGIN\");\n\ttry {\n\t\tconst result = await fn();\n\t\tawait database.execute(\"COMMIT\");\n\t\treturn result;\n\t} catch (err) {\n\t\tawait database.execute(\"ROLLBACK\").catch(() => undefined);\n\t\tthrow err;\n\t}\n}\n\nasync function recordProbe(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n\tscenario: string,\n\tname: string,\n\texpected: string | number,\n\tactual: string | number,\n\tmismatch: boolean,\n): Promise {\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_probe_results (\n\t\t\tphase, scenario, name, expected, actual, mismatch, created_at\n\t\t) VALUES (?, ?, ?, ?, ?, ?, ?)`,\n\t\tphase,\n\t\tscenario,\n\t\tname,\n\t\tString(expected),\n\t\tString(actual),\n\t\tmismatch ? 1 : 0,\n\t\tDate.now(),\n\t);\n}\n\nfunction firstColumn(row: unknown): unknown {\n\tif (!row || typeof row !== \"object\") return undefined;\n\tconst values = Object.values(row as Record);\n\treturn values[0];\n}\n\nasync function ensureAccounts(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n): Promise {\n\tawait database.execute(\"BEGIN\");\n\ttry {\n\t\tfor (let i = 0; i < ACCOUNT_COUNT; i += 1) {\n\t\t\tawait database.execute(\n\t\t\t\t\"INSERT OR IGNORE INTO fuzz_accounts (id, balance) VALUES (?, ?)\",\n\t\t\t\t`acct-${i}`,\n\t\t\t\tACCOUNT_INITIAL_BALANCE,\n\t\t\t);\n\t\t}\n\t\tawait database.execute(\"COMMIT\");\n\t} catch (err) {\n\t\tawait database.execute(\"ROLLBACK\").catch(() => undefined);\n\t\tthrow err;\n\t}\n}\n\nasync function recordItemEvent(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n\tlocalIndex: number,\n\tkind: string,\n\titemKey: string,\n\tpresent: boolean,\n\tvalue: string | null,\n\tversion: number,\n\tupdateCount: number,\n\tpayload: string,\n\tapplied: boolean,\n): Promise {\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_item_events (\n\t\t\tphase, local_index, kind, item_key, present, value, version,\n\t\t\tupdate_count, payload_checksum, payload_bytes, applied, created_at\n\t\t) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n\t\tphase,\n\t\tlocalIndex,\n\t\tkind,\n\t\titemKey,\n\t\tpresent ? 1 : 0,\n\t\tvalue,\n\t\tversion,\n\t\tupdateCount,\n\t\tchecksum(payload),\n\t\tpayload.length,\n\t\tapplied ? 1 : 0,\n\t\tDate.now(),\n\t);\n}\n\nasync function upsertLiveItem(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\trow: ItemRow,\n\tpayload: string,\n): Promise {\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_items (\n\t\t\titem_key, value, version, update_count, payload, payload_checksum,\n\t\t\tpayload_bytes, updated_at\n\t\t) VALUES (?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON CONFLICT(item_key) DO UPDATE SET\n\t\t\tvalue = excluded.value,\n\t\t\tversion = excluded.version,\n\t\t\tupdate_count = excluded.update_count,\n\t\t\tpayload = excluded.payload,\n\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\tpayload_bytes = excluded.payload_bytes,\n\t\t\tupdated_at = excluded.updated_at`,\n\t\trow.item_key,\n\t\trow.value,\n\t\trow.version,\n\t\trow.update_count,\n\t\tpayload,\n\t\trow.payload_checksum,\n\t\trow.payload_bytes,\n\t\tDate.now(),\n\t);\n}\n\nasync function applyItemOperation(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tlocalIndex: number;\n\t\tkind: \"insert\" | \"update\" | \"delete\" | \"upsert\";\n\t\titemKey: string;\n\t\tpayloadBytes: number;\n\t},\n): Promise {\n\tlet current: ItemRow | undefined;\n\ttry {\n\t\tcurrent = await queryOne(\n\t\t\tdatabase,\n\t\t\t\"SELECT item_key, value, version, update_count, payload, payload_checksum, payload_bytes FROM fuzz_items WHERE item_key = ?\",\n\t\t\topts.itemKey,\n\t\t);\n\t} catch (error) {\n\t\tthrow new Error(\n\t\t\t`item operation select failed for kind ${opts.kind} key ${JSON.stringify(opts.itemKey)} payloadBytes ${opts.payloadBytes}`,\n\t\t\t{ cause: error },\n\t\t);\n\t}\n\tconst payload = payloadFor(\n\t\topts.seed,\n\t\topts.phase,\n\t\topts.localIndex,\n\t\topts.payloadBytes,\n\t);\n\tconst nextVersion = (current?.version ?? 0) + 1;\n\tconst nextUpdateCount = (current?.update_count ?? 0) + 1;\n\tconst nextValue = `${opts.kind}:${opts.phase}:${opts.localIndex}:${nextVersion}`;\n\n\tif (opts.kind === \"delete\") {\n\t\ttry {\n\t\t\tawait recordItemEvent(\n\t\t\t\tdatabase,\n\t\t\t\topts.phase,\n\t\t\t\topts.localIndex,\n\t\t\t\topts.kind,\n\t\t\t\topts.itemKey,\n\t\t\t\tfalse,\n\t\t\t\tnull,\n\t\t\t\tnextVersion,\n\t\t\t\tnextUpdateCount,\n\t\t\t\t\"\",\n\t\t\t\tcurrent !== undefined,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`item operation event insert failed for delete key ${JSON.stringify(opts.itemKey)}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\ttry {\n\t\t\tawait database.execute(\"DELETE FROM fuzz_items WHERE item_key = ?\", opts.itemKey);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`item operation delete failed for key ${JSON.stringify(opts.itemKey)}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\treturn;\n\t}\n\n\tif (opts.kind === \"insert\" && current) {\n\t\ttry {\n\t\t\tawait recordItemEvent(\n\t\t\t\tdatabase,\n\t\t\t\topts.phase,\n\t\t\t\topts.localIndex,\n\t\t\t\topts.kind,\n\t\t\t\topts.itemKey,\n\t\t\t\ttrue,\n\t\t\t\tcurrent.value,\n\t\t\t\tcurrent.version,\n\t\t\t\tcurrent.update_count,\n\t\t\t\tcurrent.payload ?? \"\",\n\t\t\t\tfalse,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tthrow new Error(\n\t\t\t\t`item operation event insert failed for noop insert key ${JSON.stringify(opts.itemKey)}`,\n\t\t\t\t{ cause: error },\n\t\t\t);\n\t\t}\n\t\treturn;\n\t}\n\n\tconst row: ItemRow = {\n\t\titem_key: opts.itemKey,\n\t\tvalue: nextValue,\n\t\tversion: nextVersion,\n\t\tupdate_count: nextUpdateCount,\n\t\tpayload_checksum: checksum(payload),\n\t\tpayload_bytes: payload.length,\n\t};\n\n\ttry {\n\t\tawait recordItemEvent(\n\t\t\tdatabase,\n\t\t\topts.phase,\n\t\t\topts.localIndex,\n\t\t\topts.kind,\n\t\t\topts.itemKey,\n\t\t\ttrue,\n\t\t\trow.value,\n\t\t\trow.version,\n\t\t\trow.update_count,\n\t\t\tpayload,\n\t\t\ttrue,\n\t\t);\n\t} catch (error) {\n\t\tthrow new Error(\n\t\t\t`item operation event insert failed for kind ${opts.kind} key ${JSON.stringify(opts.itemKey)} payloadBytes ${opts.payloadBytes}`,\n\t\t\t{ cause: error },\n\t\t);\n\t}\n\ttry {\n\t\tawait upsertLiveItem(database, row, payload);\n\t} catch (error) {\n\t\tthrow new Error(\n\t\t\t`item operation live-row upsert failed for kind ${opts.kind} key ${JSON.stringify(opts.itemKey)} payloadBytes ${opts.payloadBytes}`,\n\t\t\t{ cause: error },\n\t\t);\n\t}\n}\n\nasync function applyHotUpdates(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tlocalIndex: number;\n\t\titemKey: string;\n\t\tupdates: number;\n\t\tpayloadBytes: number;\n\t},\n): Promise {\n\tfor (let i = 0; i < opts.updates; i += 1) {\n\t\ttry {\n\t\t\tawait applyItemOperation(database, {\n\t\t\t\tseed: opts.seed,\n\t\t\t\tphase: opts.phase,\n\t\t\t\tlocalIndex: opts.localIndex * 1000 + i,\n\t\t\t\tkind: \"update\",\n\t\t\t\titemKey: opts.itemKey,\n\t\t\t\tpayloadBytes: opts.payloadBytes,\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tthrow new Error(\n\t\t\t\t`hot update failed for ${opts.itemKey} at sub-update ${i + 1}/${opts.updates} with payloadBytes ${opts.payloadBytes}`,\n\t\t\t\t{ cause: error },\n\t\t\t);\n\t\t}\n\t}\n}\n\nasync function applyTransfer(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tphase: number;\n\t\tlocalIndex: number;\n\t\tfromAccount: string;\n\t\ttoAccount: string;\n\t\tamount: number;\n\t},\n): Promise {\n\tawait transaction(database, async () => {\n\t\tconst before = await queryOne<{ total: number }>(\n\t\t\tdatabase,\n\t\t\t\"SELECT COALESCE(SUM(balance), 0) AS total FROM fuzz_accounts\",\n\t\t);\n\t\tawait database.execute(\n\t\t\t\"UPDATE fuzz_accounts SET balance = balance - ? WHERE id = ?\",\n\t\t\topts.amount,\n\t\t\topts.fromAccount,\n\t\t);\n\t\tawait database.execute(\n\t\t\t\"UPDATE fuzz_accounts SET balance = balance + ? WHERE id = ?\",\n\t\t\topts.amount,\n\t\t\topts.toAccount,\n\t\t);\n\t\tconst after = await queryOne<{ total: number }>(\n\t\t\tdatabase,\n\t\t\t\"SELECT COALESCE(SUM(balance), 0) AS total FROM fuzz_accounts\",\n\t\t);\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_transfer_events (\n\t\t\t\tphase, local_index, from_account, to_account, amount,\n\t\t\t\tbalance_sum_before, balance_sum_after, created_at\n\t\t\t) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,\n\t\t\topts.phase,\n\t\t\topts.localIndex,\n\t\t\topts.fromAccount,\n\t\t\topts.toAccount,\n\t\t\topts.amount,\n\t\t\tbefore?.total ?? 0,\n\t\t\tafter?.total ?? 0,\n\t\t\tDate.now(),\n\t\t);\n\t});\n}\n\nasync function applyEdgePayloads(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tmaxPayloadBytes: number;\n\t},\n): Promise {\n\tconst writeEdgePayload = async (\n\t\tid: string,\n\t\tkind: string,\n\t\tpayload: string,\n\t\tsizeLabel: string,\n\t): Promise => {\n\t\tconst payloadChecksum = checksum(payload);\n\t\tconst payloadBytes = payload.length;\n\t\ttry {\n\t\t\tawait database.execute(\"BEGIN\");\n\t\t} catch (error) {\n\t\t\tthrow new Error(`edge payload begin failed for ${sizeLabel}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\ttry {\n\t\t\ttry {\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`INSERT INTO fuzz_edge_payloads (\n\t\t\t\t\t\tid, kind, payload, payload_checksum, payload_bytes, updated_at\n\t\t\t\t\t) VALUES (?, ?, ?, ?, ?, ?)\n\t\t\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\t\t\tkind = excluded.kind,\n\t\t\t\t\t\tpayload = excluded.payload,\n\t\t\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\t\t\tpayload_bytes = excluded.payload_bytes,\n\t\t\t\t\t\tupdated_at = excluded.updated_at`,\n\t\t\t\t\tid,\n\t\t\t\t\tkind,\n\t\t\t\t\tpayload,\n\t\t\t\t\tpayloadChecksum,\n\t\t\t\t\tpayloadBytes,\n\t\t\t\t\tDate.now(),\n\t\t\t\t);\n\t\t\t} catch (error) {\n\t\t\t\tthrow new Error(`edge payload row upsert failed for ${sizeLabel}`, {\n\t\t\t\t\tcause: error,\n\t\t\t\t});\n\t\t\t}\n\t\t\ttry {\n\t\t\t\tawait database.execute(\n\t\t\t\t\t`INSERT INTO fuzz_edge_expectations (\n\t\t\t\t\t\tid, present, payload_checksum, payload_bytes\n\t\t\t\t\t) VALUES (?, 1, ?, ?)\n\t\t\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\t\t\tpresent = excluded.present,\n\t\t\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\t\t\tpayload_bytes = excluded.payload_bytes`,\n\t\t\t\t\tid,\n\t\t\t\t\tpayloadChecksum,\n\t\t\t\t\tpayloadBytes,\n\t\t\t\t);\n\t\t\t} catch (error) {\n\t\t\t\tthrow new Error(`edge payload expectation upsert failed for ${sizeLabel}`, {\n\t\t\t\t\tcause: error,\n\t\t\t\t});\n\t\t\t}\n\t\t\ttry {\n\t\t\t\tawait database.execute(\"COMMIT\");\n\t\t\t} catch (error) {\n\t\t\t\tthrow new Error(`edge payload commit failed for ${sizeLabel}`, {\n\t\t\t\t\tcause: error,\n\t\t\t\t});\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tawait database.execute(\"ROLLBACK\").catch(() => undefined);\n\t\t\tthrow error;\n\t\t}\n\t};\n\n\tconst sizes = PAGE_BOUNDARY_SIZES.filter((size) => size <= opts.maxPayloadBytes);\n\tif (!sizes.includes(opts.maxPayloadBytes)) sizes.push(opts.maxPayloadBytes);\n\n\tlet ops = 0;\n\tfor (const size of sizes) {\n\t\tconst id = `edge-${opts.phase}-${size}`;\n\t\tconst payload = payloadFor(opts.seed, opts.phase, size, size);\n\t\ttry {\n\t\t\tawait writeEdgePayload(id, \"boundary\", payload, `size ${size}`);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`edge payload write failed for size ${size}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\tops += 1;\n\t}\n\n\tconst unicodePayload = `escaped-nul:\\\\0 unicode:☃️ phase:${opts.phase} seed:${opts.seed}`;\n\tconst unicodeId = `edge-${opts.phase}-unicode-nul`;\n\ttry {\n\t\tawait writeEdgePayload(\n\t\t\tunicodeId,\n\t\t\t\"unicode-nul\",\n\t\t\tunicodePayload,\n\t\t\t\"unicode escaped-nul payload\",\n\t\t);\n\t} catch (error) {\n\t\tthrow new Error(\"edge payload write failed for unicode escaped-nul payload\", {\n\t\t\tcause: error,\n\t\t});\n\t}\n\n\treturn ops + 1;\n}\n\nasync function applyActualNulPayload(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t},\n): Promise {\n\tconst payload = `actual-nul:\\0 phase:${opts.phase} seed:${opts.seed}`;\n\tconst id = `actual-nul-${opts.phase}`;\n\tawait transaction(database, async () => {\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_edge_payloads (\n\t\t\t\tid, kind, payload, payload_checksum, payload_bytes, updated_at\n\t\t\t) VALUES (?, 'actual-nul', ?, ?, ?, ?)\n\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\tpayload = excluded.payload,\n\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\tpayload_bytes = excluded.payload_bytes,\n\t\t\t\tupdated_at = excluded.updated_at`,\n\t\t\tid,\n\t\t\tpayload,\n\t\t\tchecksum(payload),\n\t\t\tpayload.length,\n\t\t\tDate.now(),\n\t\t);\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_edge_expectations (\n\t\t\t\tid, present, payload_checksum, payload_bytes\n\t\t\t) VALUES (?, 1, ?, ?)\n\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\tpresent = 1,\n\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\tpayload_bytes = excluded.payload_bytes`,\n\t\t\tid,\n\t\t\tchecksum(payload),\n\t\t\tpayload.length,\n\t\t);\n\t});\n\treturn 1;\n}\n\nasync function applyFragmentationChurn(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\trng: () => number;\n\t\titerations: number;\n\t\tmaxPayloadBytes: number;\n\t},\n): Promise {\n\tconst rows = Math.max(12, Math.floor(opts.iterations / 2));\n\tlet ops = 0;\n\n\tfor (let i = 0; i < rows; i += 1) {\n\t\tconst size = intBetween(opts.rng, 32, Math.max(32, opts.maxPayloadBytes));\n\t\tconst id = `frag-${opts.phase}-${i}`;\n\t\tconst payload = payloadFor(opts.seed, opts.phase, 10_000 + i, size);\n\t\ttry {\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_edge_payloads (\n\t\t\t\t\tid, kind, payload, payload_checksum, payload_bytes, updated_at\n\t\t\t\t) VALUES (?, 'fragment', ?, ?, ?, ?)\n\t\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\t\tpayload = excluded.payload,\n\t\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\t\tpayload_bytes = excluded.payload_bytes,\n\t\t\t\t\tupdated_at = excluded.updated_at`,\n\t\t\t\tid,\n\t\t\t\tpayload,\n\t\t\t\tchecksum(payload),\n\t\t\t\tpayload.length,\n\t\t\t\tDate.now(),\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`fragmentation payload upsert failed for ${id} at size ${size}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\ttry {\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_edge_expectations (\n\t\t\t\t\tid, present, payload_checksum, payload_bytes\n\t\t\t\t) VALUES (?, 1, ?, ?)\n\t\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\t\tpresent = 1,\n\t\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\t\tpayload_bytes = excluded.payload_bytes`,\n\t\t\t\tid,\n\t\t\t\tchecksum(payload),\n\t\t\t\tpayload.length,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`fragmentation expectation upsert failed for ${id} at size ${size}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\tops += 1;\n\t}\n\n\tfor (let i = 0; i < rows; i += 3) {\n\t\tconst id = `frag-${opts.phase}-${i}`;\n\t\ttry {\n\t\t\tawait database.execute(\"DELETE FROM fuzz_edge_payloads WHERE id = ?\", id);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`fragmentation delete failed for ${id}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\ttry {\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_edge_expectations (\n\t\t\t\t\tid, present, payload_checksum, payload_bytes\n\t\t\t\t) VALUES (?, 0, 0, 0)\n\t\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\t\tpresent = 0,\n\t\t\t\t\tpayload_checksum = 0,\n\t\t\t\t\tpayload_bytes = 0`,\n\t\t\t\tid,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`fragmentation tombstone expectation failed for ${id}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\tops += 1;\n\t}\n\n\tfor (let i = 1; i < rows; i += 4) {\n\t\tconst size = intBetween(opts.rng, 1, Math.max(1, opts.maxPayloadBytes));\n\t\tconst id = `frag-${opts.phase}-${i}`;\n\t\tconst payload = payloadFor(opts.seed, opts.phase, 20_000 + i, size);\n\t\ttry {\n\t\t\tawait database.execute(\n\t\t\t\t`UPDATE fuzz_edge_payloads\n\t\t\t\tSET payload = ?, payload_checksum = ?, payload_bytes = ?, updated_at = ?\n\t\t\t\tWHERE id = ?`,\n\t\t\t\tpayload,\n\t\t\t\tchecksum(payload),\n\t\t\t\tpayload.length,\n\t\t\t\tDate.now(),\n\t\t\t\tid,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`fragmentation payload rewrite failed for ${id} at size ${size}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\ttry {\n\t\t\tawait database.execute(\n\t\t\t\t`UPDATE fuzz_edge_expectations\n\t\t\t\tSET payload_checksum = ?, payload_bytes = ?\n\t\t\t\tWHERE id = ? AND present = 1`,\n\t\t\t\tchecksum(payload),\n\t\t\t\tpayload.length,\n\t\t\t\tid,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tthrow new Error(`fragmentation expectation rewrite failed for ${id} at size ${size}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\tops += 1;\n\t}\n\n\tif (opts.phase % 2 === 1) {\n\t\ttry {\n\t\t\tawait database.execute(\"VACUUM\");\n\t\t} catch (error) {\n\t\t\tthrow new Error(`fragmentation vacuum failed for phase ${opts.phase}`, {\n\t\t\t\tcause: error,\n\t\t\t});\n\t\t}\n\t\tops += 1;\n\t}\n\n\treturn ops;\n}\n\nasync function applySchemaChurn(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n): Promise {\n\tconst table = `fuzz_schema_phase_${phase}`;\n\tconst index = `idx_fuzz_schema_phase_${phase}_name`;\n\tconst view = `view_fuzz_schema_phase_${phase}`;\n\tconst dropIndex = `idx_fuzz_schema_drop_probe_${phase}`;\n\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS ${table} (\n\t\t\tid INTEGER PRIMARY KEY,\n\t\t\tname TEXT NOT NULL UNIQUE,\n\t\t\tvalue INTEGER NOT NULL DEFAULT 0,\n\t\t\textra TEXT\n\t\t)`,\n\t);\n\tawait database.execute(`CREATE INDEX IF NOT EXISTS ${index} ON ${table}(name, value)`);\n\ttry {\n\t\tawait database.execute(`ALTER TABLE ${table} ADD COLUMN altered_${phase} TEXT DEFAULT 'altered'`);\n\t} catch {\n\t\tconst column = await queryOne<{ count: number }>(\n\t\t\tdatabase,\n\t\t\t`SELECT COUNT(*) AS count FROM pragma_table_info('${table}') WHERE name = ?`,\n\t\t\t`altered_${phase}`,\n\t\t);\n\t\tif ((column?.count ?? 0) !== 1) throw new Error(`failed to add altered_${phase}`);\n\t}\n\tawait database.execute(`CREATE VIEW IF NOT EXISTS ${view} AS SELECT id, name, value FROM ${table}`);\n\tawait database.execute(\n\t\t`INSERT INTO ${table} (name, value, extra)\n\t\tVALUES (?, ?, ?)\n\t\tON CONFLICT(name) DO UPDATE SET\n\t\t\tvalue = excluded.value,\n\t\t\textra = excluded.extra`,\n\t\t`schema-${phase}`,\n\t\tphase,\n\t\t`extra-${phase}`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS fuzz_without_rowid (\n\t\t\tid TEXT PRIMARY KEY,\n\t\t\tvalue INTEGER NOT NULL\n\t\t) WITHOUT ROWID`,\n\t);\n\tawait database.execute(`\n\t\tCREATE TRIGGER IF NOT EXISTS trg_fuzz_edge_payload_update\n\t\tAFTER UPDATE ON fuzz_edge_payloads\n\t\tBEGIN\n\t\t\tINSERT INTO fuzz_trigger_audit (\n\t\t\t\tpayload_id, old_checksum, new_checksum, created_at\n\t\t\t) VALUES (\n\t\t\t\tnew.id, old.payload_checksum, new.payload_checksum, strftime('%s', 'now') * 1000\n\t\t\t);\n\t\tEND\n\t`);\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_without_rowid (id, value)\n\t\tVALUES (?, ?)\n\t\tON CONFLICT(id) DO UPDATE SET value = excluded.value`,\n\t\t`phase-${phase}`,\n\t\tphase,\n\t);\n\n\tfor (const [name, type] of [\n\t\t[table, \"table\"],\n\t\t[index, \"index\"],\n\t\t[view, \"view\"],\n\t\t[\"trg_fuzz_edge_payload_update\", \"trigger\"],\n\t\t[\"fuzz_without_rowid\", \"table\"],\n\t] as const) {\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_schema_registry (name, type)\n\t\t\tVALUES (?, ?)\n\t\t\tON CONFLICT(name) DO UPDATE SET type = excluded.type`,\n\t\t\tname,\n\t\t\ttype,\n\t\t);\n\t}\n\n\tawait database.execute(\"CREATE TEMP TABLE IF NOT EXISTS fuzz_temp_probe (id INTEGER PRIMARY KEY, value TEXT)\");\n\tawait database.execute(\"INSERT INTO fuzz_temp_probe (value) VALUES (?)\", `temp-${phase}`);\n\tawait database.execute(\"DROP TABLE fuzz_temp_probe\");\n\tawait database.execute(`CREATE INDEX IF NOT EXISTS ${dropIndex} ON fuzz_schema_registry(type)`);\n\tawait database.execute(`DROP INDEX IF EXISTS ${dropIndex}`);\n\tconst dropped = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'index' AND name = ?\",\n\t\tdropIndex,\n\t);\n\tawait recordProbe(\n\t\tdatabase,\n\t\tphase,\n\t\t\"schema\",\n\t\t\"drop-index\",\n\t\t0,\n\t\tdropped?.count ?? -1,\n\t\t(dropped?.count ?? -1) !== 0,\n\t);\n\n\treturn 13;\n}\n\nasync function applyIndexProbe(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\trng: () => number;\n\t\titerations: number;\n\t},\n): Promise {\n\tconst rows = Math.max(20, opts.iterations);\n\tawait transaction(database, async () => {\n\t\tfor (let i = 0; i < rows; i += 1) {\n\t\t\tconst tenant = `tenant-${intBetween(opts.rng, 0, 5)}`;\n\t\t\tconst bucket = intBetween(opts.rng, 0, 12);\n\t\t\tconst score = intBetween(opts.rng, -500, 500);\n\t\t\tconst label = `${opts.seed}:${opts.phase}:${i}`;\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_indexed (tenant, bucket, score, label, payload)\n\t\t\t\tVALUES (?, ?, ?, ?, ?)`,\n\t\t\t\ttenant,\n\t\t\t\tbucket,\n\t\t\t\tscore,\n\t\t\t\tlabel,\n\t\t\t\tpayloadFor(opts.seed, opts.phase, 30_000 + i, intBetween(opts.rng, 8, 256)),\n\t\t\t);\n\t\t}\n\t});\n\treturn rows;\n}\n\nasync function applyPreparedChurn(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\titerations: number;\n\t\tmaxPayloadBytes: number;\n\t},\n): Promise {\n\tconst rows = Math.max(32, opts.iterations);\n\tfor (let i = 0; i < rows; i += 1) {\n\t\tconst id = `prep-${opts.phase}-${i}`;\n\t\tconst payload = payloadFor(\n\t\t\topts.seed,\n\t\t\topts.phase,\n\t\t\t70_000 + i,\n\t\t\tMath.min(opts.maxPayloadBytes, 64 + (i % 257)),\n\t\t);\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_prepared_churn (id, value, payload, payload_checksum)\n\t\t\tVALUES (?, ?, ?, ?)\n\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\tvalue = excluded.value,\n\t\t\t\tpayload = excluded.payload,\n\t\t\t\tpayload_checksum = excluded.payload_checksum\n\t\t\t/* unique-prepared-${opts.phase}-${i} */`,\n\t\t\tid,\n\t\t\ti,\n\t\t\tpayload,\n\t\t\tchecksum(payload),\n\t\t);\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_prepared_expectations (id, value, payload_checksum)\n\t\t\tVALUES (?, ?, ?)\n\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\tvalue = excluded.value,\n\t\t\t\tpayload_checksum = excluded.payload_checksum`,\n\t\t\tid,\n\t\t\ti,\n\t\t\tchecksum(payload),\n\t\t);\n\t}\n\n\tconst repeatedId = `prep-repeat-${opts.phase}`;\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_prepared_churn (id, value, payload, payload_checksum)\n\t\tVALUES (?, 0, '', 0)\n\t\tON CONFLICT(id) DO UPDATE SET value = 0, payload = '', payload_checksum = 0`,\n\t\trepeatedId,\n\t);\n\tfor (let i = 0; i < rows; i += 1) {\n\t\tconst payload = payloadFor(opts.seed, opts.phase, 80_000 + i, Math.min(512, opts.maxPayloadBytes));\n\t\tawait database.execute(\n\t\t\t`UPDATE fuzz_prepared_churn\n\t\t\tSET value = value + ?, payload = ?, payload_checksum = ?\n\t\t\tWHERE id = ?`,\n\t\t\t1,\n\t\t\tpayload,\n\t\t\tchecksum(payload),\n\t\t\trepeatedId,\n\t\t);\n\t}\n\tconst finalPayload = payloadFor(opts.seed, opts.phase, 80_000 + rows - 1, Math.min(512, opts.maxPayloadBytes));\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_prepared_expectations (id, value, payload_checksum)\n\t\tVALUES (?, ?, ?)\n\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\tvalue = excluded.value,\n\t\t\tpayload_checksum = excluded.payload_checksum`,\n\t\trepeatedId,\n\t\trows,\n\t\tchecksum(finalPayload),\n\t);\n\n\treturn rows * 2 + 1;\n}\n\nasync function applyReadWriteProbe(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\trng: () => number;\n\t\titerations: number;\n\t},\n): Promise {\n\tif ((await queryOne<{ count: number }>(database, \"SELECT COUNT(*) AS count FROM fuzz_indexed\"))?.count === 0) {\n\t\tawait applyIndexProbe(database, opts);\n\t}\n\n\tconst read = database.execute(\n\t\t`SELECT\n\t\t\tCOUNT(*) AS joined_rows,\n\t\t\tCOALESCE(SUM(a.score + b.score), 0) AS score_sum\n\t\tFROM fuzz_indexed a\n\t\tJOIN fuzz_indexed b ON b.bucket = a.bucket\n\t\tWHERE a.tenant <= 'tenant-3'`,\n\t);\n\tconst write = applyIndexProbe(database, {\n\t\tseed: opts.seed,\n\t\tphase: opts.phase,\n\t\trng: opts.rng,\n\t\titerations: Math.max(10, Math.floor(opts.iterations / 2)),\n\t});\n\tconst [readRows, writeOps] = await Promise.all([read, write]);\n\tconst row = readRows[0] as { joined_rows?: number; score_sum?: number } | undefined;\n\tconst joinedRows = Number(row?.joined_rows ?? -1);\n\tconst scoreSum = Number(row?.score_sum ?? Number.NaN);\n\tawait recordProbe(\n\t\tdatabase,\n\t\topts.phase,\n\t\t\"readwrite\",\n\t\t\"long-read-while-write\",\n\t\t\"nonnegative-finite\",\n\t\t`${joinedRows}:${scoreSum}`,\n\t\tjoinedRows < 0 || !Number.isFinite(scoreSum),\n\t);\n\treturn writeOps + 1;\n}\n\nasync function applyBoundaryKeys(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tmaxPayloadBytes: number;\n\t},\n): Promise {\n\tconst keys = [\n\t\t\"\",\n\t\t\" \",\n\t\t`long-${\"k\".repeat(2048)}`,\n\t\t\"slash/key\",\n\t\t\"comma,key\",\n\t\t\"percent%key\",\n\t\t\"CaseKey\",\n\t\t\"casekey\",\n\t];\n\tlet ops = 0;\n\tfor (const [index, key] of keys.entries()) {\n\t\ttry {\n\t\t\tawait applyItemOperation(database, {\n\t\t\t\tseed: opts.seed,\n\t\t\t\tphase: opts.phase,\n\t\t\t\tlocalIndex: 90_000 + index,\n\t\t\t\tkind: \"upsert\",\n\t\t\t\titemKey: key,\n\t\t\t\tpayloadBytes: Math.min(opts.maxPayloadBytes, 128 + index),\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tthrow new Error(\n\t\t\t\t`boundary key write failed for literal key ${JSON.stringify(key)} at index ${index}`,\n\t\t\t\t{ cause: error },\n\t\t\t);\n\t\t}\n\t\tops += 1;\n\t}\n\tfor (let i = 0; i < 128; i += 1) {\n\t\tconst itemKey = `seq-${opts.phase}-${i.toString().padStart(4, \"0\")}`;\n\t\ttry {\n\t\t\tawait applyItemOperation(database, {\n\t\t\t\tseed: opts.seed,\n\t\t\t\tphase: opts.phase,\n\t\t\t\tlocalIndex: 91_000 + i,\n\t\t\t\tkind: i % 4 === 0 ? \"delete\" : \"upsert\",\n\t\t\t\titemKey,\n\t\t\t\tpayloadBytes: Math.min(opts.maxPayloadBytes, 32 + (i % 97)),\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tthrow new Error(\n\t\t\t\t`boundary key write failed for sequential key ${JSON.stringify(itemKey)} at index ${i}`,\n\t\t\t\t{ cause: error },\n\t\t\t);\n\t\t}\n\t\tops += 1;\n\t}\n\tawait recordProbe(database, opts.phase, \"boundary-keys\", \"keys-written\", 136, ops, ops !== 136);\n\treturn ops;\n}\n\nasync function applyGrowthProbe(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tmaxPayloadBytes: number;\n\t\tgrowthTargetBytes: number;\n\t},\n): Promise {\n\tconst chunkBytes = Math.max(1, Math.min(LARGE_WRITE_CHUNK_BYTES, opts.maxPayloadBytes));\n\tconst rows = Math.max(1, Math.ceil(opts.growthTargetBytes / chunkBytes));\n\tlet written = 0;\n\tfor (let i = 0; i < rows; i += 1) {\n\t\tconst size = Math.min(chunkBytes, opts.growthTargetBytes - written);\n\t\tconst id = `growth-${opts.phase}-${opts.growthTargetBytes}-${i}`;\n\t\tconst payload = payloadFor(opts.seed, opts.phase, 100_000 + i, size);\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_edge_payloads (\n\t\t\t\tid, kind, payload, payload_checksum, payload_bytes, updated_at\n\t\t\t) VALUES (?, 'growth', ?, ?, ?, ?)\n\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\tpayload = excluded.payload,\n\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\tpayload_bytes = excluded.payload_bytes,\n\t\t\t\tupdated_at = excluded.updated_at`,\n\t\t\tid,\n\t\t\tpayload,\n\t\t\tchecksum(payload),\n\t\t\tpayload.length,\n\t\t\tDate.now(),\n\t\t);\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_edge_expectations (\n\t\t\t\tid, present, payload_checksum, payload_bytes\n\t\t\t) VALUES (?, 1, ?, ?)\n\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\tpresent = 1,\n\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\tpayload_bytes = excluded.payload_bytes`,\n\t\t\tid,\n\t\t\tchecksum(payload),\n\t\t\tpayload.length,\n\t\t);\n\t\twritten += size;\n\t}\n\tawait recordProbe(\n\t\tdatabase,\n\t\topts.phase,\n\t\t\"growth\",\n\t\t\"target-bytes-written\",\n\t\topts.growthTargetBytes,\n\t\twritten,\n\t\twritten !== opts.growthTargetBytes,\n\t);\n\treturn rows;\n}\n\nasync function applyTruncateRecreateProbe(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tmaxPayloadBytes: number;\n\t},\n): Promise {\n\tconst id = `truncate-${opts.phase}`;\n\tconst largeSize = Math.max(1, Math.min(opts.maxPayloadBytes, 131072));\n\tconst largePayload = payloadFor(opts.seed, opts.phase, 110_000, largeSize);\n\tconst tinyPayload = payloadFor(opts.seed, opts.phase, 110_001, 1);\n\tconst recreatedPayload = payloadFor(opts.seed, opts.phase, 110_002, Math.min(4096, largeSize));\n\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_edge_payloads (\n\t\t\tid, kind, payload, payload_checksum, payload_bytes, updated_at\n\t\t) VALUES (?, 'truncate', ?, ?, ?, ?)\n\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\tpayload = excluded.payload,\n\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\tpayload_bytes = excluded.payload_bytes,\n\t\t\tupdated_at = excluded.updated_at`,\n\t\tid,\n\t\tlargePayload,\n\t\tchecksum(largePayload),\n\t\tlargePayload.length,\n\t\tDate.now(),\n\t);\n\tawait database.execute(\n\t\t\"UPDATE fuzz_edge_payloads SET payload = ?, payload_checksum = ?, payload_bytes = ?, updated_at = ? WHERE id = ?\",\n\t\ttinyPayload,\n\t\tchecksum(tinyPayload),\n\t\ttinyPayload.length,\n\t\tDate.now(),\n\t\tid,\n\t);\n\tawait database.execute(\"DELETE FROM fuzz_edge_payloads WHERE id = ?\", id);\n\tawait database.execute(\"VACUUM\");\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_edge_payloads (\n\t\t\tid, kind, payload, payload_checksum, payload_bytes, updated_at\n\t\t) VALUES (?, 'truncate', ?, ?, ?, ?)`,\n\t\tid,\n\t\trecreatedPayload,\n\t\tchecksum(recreatedPayload),\n\t\trecreatedPayload.length,\n\t\tDate.now(),\n\t);\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_edge_expectations (\n\t\t\tid, present, payload_checksum, payload_bytes\n\t\t) VALUES (?, 1, ?, ?)\n\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\tpresent = 1,\n\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\tpayload_bytes = excluded.payload_bytes`,\n\t\tid,\n\t\tchecksum(recreatedPayload),\n\t\trecreatedPayload.length,\n\t);\n\treturn 5;\n}\n\nasync function updateShadowChecksums(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n): Promise {\n\tconst item = await queryOne<{ rows: number; value: number }>(\n\t\tdatabase,\n\t\t`SELECT COUNT(*) AS rows, COALESCE(SUM(payload_checksum + version + update_count), 0) AS value\n\t\tFROM fuzz_items`,\n\t);\n\tconst edge = await queryOne<{ rows: number; value: number }>(\n\t\tdatabase,\n\t\t`SELECT COUNT(*) AS rows, COALESCE(SUM(payload_checksum + payload_bytes), 0) AS value\n\t\tFROM fuzz_edge_payloads`,\n\t);\n\tawait transaction(database, async () => {\n\t\tfor (const [name, rows, value] of [\n\t\t\t[\"items\", item?.rows ?? 0, item?.value ?? 0],\n\t\t\t[\"edge\", edge?.rows ?? 0, edge?.value ?? 0],\n\t\t] as const) {\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_shadow_checksums (name, value, row_count)\n\t\t\t\tVALUES (?, ?, ?)\n\t\t\t\tON CONFLICT(name) DO UPDATE SET\n\t\t\t\t\tvalue = excluded.value,\n\t\t\t\t\trow_count = excluded.row_count`,\n\t\t\t\tname,\n\t\t\t\tvalue,\n\t\t\t\trows,\n\t\t\t);\n\t\t}\n\t});\n\tawait recordProbe(database, phase, \"shadow\", \"shadow-updated\", 2, 2, false);\n\treturn 2;\n}\n\nasync function applyConstraintChaos(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n): Promise {\n\tawait database.execute(\"PRAGMA foreign_keys = ON\");\n\tconst validPrefix = `valid-${phase}`;\n\tconst existingValidRows = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_constraints WHERE id LIKE ?\",\n\t\t`${validPrefix}-%`,\n\t);\n\tconst runSeq = existingValidRows?.count ?? 0;\n\tconst validId = `${validPrefix}-${runSeq}`;\n\tconst uniqValue = `uniq-${phase}-${runSeq}`;\n\tconst before = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_constraints\",\n\t);\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_constraints (id, must_not_null, qty, uniq)\n\t\tVALUES (?, ?, ?, ?)\n\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\tmust_not_null = excluded.must_not_null,\n\t\t\tqty = excluded.qty,\n\t\t\tuniq = excluded.uniq`,\n\t\tvalidId,\n\t\t\"ok\",\n\t\tphase,\n\t\tuniqValue,\n\t);\n\n\tconst attempts: Array<{\n\t\tname: string;\n\t\tsql: string;\n\t\targs: unknown[];\n\t}> = [\n\t\t{\n\t\t\tname: `not-null-${phase}-${runSeq}`,\n\t\t\tsql: \"INSERT INTO fuzz_constraints (id, must_not_null, qty, uniq) VALUES (?, ?, ?, ?)\",\n\t\t\targs: [`bad-null-${phase}-${runSeq}`, null, 1, `bad-null-${phase}-${runSeq}`],\n\t\t},\n\t\t{\n\t\t\tname: `check-${phase}-${runSeq}`,\n\t\t\tsql: \"INSERT INTO fuzz_constraints (id, must_not_null, qty, uniq) VALUES (?, ?, ?, ?)\",\n\t\t\targs: [`bad-check-${phase}-${runSeq}`, \"ok\", -1, `bad-check-${phase}-${runSeq}`],\n\t\t},\n\t\t{\n\t\t\tname: `unique-${phase}-${runSeq}`,\n\t\t\tsql: \"INSERT INTO fuzz_constraints (id, must_not_null, qty, uniq) VALUES (?, ?, ?, ?)\",\n\t\t\targs: [`bad-unique-${phase}-${runSeq}`, \"ok\", 1, uniqValue],\n\t\t},\n\t];\n\n\tfor (const attempt of attempts) {\n\t\tconst attemptBefore = await queryOne<{ count: number }>(\n\t\t\tdatabase,\n\t\t\t\"SELECT COUNT(*) AS count FROM fuzz_constraints\",\n\t\t);\n\t\tlet failed = false;\n\t\ttry {\n\t\t\tawait database.execute(attempt.sql, ...attempt.args);\n\t\t} catch {\n\t\t\tfailed = true;\n\t\t}\n\t\tconst attemptAfter = await queryOne<{ count: number }>(\n\t\t\tdatabase,\n\t\t\t\"SELECT COUNT(*) AS count FROM fuzz_constraints\",\n\t\t);\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_constraint_attempts (\n\t\t\t\tname, expected_failed, actually_failed, before_count, after_count\n\t\t\t) VALUES (?, 1, ?, ?, ?)`,\n\t\t\tattempt.name,\n\t\t\tfailed ? 1 : 0,\n\t\t\tattemptBefore?.count ?? 0,\n\t\t\tattemptAfter?.count ?? 0,\n\t\t);\n\t}\n\n\tconst after = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_constraints\",\n\t);\n\tif ((after?.count ?? 0) !== (before?.count ?? 0) + 1) {\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_constraint_attempts (\n\t\t\t\tname, expected_failed, actually_failed, before_count, after_count\n\t\t\t) VALUES (?, 0, 0, ?, ?)`,\n\t\t\t`valid-count-${phase}-${runSeq}`,\n\t\t\tbefore?.count ?? 0,\n\t\t\tafter?.count ?? 0,\n\t\t);\n\t}\n\n\tconst parentId = `fk-parent-${phase}-${runSeq}`;\n\tconst childId = `fk-child-${phase}-${runSeq}`;\n\tawait database.execute(\n\t\t\"INSERT INTO fuzz_fk_parent (id) VALUES (?) ON CONFLICT(id) DO NOTHING\",\n\t\tparentId,\n\t);\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_fk_child (id, parent_id)\n\t\tVALUES (?, ?)\n\t\tON CONFLICT(id) DO UPDATE SET parent_id = excluded.parent_id`,\n\t\tchildId,\n\t\tparentId,\n\t);\n\tconst childBeforeDelete = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_fk_child WHERE parent_id = ?\",\n\t\tparentId,\n\t);\n\tawait database.execute(\"DELETE FROM fuzz_fk_parent WHERE id = ?\", parentId);\n\tconst childAfterDelete = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_fk_child WHERE parent_id = ?\",\n\t\tparentId,\n\t);\n\tawait recordProbe(\n\t\tdatabase,\n\t\tphase,\n\t\t\"constraints\",\n\t\t\"fk-cascade-delete\",\n\t\t0,\n\t\tchildAfterDelete?.count ?? -1,\n\t\t(childBeforeDelete?.count ?? 0) !== 1 || (childAfterDelete?.count ?? -1) !== 0,\n\t);\n\n\tconst fkBefore = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_fk_child\",\n\t);\n\tlet fkFailed = false;\n\ttry {\n\t\tawait database.execute(\n\t\t\t\"INSERT INTO fuzz_fk_child (id, parent_id) VALUES (?, ?)\",\n\t\t\t`fk-orphan-${phase}-${runSeq}`,\n\t\t\t`missing-parent-${phase}-${runSeq}`,\n\t\t);\n\t} catch {\n\t\tfkFailed = true;\n\t}\n\tconst fkAfter = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_fk_child\",\n\t);\n\tawait recordProbe(\n\t\tdatabase,\n\t\tphase,\n\t\t\"constraints\",\n\t\t\"fk-failure-isolation\",\n\t\t`${fkBefore?.count ?? 0}:failed`,\n\t\t`${fkAfter?.count ?? 0}:${fkFailed ? \"failed\" : \"inserted\"}`,\n\t\t!fkFailed || (fkAfter?.count ?? 0) !== (fkBefore?.count ?? 0),\n\t);\n\n\treturn attempts.length + 3;\n}\n\nasync function applyPragmaProbe(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n): Promise {\n\tlet ops = 0;\n\tfor (const [name, setupSql, checkSql, expected] of [\n\t\t[\"journal_mode\", \"PRAGMA journal_mode = DELETE\", \"PRAGMA journal_mode\", \"nonempty\"],\n\t\t[\"synchronous\", \"PRAGMA synchronous = NORMAL\", \"PRAGMA synchronous\", \"nonempty\"],\n\t\t[\"cache_size\", \"PRAGMA cache_size = -2000\", \"PRAGMA cache_size\", \"-2000\"],\n\t\t[\"foreign_keys\", \"PRAGMA foreign_keys = ON\", \"PRAGMA foreign_keys\", \"1\"],\n\t\t[\"auto_vacuum\", \"PRAGMA auto_vacuum\", \"PRAGMA auto_vacuum\", \"nonempty\"],\n\t] as const) {\n\t\ttry {\n\t\t\tawait database.execute(setupSql);\n\t\t\tconst rows = await database.execute(checkSql);\n\t\t\tconst actual = String(firstColumn(rows[0]) ?? \"\");\n\t\t\tawait recordProbe(\n\t\t\t\tdatabase,\n\t\t\t\tphase,\n\t\t\t\t\"pragma\",\n\t\t\t\tname,\n\t\t\t\texpected,\n\t\t\t\tactual,\n\t\t\t\texpected === \"nonempty\" ? actual.length === 0 : actual !== expected,\n\t\t\t);\n\t\t} catch (err) {\n\t\t\tawait recordProbe(\n\t\t\t\tdatabase,\n\t\t\t\tphase,\n\t\t\t\t\"pragma\",\n\t\t\t\tname,\n\t\t\t\texpected,\n\t\t\t\terr instanceof Error ? err.message : \"unknown error\",\n\t\t\t\ttrue,\n\t\t\t);\n\t\t}\n\t\tops += 1;\n\t}\n\treturn ops;\n}\n\nasync function applySavepointScenario(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n): Promise {\n\tconst keepId = `save-keep-${phase}`;\n\tconst rolledBackId = `save-rolled-back-${phase}`;\n\n\tawait database.execute(\"BEGIN\");\n\ttry {\n\t\tawait database.execute(\n\t\t\t\"INSERT INTO fuzz_savepoints (id, value) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET value = excluded.value\",\n\t\t\tkeepId,\n\t\t\tphase,\n\t\t);\n\t\tawait database.execute(\"SAVEPOINT sp_rollback_probe\");\n\t\tawait database.execute(\n\t\t\t\"INSERT INTO fuzz_savepoints (id, value) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET value = excluded.value\",\n\t\t\trolledBackId,\n\t\t\t999_000 + phase,\n\t\t);\n\t\tawait database.execute(\n\t\t\t\"UPDATE fuzz_savepoints SET value = value + 1000 WHERE id = ?\",\n\t\t\tkeepId,\n\t\t);\n\t\tawait database.execute(\"ROLLBACK TO sp_rollback_probe\");\n\t\tawait database.execute(\"RELEASE sp_rollback_probe\");\n\t\tawait database.execute(\"COMMIT\");\n\t} catch (err) {\n\t\tawait database.execute(\"ROLLBACK\").catch(() => undefined);\n\t\tthrow err;\n\t}\n\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_savepoint_expectations (id, present, value)\n\t\tVALUES (?, 1, ?)\n\t\tON CONFLICT(id) DO UPDATE SET present = 1, value = excluded.value`,\n\t\tkeepId,\n\t\tphase,\n\t);\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_savepoint_expectations (id, present, value)\n\t\tVALUES (?, 0, 0)\n\t\tON CONFLICT(id) DO UPDATE SET present = 0, value = 0`,\n\t\trolledBackId,\n\t);\n\n\treturn 5;\n}\n\nasync function applyIdempotentReplay(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n): Promise {\n\tconst targetId = `idem-target-${phase % 3}`;\n\tawait database.execute(\n\t\t\"INSERT OR IGNORE INTO fuzz_idempotent_targets (id, value) VALUES (?, 0)\",\n\t\ttargetId,\n\t);\n\n\tfor (let i = 0; i < 8; i += 1) {\n\t\tconst opId = `idem-${phase}-${i}`;\n\t\tconst amount = phase + i + 1;\n\t\tfor (let attempt = 0; attempt < 3; attempt += 1) {\n\t\t\tawait transaction(database, async () => {\n\t\t\t\tconst existing = await queryOne<{ op_id: string }>(\n\t\t\t\t\tdatabase,\n\t\t\t\t\t\"SELECT op_id FROM fuzz_idempotent_ops WHERE op_id = ?\",\n\t\t\t\t\topId,\n\t\t\t\t);\n\t\t\t\tif (!existing) {\n\t\t\t\t\tawait database.execute(\n\t\t\t\t\t\t\"INSERT INTO fuzz_idempotent_ops (op_id, target_id, amount) VALUES (?, ?, ?)\",\n\t\t\t\t\t\topId,\n\t\t\t\t\t\ttargetId,\n\t\t\t\t\t\tamount,\n\t\t\t\t\t);\n\t\t\t\t\tawait database.execute(\n\t\t\t\t\t\t\"UPDATE fuzz_idempotent_targets SET value = value + ? WHERE id = ?\",\n\t\t\t\t\t\tamount,\n\t\t\t\t\t\ttargetId,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t});\n\t\t}\n\t}\n\n\treturn 24;\n}\n\nasync function ensureRelationalSeed(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n): Promise {\n\tawait transaction(database, async () => {\n\t\tfor (let i = 0; i < 8; i += 1) {\n\t\t\tawait database.execute(\n\t\t\t\t\"INSERT OR IGNORE INTO fuzz_rel_users (id, name) VALUES (?, ?)\",\n\t\t\t\t`user-${i}`,\n\t\t\t\t`User ${i}`,\n\t\t\t);\n\t\t}\n\t\tfor (let i = 0; i < 12; i += 1) {\n\t\t\tconst productId = `product-${i}`;\n\t\t\tconst initialQty = 10_000;\n\t\t\tawait database.execute(\n\t\t\t\t\"INSERT OR IGNORE INTO fuzz_rel_products (id, price) VALUES (?, ?)\",\n\t\t\t\tproductId,\n\t\t\t\t(i + 1) * 7,\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT OR IGNORE INTO fuzz_inventory (\n\t\t\t\t\tproduct_id, initial_qty, sold_qty, stock_qty\n\t\t\t\t) VALUES (?, ?, 0, ?)`,\n\t\t\t\tproductId,\n\t\t\t\tinitialQty,\n\t\t\t\tinitialQty,\n\t\t\t);\n\t\t}\n\t});\n}\n\nasync function applyRelationalOrder(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tphase: number;\n\t\tlocalIndex: number;\n\t\trng: () => number;\n\t},\n): Promise {\n\tawait ensureRelationalSeed(database);\n\tconst orderPrefix = `order-${opts.phase}-${opts.localIndex}`;\n\tconst existingOrders = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_orders WHERE id LIKE ?\",\n\t\t`${orderPrefix}-%`,\n\t);\n\tconst orderId = `${orderPrefix}-${existingOrders?.count ?? 0}`;\n\tconst userId = `user-${intBetween(opts.rng, 0, 7)}`;\n\tconst itemCount = intBetween(opts.rng, 1, 4);\n\tlet total = 0;\n\n\tawait transaction(database, async () => {\n\t\tawait database.execute(\n\t\t\t\"INSERT INTO fuzz_orders (id, user_id, total, status) VALUES (?, ?, 0, 'open')\",\n\t\t\torderId,\n\t\t\tuserId,\n\t\t);\n\t\tfor (let i = 0; i < itemCount; i += 1) {\n\t\t\tconst productId = `product-${intBetween(opts.rng, 0, 11)}`;\n\t\t\tconst product = await queryOne<{ price: number }>(\n\t\t\t\tdatabase,\n\t\t\t\t\"SELECT price FROM fuzz_rel_products WHERE id = ?\",\n\t\t\t\tproductId,\n\t\t\t);\n\t\t\tconst quantity = intBetween(opts.rng, 1, 5);\n\t\t\tconst price = product?.price ?? 0;\n\t\t\ttotal += price * quantity;\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_order_items (\n\t\t\t\t\torder_id, product_id, quantity, price\n\t\t\t\t) VALUES (?, ?, ?, ?)`,\n\t\t\t\torderId,\n\t\t\t\tproductId,\n\t\t\t\tquantity,\n\t\t\t\tprice,\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t`UPDATE fuzz_inventory\n\t\t\t\tSET sold_qty = sold_qty + ?, stock_qty = stock_qty - ?\n\t\t\t\tWHERE product_id = ?`,\n\t\t\t\tquantity,\n\t\t\t\tquantity,\n\t\t\t\tproductId,\n\t\t\t);\n\t\t}\n\t\tawait database.execute(\n\t\t\t\"UPDATE fuzz_orders SET total = ?, status = 'paid' WHERE id = ?\",\n\t\t\ttotal,\n\t\t\torderId,\n\t\t);\n\t\tawait database.execute(\n\t\t\t\"INSERT INTO fuzz_payments (order_id, amount, status) VALUES (?, ?, 'captured')\",\n\t\t\torderId,\n\t\t\ttotal,\n\t\t);\n\t});\n\n\treturn itemCount + 4;\n}\n\nasync function applyRollbackProbe(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tphase: number,\n\trowCount = 20,\n): Promise {\n\tconst before = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_items WHERE item_key LIKE ?\",\n\t\t`rollback-${phase}-%`,\n\t);\n\tawait database.execute(\"BEGIN\");\n\ttry {\n\t\tfor (let i = 0; i < rowCount; i += 1) {\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_items (\n\t\t\t\t\titem_key, value, version, update_count, payload, payload_checksum,\n\t\t\t\t\tpayload_bytes, updated_at\n\t\t\t\t) VALUES (?, ?, 1, 1, ?, ?, ?, ?)`,\n\t\t\t\t`rollback-${phase}-${i}`,\n\t\t\t\t\"should-not-survive\",\n\t\t\t\t\"rollback-payload\",\n\t\t\t\tchecksum(\"rollback-payload\"),\n\t\t\t\t\"rollback-payload\".length,\n\t\t\t\tDate.now(),\n\t\t\t);\n\t\t}\n\t\tthrow new Error(\"intentional rollback probe\");\n\t} catch {\n\t\tawait database.execute(\"ROLLBACK\");\n\t}\n\tconst after = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_items WHERE item_key LIKE ?\",\n\t\t`rollback-${phase}-%`,\n\t);\n\tawait database.execute(\n\t\t`INSERT INTO fuzz_constraint_attempts (\n\t\t\tname, expected_failed, actually_failed, before_count, after_count\n\t\t) VALUES (?, 1, 1, ?, ?)`,\n\t\t`rollback-probe-${phase}`,\n\t\tbefore?.count ?? 0,\n\t\tafter?.count ?? 0,\n\t);\n\treturn rowCount;\n}\n\nasync function applyNastyScript(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tmaxPayloadBytes: number;\n\t\tintense?: boolean;\n\t},\n): Promise {\n\tconst growKey = `nasty-grow-${opts.phase}`;\n\tlet ops = 0;\n\tconst growMax = Math.min(opts.maxPayloadBytes, 131072);\n\tconst growSizes = PAGE_BOUNDARY_SIZES.filter((size) => size <= growMax);\n\tif (!growSizes.includes(1)) growSizes.unshift(1);\n\tif (!growSizes.includes(growMax)) growSizes.push(growMax);\n\tfor (const size of growSizes) {\n\t\tawait applyItemOperation(database, {\n\t\t\tseed: opts.seed,\n\t\t\tphase: opts.phase,\n\t\t\tlocalIndex: 50_000 + size,\n\t\t\tkind: \"upsert\",\n\t\t\titemKey: growKey,\n\t\t\tpayloadBytes: Math.min(size, opts.maxPayloadBytes),\n\t\t});\n\t\tops += 1;\n\t}\n\n\tconst hotUpdates = opts.intense ? 10_000 : 250;\n\tawait applyHotUpdates(database, {\n\t\tseed: opts.seed,\n\t\tphase: opts.phase,\n\t\tlocalIndex: 60_000,\n\t\titemKey: `nasty-hot-${opts.phase}`,\n\t\tupdates: hotUpdates,\n\t\tpayloadBytes: Math.min(1024, opts.maxPayloadBytes),\n\t});\n\tops += hotUpdates;\n\n\tif (opts.intense) {\n\t\tawait database.execute(\"CREATE INDEX IF NOT EXISTS idx_nasty_heavy_write ON fuzz_items(value, version)\");\n\t\tfor (let i = 0; i < 10_000; i += 1) {\n\t\t\tawait applyItemOperation(database, {\n\t\t\t\tseed: opts.seed,\n\t\t\t\tphase: opts.phase,\n\t\t\t\tlocalIndex: 120_000 + i,\n\t\t\t\tkind: \"upsert\",\n\t\t\t\titemKey: `nasty-bulk-${opts.phase}-${i}`,\n\t\t\t\tpayloadBytes: Math.min(256, opts.maxPayloadBytes),\n\t\t\t});\n\t\t\tops += 1;\n\t\t}\n\t\tfor (let i = 0; i < 10_000; i += 2) {\n\t\t\tawait applyItemOperation(database, {\n\t\t\t\tseed: opts.seed,\n\t\t\t\tphase: opts.phase,\n\t\t\t\tlocalIndex: 140_000 + i,\n\t\t\t\tkind: \"delete\",\n\t\t\t\titemKey: `nasty-bulk-${opts.phase}-${i}`,\n\t\t\t\tpayloadBytes: 1,\n\t\t\t});\n\t\t\tops += 1;\n\t\t}\n\t\tawait database.execute(\"DROP INDEX IF EXISTS idx_nasty_heavy_write\");\n\t}\n\n\tconst rollbackRows = opts.intense ? 1000 : 20;\n\tawait applyRollbackProbe(database, opts.phase, rollbackRows);\n\tops += rollbackRows;\n\n\treturn ops;\n}\n\nasync function applyDeterministicNastyScript(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tmaxPayloadBytes: number;\n\t},\n): Promise {\n\tlet ops = 0;\n\tconst growId = `nasty-script-grow-${opts.phase}`;\n\tconst maxGrowBytes = Math.min(opts.maxPayloadBytes, 131072);\n\tlet finalGrowPayload = \"\";\n\tawait transaction(database, async () => {\n\t\tfor (let i = 0; i < 256; i += 1) {\n\t\t\tconst size = Math.max(1, Math.floor(1 + ((maxGrowBytes - 1) * i) / 255));\n\t\t\tconst payload = payloadFor(opts.seed, opts.phase, 160_000 + i, size);\n\t\t\tfinalGrowPayload = payload;\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_edge_payloads (\n\t\t\t\t\tid, kind, payload, payload_checksum, payload_bytes, updated_at\n\t\t\t\t) VALUES (?, 'nasty-grow', ?, ?, ?, ?)\n\t\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\t\tpayload = excluded.payload,\n\t\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\t\tpayload_bytes = excluded.payload_bytes,\n\t\t\t\t\tupdated_at = excluded.updated_at`,\n\t\t\t\tgrowId,\n\t\t\t\tpayload,\n\t\t\t\tchecksum(payload),\n\t\t\t\tpayload.length,\n\t\t\t\tDate.now(),\n\t\t\t);\n\t\t\tops += 1;\n\t\t}\n\t\tawait database.execute(\n\t\t\t`INSERT INTO fuzz_edge_expectations (\n\t\t\t\tid, present, payload_checksum, payload_bytes\n\t\t\t) VALUES (?, 1, ?, ?)\n\t\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\t\tpresent = 1,\n\t\t\t\tpayload_checksum = excluded.payload_checksum,\n\t\t\t\tpayload_bytes = excluded.payload_bytes`,\n\t\t\tgrowId,\n\t\t\tchecksum(finalGrowPayload),\n\t\t\tfinalGrowPayload.length,\n\t\t);\n\t});\n\n\tconst counterId = `nasty-counter-${opts.phase}`;\n\tawait database.execute(\n\t\t\"INSERT INTO fuzz_nasty_counter (id, value) VALUES (?, 0) ON CONFLICT(id) DO UPDATE SET value = 0\",\n\t\tcounterId,\n\t);\n\tawait transaction(database, async () => {\n\t\tfor (let i = 0; i < 10_000; i += 1) {\n\t\t\tawait database.execute(\n\t\t\t\t\"UPDATE fuzz_nasty_counter SET value = value + 1 WHERE id = ?\",\n\t\t\t\tcounterId,\n\t\t\t);\n\t\t\tops += 1;\n\t\t}\n\t});\n\tconst counter = await queryOne<{ value: number }>(\n\t\tdatabase,\n\t\t\"SELECT value FROM fuzz_nasty_counter WHERE id = ?\",\n\t\tcounterId,\n\t);\n\tawait recordProbe(\n\t\tdatabase,\n\t\topts.phase,\n\t\t\"nasty-script\",\n\t\t\"same-row-10k-updates\",\n\t\t10_000,\n\t\tcounter?.value ?? -1,\n\t\t(counter?.value ?? -1) !== 10_000,\n\t);\n\n\tconst groupId = `nasty-bulk-${opts.phase}`;\n\tawait database.execute(\"CREATE INDEX IF NOT EXISTS idx_fuzz_nasty_rows_group_n ON fuzz_nasty_rows(group_id, n)\");\n\tawait transaction(database, async () => {\n\t\tfor (let i = 0; i < 10_000; i += 1) {\n\t\t\tawait database.execute(\n\t\t\t\t`INSERT INTO fuzz_nasty_rows (group_id, n, payload)\n\t\t\t\tVALUES (?, ?, ?)\n\t\t\t\tON CONFLICT(group_id, n) DO UPDATE SET payload = excluded.payload`,\n\t\t\t\tgroupId,\n\t\t\t\ti,\n\t\t\t\tpayloadFor(opts.seed, opts.phase, 170_000 + i, 64),\n\t\t\t);\n\t\t\tops += 1;\n\t\t}\n\t\tawait database.execute(\"DELETE FROM fuzz_nasty_rows WHERE group_id = ? AND n % 2 = 0\", groupId);\n\t\tops += 1;\n\t});\n\tawait database.execute(\"DROP INDEX IF EXISTS idx_fuzz_nasty_rows_group_n\");\n\tconst remaining = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_nasty_rows WHERE group_id = ?\",\n\t\tgroupId,\n\t);\n\tawait recordProbe(\n\t\tdatabase,\n\t\topts.phase,\n\t\t\"nasty-script\",\n\t\t\"insert-10k-delete-every-other\",\n\t\t5000,\n\t\tremaining?.count ?? -1,\n\t\t(remaining?.count ?? -1) !== 5000,\n\t);\n\tconst indexLeft = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'index' AND name = 'idx_fuzz_nasty_rows_group_n'\",\n\t);\n\tawait recordProbe(\n\t\tdatabase,\n\t\topts.phase,\n\t\t\"nasty-script\",\n\t\t\"create-drop-index-around-heavy-writes\",\n\t\t0,\n\t\tindexLeft?.count ?? -1,\n\t\t(indexLeft?.count ?? -1) !== 0,\n\t);\n\n\tconst rollbackGroupId = `nasty-rollback-${opts.phase}`;\n\tconst beforeRollback = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_nasty_rows WHERE group_id = ?\",\n\t\trollbackGroupId,\n\t);\n\tawait database.execute(\"BEGIN\");\n\ttry {\n\t\tfor (let i = 0; i < 1000; i += 1) {\n\t\t\tawait database.execute(\n\t\t\t\t\"INSERT INTO fuzz_nasty_rows (group_id, n, payload) VALUES (?, ?, ?)\",\n\t\t\t\trollbackGroupId,\n\t\t\t\ti,\n\t\t\t\t\"rollback\",\n\t\t\t);\n\t\t\tops += 1;\n\t\t}\n\t\tawait database.execute(\"ROLLBACK\");\n\t} catch (err) {\n\t\tawait database.execute(\"ROLLBACK\").catch(() => undefined);\n\t\tthrow err;\n\t}\n\tconst afterRollback = await queryOne<{ count: number }>(\n\t\tdatabase,\n\t\t\"SELECT COUNT(*) AS count FROM fuzz_nasty_rows WHERE group_id = ?\",\n\t\trollbackGroupId,\n\t);\n\tawait recordProbe(\n\t\tdatabase,\n\t\topts.phase,\n\t\t\"nasty-script\",\n\t\t\"rollback-1k-inserts\",\n\t\tbeforeRollback?.count ?? 0,\n\t\tafterRollback?.count ?? -1,\n\t\t(afterRollback?.count ?? -1) !== (beforeRollback?.count ?? 0),\n\t);\n\n\treturn ops;\n}\n\nfunction shouldRunDeepScenario(mode: WorkloadMode, scenario: WorkloadMode): boolean {\n\treturn mode === scenario || mode === \"kitchen-sink\" || mode === \"nasty\";\n}\n\nasync function applyDeepScenarios(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\topts: {\n\t\tseed: string;\n\t\tphase: number;\n\t\tmode: WorkloadMode;\n\t\titerations: number;\n\t\trng: () => number;\n\t\tmaxPayloadBytes: number;\n\t\tgrowthTargetBytes: number;\n\t\tops: Record;\n\t},\n): Promise {\n\tconst runScenario = async (name: string, fn: () => Promise): Promise => {\n\t\ttry {\n\t\t\treturn await fn();\n\t\t} catch (error) {\n\t\t\tconst detail =\n\t\t\t\terror instanceof Error ? error.message : typeof error === \"string\" ? error : String(error);\n\t\t\tthrow new Error(\n\t\t\t\t`deep scenario ${name} failed in mode ${opts.mode} during phase ${opts.phase}: ${detail}`,\n\t\t\t\t{ cause: error },\n\t\t\t);\n\t\t}\n\t};\n\n\tif (shouldRunDeepScenario(opts.mode, \"edge\") || opts.mode === \"payloads\") {\n\t\topts.ops.edgePayload = (opts.ops.edgePayload ?? 0) +\n\t\t\tawait runScenario(\"edge\", () => applyEdgePayloads(database, opts));\n\t}\n\tif (opts.mode === \"actual-nul\") {\n\t\topts.ops.actualNul = (opts.ops.actualNul ?? 0) +\n\t\t\tawait runScenario(\"actual-nul\", () => applyActualNulPayload(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"fragmentation\")) {\n\t\topts.ops.fragmentation = (opts.ops.fragmentation ?? 0) +\n\t\t\tawait runScenario(\"fragmentation\", () => applyFragmentationChurn(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"schema\")) {\n\t\topts.ops.schema = (opts.ops.schema ?? 0) +\n\t\t\tawait runScenario(\"schema\", () => applySchemaChurn(database, opts.phase));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"index\")) {\n\t\topts.ops.index = (opts.ops.index ?? 0) +\n\t\t\tawait runScenario(\"index\", () => applyIndexProbe(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"constraints\")) {\n\t\topts.ops.constraints = (opts.ops.constraints ?? 0) +\n\t\t\tawait runScenario(\"constraints\", () => applyConstraintChaos(database, opts.phase));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"savepoints\")) {\n\t\topts.ops.savepoints = (opts.ops.savepoints ?? 0) +\n\t\t\tawait runScenario(\"savepoints\", () => applySavepointScenario(database, opts.phase));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"pragma\")) {\n\t\topts.ops.pragma = (opts.ops.pragma ?? 0) +\n\t\t\tawait runScenario(\"pragma\", () => applyPragmaProbe(database, opts.phase));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"prepared\")) {\n\t\topts.ops.prepared = (opts.ops.prepared ?? 0) +\n\t\t\tawait runScenario(\"prepared\", () => applyPreparedChurn(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"growth\")) {\n\t\topts.ops.growth = (opts.ops.growth ?? 0) +\n\t\t\tawait runScenario(\"growth\", () => applyGrowthProbe(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"readwrite\")) {\n\t\topts.ops.readwrite = (opts.ops.readwrite ?? 0) +\n\t\t\tawait runScenario(\"readwrite\", () => applyReadWriteProbe(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"truncate\")) {\n\t\topts.ops.truncate = (opts.ops.truncate ?? 0) +\n\t\t\tawait runScenario(\"truncate\", () => applyTruncateRecreateProbe(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"boundary-keys\")) {\n\t\topts.ops.boundaryKeys = (opts.ops.boundaryKeys ?? 0) +\n\t\t\tawait runScenario(\"boundary-keys\", () => applyBoundaryKeys(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"relational\")) {\n\t\tconst orders = Math.max(1, Math.floor(opts.iterations / 20));\n\t\tfor (let i = 0; i < orders; i += 1) {\n\t\t\topts.ops.relational = (opts.ops.relational ?? 0) +\n\t\t\t\tawait runScenario(\"relational\", () =>\n\t\t\t\t\tapplyRelationalOrder(database, {\n\t\t\t\t\t\tphase: opts.phase,\n\t\t\t\t\t\tlocalIndex: i,\n\t\t\t\t\t\trng: opts.rng,\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t}\n\t}\n\tif (opts.mode === \"kitchen-sink\" || opts.mode === \"nasty\") {\n\t\topts.ops.idempotent = (opts.ops.idempotent ?? 0) +\n\t\t\tawait runScenario(\"idempotent\", () => applyIdempotentReplay(database, opts.phase));\n\t\topts.ops.nasty = (opts.ops.nasty ?? 0) +\n\t\t\tawait runScenario(\"nasty\", () =>\n\t\t\t\tapplyNastyScript(database, { ...opts, intense: opts.mode === \"nasty\" }),\n\t\t\t);\n\t}\n\tif (opts.mode === \"nasty-script\") {\n\t\topts.ops.nasty = (opts.ops.nasty ?? 0) +\n\t\t\tawait runScenario(\"nasty-script\", () => applyDeterministicNastyScript(database, opts));\n\t}\n\tif (shouldRunDeepScenario(opts.mode, \"shadow\")) {\n\t\topts.ops.shadow = (opts.ops.shadow ?? 0) +\n\t\t\tawait runScenario(\"shadow\", () => updateShadowChecksums(database, opts.phase));\n\t}\n}\n\nfunction chooseKind(\n\tmode: WorkloadMode,\n\trng: () => number,\n): \"insert\" | \"update\" | \"delete\" | \"upsert\" | \"hot\" | \"transfer\" {\n\tconst roll = rng();\n\tif (mode === \"transactions\") {\n\t\tif (roll < 0.55) return \"transfer\";\n\t\tif (roll < 0.75) return \"upsert\";\n\t\tif (roll < 0.9) return \"update\";\n\t\treturn \"delete\";\n\t}\n\tif (mode === \"hot\") {\n\t\tif (roll < 0.6) return \"hot\";\n\t\tif (roll < 0.75) return \"upsert\";\n\t\tif (roll < 0.9) return \"update\";\n\t\treturn \"delete\";\n\t}\n\tif (mode === \"payloads\") {\n\t\tif (roll < 0.4) return \"upsert\";\n\t\tif (roll < 0.7) return \"insert\";\n\t\tif (roll < 0.9) return \"update\";\n\t\treturn \"delete\";\n\t}\n\tif (roll < 0.2) return \"insert\";\n\tif (roll < 0.45) return \"update\";\n\tif (roll < 0.65) return \"delete\";\n\tif (roll < 0.85) return \"upsert\";\n\tif (roll < 0.95) return \"hot\";\n\treturn \"transfer\";\n}\n\nasync function validate(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n): Promise {\n\tconst integrity = await queryOne<{ integrity_check: string }>(\n\t\tdatabase,\n\t\t\"PRAGMA integrity_check\",\n\t);\n\tconst quick = await queryOne<{ quick_check: string }>(\n\t\tdatabase,\n\t\t\"PRAGMA quick_check\",\n\t);\n\tconst totals = await queryOne<{\n\t\ttotal_events: number;\n\t\tactive_rows: number;\n\t\texpected_rows: number;\n\t\tactual_version_sum: number;\n\t\texpected_version_sum: number;\n\t\tactual_payload_checksum_sum: number;\n\t\texpected_payload_checksum_sum: number;\n\t}>(\n\t\tdatabase,\n\t\t`WITH latest AS (\n\t\t\tSELECT e.*\n\t\t\tFROM fuzz_item_events e\n\t\t\tJOIN (\n\t\t\t\tSELECT item_key, MAX(seq) AS seq\n\t\t\t\tFROM fuzz_item_events\n\t\t\t\tGROUP BY item_key\n\t\t\t) m ON m.item_key = e.item_key AND m.seq = e.seq\n\t\t)\n\t\tSELECT\n\t\t\t(SELECT COUNT(*) FROM fuzz_item_events) AS total_events,\n\t\t\t(SELECT COUNT(*) FROM fuzz_items) AS active_rows,\n\t\t\t(SELECT COUNT(*) FROM latest WHERE present = 1) AS expected_rows,\n\t\t\t(SELECT COALESCE(SUM(version), 0) FROM fuzz_items) AS actual_version_sum,\n\t\t\t(SELECT COALESCE(SUM(version), 0) FROM latest WHERE present = 1) AS expected_version_sum,\n\t\t\t(SELECT COALESCE(SUM(payload_checksum), 0) FROM fuzz_items) AS actual_payload_checksum_sum,\n\t\t\t(SELECT COALESCE(SUM(payload_checksum), 0) FROM latest WHERE present = 1) AS expected_payload_checksum_sum`,\n\t);\n\tconst mismatches = await queryOne<{\n\t\tmissing_rows: number;\n\t\textra_rows: number;\n\t\tmismatched_rows: number;\n\t\tduplicate_keys: number;\n\t}>(\n\t\tdatabase,\n\t\t`WITH latest AS (\n\t\t\tSELECT e.*\n\t\t\tFROM fuzz_item_events e\n\t\t\tJOIN (\n\t\t\t\tSELECT item_key, MAX(seq) AS seq\n\t\t\t\tFROM fuzz_item_events\n\t\t\t\tGROUP BY item_key\n\t\t\t) m ON m.item_key = e.item_key AND m.seq = e.seq\n\t\t)\n\t\tSELECT\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM latest l\n\t\t\t\tLEFT JOIN fuzz_items i ON i.item_key = l.item_key\n\t\t\t\tWHERE l.present = 1 AND i.item_key IS NULL\n\t\t\t) AS missing_rows,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_items i\n\t\t\t\tLEFT JOIN latest l ON l.item_key = i.item_key\n\t\t\t\tWHERE l.item_key IS NULL OR l.present = 0\n\t\t\t) AS extra_rows,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM latest l\n\t\t\t\tJOIN fuzz_items i ON i.item_key = l.item_key\n\t\t\t\tWHERE l.present = 1\n\t\t\t\t\tAND (\n\t\t\t\t\t\ti.value != l.value OR\n\t\t\t\t\t\ti.version != l.version OR\n\t\t\t\t\t\ti.update_count != l.update_count OR\n\t\t\t\t\t\ti.payload_checksum != l.payload_checksum OR\n\t\t\t\t\t\ti.payload_bytes != l.payload_bytes\n\t\t\t\t\t)\n\t\t\t) AS mismatched_rows,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM (\n\t\t\t\t\tSELECT item_key\n\t\t\t\t\tFROM fuzz_items\n\t\t\t\t\tGROUP BY item_key\n\t\t\t\t\tHAVING COUNT(*) > 1\n\t\t\t\t)\n\t\t\t) AS duplicate_keys`,\n\t);\n\tconst accounts = await queryOne<{\n\t\taccount_count: number;\n\t\taccount_balance_sum: number;\n\t\taccount_balance_mismatch: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\tCOUNT(*) AS account_count,\n\t\t\tCOALESCE(SUM(balance), 0) AS account_balance_sum,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_transfer_events\n\t\t\t\tWHERE balance_sum_before != ? OR balance_sum_after != ?\n\t\t\t) AS account_balance_mismatch\n\t\tFROM fuzz_accounts`,\n\t\tACCOUNT_COUNT * ACCOUNT_INITIAL_BALANCE,\n\t\tACCOUNT_COUNT * ACCOUNT_INITIAL_BALANCE,\n\t);\n\tconst edge = await queryOne<{\n\t\tedge_rows: number;\n\t\tedge_expected_rows: number;\n\t\tedge_mismatches: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\t(SELECT COUNT(*) FROM fuzz_edge_payloads) AS edge_rows,\n\t\t\t(SELECT COUNT(*) FROM fuzz_edge_expectations WHERE present = 1) AS edge_expected_rows,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_edge_expectations e\n\t\t\t\tLEFT JOIN fuzz_edge_payloads p ON p.id = e.id\n\t\t\t\tWHERE\n\t\t\t\t\t(e.present = 1 AND p.id IS NULL) OR\n\t\t\t\t\t(e.present = 0 AND p.id IS NOT NULL) OR\n\t\t\t\t\t(e.present = 1 AND (\n\t\t\t\t\t\tp.payload_checksum != e.payload_checksum OR\n\t\t\t\t\t\tp.payload_bytes != e.payload_bytes\n\t\t\t\t\t))\n\t\t\t) AS edge_mismatches`,\n\t);\n\tconst indexProbe = await queryOne<{\n\t\tindex_rows: number;\n\t\tindex_mismatches: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\t(SELECT COUNT(*) FROM fuzz_indexed) AS index_rows,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM (\n\t\t\t\t\tSELECT id FROM fuzz_indexed\n\t\t\t\t\tWHERE tenant = 'tenant-1' AND bucket BETWEEN 2 AND 8 AND score BETWEEN -250 AND 250\n\t\t\t\t\tEXCEPT\n\t\t\t\t\tSELECT id FROM fuzz_indexed NOT INDEXED\n\t\t\t\t\tWHERE tenant = 'tenant-1' AND bucket BETWEEN 2 AND 8 AND score BETWEEN -250 AND 250\n\t\t\t\t)\n\t\t\t) + (\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM (\n\t\t\t\t\tSELECT id FROM fuzz_indexed NOT INDEXED\n\t\t\t\t\tWHERE tenant = 'tenant-1' AND bucket BETWEEN 2 AND 8 AND score BETWEEN -250 AND 250\n\t\t\t\t\tEXCEPT\n\t\t\t\t\tSELECT id FROM fuzz_indexed\n\t\t\t\t\tWHERE tenant = 'tenant-1' AND bucket BETWEEN 2 AND 8 AND score BETWEEN -250 AND 250\n\t\t\t\t)\n\t\t\t) AS index_mismatches`,\n\t);\n\tconst relational = await queryOne<{\n\t\trelational_orders: number;\n\t\trelational_mismatches: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\t(SELECT COUNT(*) FROM fuzz_orders) AS relational_orders,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_orders o\n\t\t\t\tLEFT JOIN (\n\t\t\t\t\tSELECT order_id, COALESCE(SUM(quantity * price), 0) AS item_total\n\t\t\t\t\tFROM fuzz_order_items\n\t\t\t\t\tGROUP BY order_id\n\t\t\t\t) i ON i.order_id = o.id\n\t\t\t\tWHERE o.total != COALESCE(i.item_total, 0)\n\t\t\t) + (\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_orders o\n\t\t\t\tLEFT JOIN (\n\t\t\t\t\tSELECT order_id, COALESCE(SUM(amount), 0) AS payment_total\n\t\t\t\t\tFROM fuzz_payments\n\t\t\t\t\tWHERE status = 'captured'\n\t\t\t\t\tGROUP BY order_id\n\t\t\t\t) p ON p.order_id = o.id\n\t\t\t\tWHERE o.status = 'paid' AND o.total != COALESCE(p.payment_total, 0)\n\t\t\t) + (\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_inventory\n\t\t\t\tWHERE initial_qty != sold_qty + stock_qty OR stock_qty < 0\n\t\t\t) AS relational_mismatches`,\n\t);\n\tconst constraints = await queryOne<{\n\t\tconstraint_attempts: number;\n\t\tconstraint_leaks: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\tCOUNT(*) AS constraint_attempts,\n\t\t\tCOALESCE(SUM(\n\t\t\t\tCASE\n\t\t\t\t\tWHEN expected_failed = 1 AND (actually_failed != 1 OR before_count != after_count) THEN 1\n\t\t\t\t\tWHEN expected_failed = 0 AND after_count != before_count + 1 THEN 1\n\t\t\t\t\tELSE 0\n\t\t\t\tEND\n\t\t\t), 0) AS constraint_leaks\n\t\tFROM fuzz_constraint_attempts`,\n\t);\n\tconst savepoints = await queryOne<{\n\t\tsavepoint_rows: number;\n\t\tsavepoint_mismatches: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\t(SELECT COUNT(*) FROM fuzz_savepoints) AS savepoint_rows,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_savepoint_expectations e\n\t\t\t\tLEFT JOIN fuzz_savepoints s ON s.id = e.id\n\t\t\t\tWHERE\n\t\t\t\t\t(e.present = 1 AND s.id IS NULL) OR\n\t\t\t\t\t(e.present = 0 AND s.id IS NOT NULL) OR\n\t\t\t\t\t(e.present = 1 AND s.value != e.value)\n\t\t\t) AS savepoint_mismatches`,\n\t);\n\tconst idempotency = await queryOne<{\n\t\tidempotent_ops: number;\n\t\tidempotent_mismatches: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\t(SELECT COUNT(*) FROM fuzz_idempotent_ops) AS idempotent_ops,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_idempotent_targets t\n\t\t\t\tLEFT JOIN (\n\t\t\t\t\tSELECT target_id, COALESCE(SUM(amount), 0) AS expected\n\t\t\t\t\tFROM fuzz_idempotent_ops\n\t\t\t\t\tGROUP BY target_id\n\t\t\t\t) o ON o.target_id = t.id\n\t\t\t\tWHERE t.value != COALESCE(o.expected, 0)\n\t\t\t) AS idempotent_mismatches`,\n\t);\n\tconst schema = await queryOne<{\n\t\tschema_objects: number;\n\t\tschema_missing_objects: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\tCOUNT(*) AS schema_objects,\n\t\t\tCOALESCE(SUM(CASE WHEN m.name IS NULL THEN 1 ELSE 0 END), 0) AS schema_missing_objects\n\t\tFROM fuzz_schema_registry r\n\t\tLEFT JOIN sqlite_master m ON m.name = r.name AND m.type = r.type`,\n\t);\n\tconst probes = await queryOne<{\n\t\tprobe_rows: number;\n\t\tprobe_mismatches: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\tCOUNT(*) AS probe_rows,\n\t\t\tCOALESCE(SUM(mismatch), 0) AS probe_mismatches\n\t\tFROM fuzz_probe_results`,\n\t);\n\tconst prepared = await queryOne<{\n\t\tprepared_rows: number;\n\t\tprepared_mismatches: number;\n\t}>(\n\t\tdatabase,\n\t\t`SELECT\n\t\t\t(SELECT COUNT(*) FROM fuzz_prepared_churn) AS prepared_rows,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_prepared_expectations e\n\t\t\t\tLEFT JOIN fuzz_prepared_churn p ON p.id = e.id\n\t\t\t\tWHERE\n\t\t\t\t\tp.id IS NULL OR\n\t\t\t\t\tp.value != e.value OR\n\t\t\t\t\tp.payload_checksum != e.payload_checksum\n\t\t\t) AS prepared_mismatches`,\n\t);\n\tconst shadow = await queryOne<{\n\t\tshadow_rows: number;\n\t\tshadow_mismatches: number;\n\t}>(\n\t\tdatabase,\n\t\t`WITH recomputed AS (\n\t\t\tSELECT 'items' AS name,\n\t\t\t\tCOUNT(*) AS row_count,\n\t\t\t\tCOALESCE(SUM(payload_checksum + version + update_count), 0) AS value\n\t\t\tFROM fuzz_items\n\t\t\tUNION ALL\n\t\t\tSELECT 'edge' AS name,\n\t\t\t\tCOUNT(*) AS row_count,\n\t\t\t\tCOALESCE(SUM(payload_checksum + payload_bytes), 0) AS value\n\t\t\tFROM fuzz_edge_payloads\n\t\t)\n\t\tSELECT\n\t\t\t(SELECT COUNT(*) FROM fuzz_shadow_checksums) AS shadow_rows,\n\t\t\t(\n\t\t\t\tSELECT COUNT(*)\n\t\t\t\tFROM fuzz_shadow_checksums s\n\t\t\t\tJOIN recomputed r ON r.name = s.name\n\t\t\t\tWHERE s.value != r.value OR s.row_count != r.row_count\n\t\t\t) AS shadow_mismatches`,\n\t);\n\n\tconst summary: ValidationSummary = {\n\t\ttotalEvents: totals?.total_events ?? 0,\n\t\tactiveRows: totals?.active_rows ?? 0,\n\t\texpectedRows: totals?.expected_rows ?? 0,\n\t\tmissingRows: mismatches?.missing_rows ?? 0,\n\t\textraRows: mismatches?.extra_rows ?? 0,\n\t\tmismatchedRows: mismatches?.mismatched_rows ?? 0,\n\t\tduplicateKeys: mismatches?.duplicate_keys ?? 0,\n\t\tactualVersionSum: totals?.actual_version_sum ?? 0,\n\t\texpectedVersionSum: totals?.expected_version_sum ?? 0,\n\t\tactualPayloadChecksumSum: totals?.actual_payload_checksum_sum ?? 0,\n\t\texpectedPayloadChecksumSum: totals?.expected_payload_checksum_sum ?? 0,\n\t\taccountCount: accounts?.account_count ?? 0,\n\t\taccountBalanceSum: accounts?.account_balance_sum ?? 0,\n\t\texpectedAccountBalanceSum: ACCOUNT_COUNT * ACCOUNT_INITIAL_BALANCE,\n\t\taccountBalanceMismatch: accounts?.account_balance_mismatch ?? 0,\n\t\tintegrityCheck: integrity?.integrity_check ?? \"missing\",\n\t\tquickCheck: quick?.quick_check ?? \"missing\",\n\t\tedgeRows: edge?.edge_rows ?? 0,\n\t\tedgeExpectedRows: edge?.edge_expected_rows ?? 0,\n\t\tedgeMismatches: edge?.edge_mismatches ?? 0,\n\t\tindexRows: indexProbe?.index_rows ?? 0,\n\t\tindexMismatches: indexProbe?.index_mismatches ?? 0,\n\t\trelationalOrders: relational?.relational_orders ?? 0,\n\t\trelationalMismatches: relational?.relational_mismatches ?? 0,\n\t\tconstraintAttempts: constraints?.constraint_attempts ?? 0,\n\t\tconstraintLeaks: constraints?.constraint_leaks ?? 0,\n\t\tsavepointRows: savepoints?.savepoint_rows ?? 0,\n\t\tsavepointMismatches: savepoints?.savepoint_mismatches ?? 0,\n\t\tidempotentOps: idempotency?.idempotent_ops ?? 0,\n\t\tidempotentMismatches: idempotency?.idempotent_mismatches ?? 0,\n\t\tschemaObjects: schema?.schema_objects ?? 0,\n\t\tschemaMissingObjects: schema?.schema_missing_objects ?? 0,\n\t\tprobeRows: probes?.probe_rows ?? 0,\n\t\tprobeMismatches: probes?.probe_mismatches ?? 0,\n\t\tpreparedRows: prepared?.prepared_rows ?? 0,\n\t\tpreparedMismatches: prepared?.prepared_mismatches ?? 0,\n\t\tshadowRows: shadow?.shadow_rows ?? 0,\n\t\tshadowMismatches: shadow?.shadow_mismatches ?? 0,\n\t};\n\n\treturn summary;\n}\n\nasync function debugItemMismatches(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tlimit = 5,\n): Promise<{\n\titemMismatches: ItemMismatchDebugRow[];\n\trecentEventsByKey: Record;\n}> {\n\tconst rows = (await database.execute(\n\t\t`WITH latest AS (\n\t\t\tSELECT e.*\n\t\t\tFROM fuzz_item_events e\n\t\t\tJOIN (\n\t\t\t\tSELECT item_key, MAX(seq) AS seq\n\t\t\t\tFROM fuzz_item_events\n\t\t\t\tGROUP BY item_key\n\t\t\t) m ON m.item_key = e.item_key AND m.seq = e.seq\n\t\t)\n\t\tSELECT\n\t\t\tCOALESCE(l.item_key, i.item_key) AS item_key,\n\t\t\ti.value AS actual_value,\n\t\t\tl.value AS expected_value,\n\t\t\ti.version AS actual_version,\n\t\t\tl.version AS expected_version,\n\t\t\ti.update_count AS actual_update_count,\n\t\t\tl.update_count AS expected_update_count,\n\t\t\ti.payload_checksum AS actual_payload_checksum,\n\t\t\tl.payload_checksum AS expected_payload_checksum,\n\t\t\ti.payload_bytes AS actual_payload_bytes,\n\t\t\tl.payload_bytes AS expected_payload_bytes\n\t\tFROM latest l\n\t\tFULL OUTER JOIN fuzz_items i ON i.item_key = l.item_key\n\t\tWHERE\n\t\t\t(l.present = 1 AND i.item_key IS NULL) OR\n\t\t\t((l.item_key IS NULL OR l.present = 0) AND i.item_key IS NOT NULL) OR\n\t\t\t(\n\t\t\t\tl.present = 1 AND i.item_key IS NOT NULL AND (\n\t\t\t\t\ti.value != l.value OR\n\t\t\t\t\ti.version != l.version OR\n\t\t\t\t\ti.update_count != l.update_count OR\n\t\t\t\t\ti.payload_checksum != l.payload_checksum OR\n\t\t\t\t\ti.payload_bytes != l.payload_bytes\n\t\t\t\t)\n\t\t\t)\n\t\tORDER BY COALESCE(l.item_key, i.item_key)\n\t\tLIMIT ?`,\n\t\tlimit,\n\t)) as Array<{\n\t\titem_key: string;\n\t\tactual_value: string | null;\n\t\texpected_value: string | null;\n\t\tactual_version: number | null;\n\t\texpected_version: number | null;\n\t\tactual_update_count: number | null;\n\t\texpected_update_count: number | null;\n\t\tactual_payload_checksum: number | null;\n\t\texpected_payload_checksum: number | null;\n\t\tactual_payload_bytes: number | null;\n\t\texpected_payload_bytes: number | null;\n\t}>;\n\n\tconst itemMismatches = rows.map((row) => ({\n\t\titemKey: row.item_key,\n\t\tactualValue: row.actual_value,\n\t\texpectedValue: row.expected_value,\n\t\tactualVersion: row.actual_version,\n\t\texpectedVersion: row.expected_version,\n\t\tactualUpdateCount: row.actual_update_count,\n\t\texpectedUpdateCount: row.expected_update_count,\n\t\tactualPayloadChecksum: row.actual_payload_checksum,\n\t\texpectedPayloadChecksum: row.expected_payload_checksum,\n\t\tactualPayloadBytes: row.actual_payload_bytes,\n\t\texpectedPayloadBytes: row.expected_payload_bytes,\n\t}));\n\n\tconst recentEventsByKey: Record = {};\n\tfor (const row of itemMismatches) {\n\t\tconst events = (await database.execute(\n\t\t\t`SELECT\n\t\t\t\tseq,\n\t\t\t\tphase,\n\t\t\t\tlocal_index,\n\t\t\t\tkind,\n\t\t\t\tpresent,\n\t\t\t\tvalue,\n\t\t\t\tversion,\n\t\t\t\tupdate_count,\n\t\t\t\tpayload_checksum,\n\t\t\t\tpayload_bytes,\n\t\t\t\tapplied\n\t\t\tFROM fuzz_item_events\n\t\t\tWHERE item_key = ?\n\t\t\tORDER BY seq DESC\n\t\t\tLIMIT 10`,\n\t\t\trow.itemKey,\n\t\t)) as Array<{\n\t\t\tseq: number;\n\t\t\tphase: number;\n\t\t\tlocal_index: number;\n\t\t\tkind: string;\n\t\t\tpresent: number;\n\t\t\tvalue: string | null;\n\t\t\tversion: number;\n\t\t\tupdate_count: number;\n\t\t\tpayload_checksum: number;\n\t\t\tpayload_bytes: number;\n\t\t\tapplied: number;\n\t\t}>;\n\t\trecentEventsByKey[row.itemKey] = events.map((event) => ({\n\t\t\tseq: event.seq,\n\t\t\tphase: event.phase,\n\t\t\tlocalIndex: event.local_index,\n\t\t\tkind: event.kind,\n\t\t\tpresent: event.present,\n\t\t\tvalue: event.value,\n\t\t\tversion: event.version,\n\t\t\tupdateCount: event.update_count,\n\t\t\tpayloadChecksum: event.payload_checksum,\n\t\t\tpayloadBytes: event.payload_bytes,\n\t\t\tapplied: event.applied,\n\t\t}));\n\t}\n\n\treturn { itemMismatches, recentEventsByKey };\n}\n\nexport const rawSqliteFuzzer = actor({\n\toptions: {\n\t\tactionTimeout: 300_000,\n\t},\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_items (\n\t\t\t\t\titem_key TEXT PRIMARY KEY,\n\t\t\t\t\tvalue TEXT NOT NULL,\n\t\t\t\t\tversion INTEGER NOT NULL,\n\t\t\t\t\tupdate_count INTEGER NOT NULL,\n\t\t\t\t\tpayload TEXT NOT NULL,\n\t\t\t\t\tpayload_checksum INTEGER NOT NULL,\n\t\t\t\t\tpayload_bytes INTEGER NOT NULL,\n\t\t\t\t\tupdated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_item_events (\n\t\t\t\t\tseq INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tphase INTEGER NOT NULL,\n\t\t\t\t\tlocal_index INTEGER NOT NULL,\n\t\t\t\t\tkind TEXT NOT NULL,\n\t\t\t\t\titem_key TEXT NOT NULL,\n\t\t\t\t\tpresent INTEGER NOT NULL,\n\t\t\t\t\tvalue TEXT,\n\t\t\t\t\tversion INTEGER NOT NULL,\n\t\t\t\t\tupdate_count INTEGER NOT NULL,\n\t\t\t\t\tpayload_checksum INTEGER NOT NULL,\n\t\t\t\t\tpayload_bytes INTEGER NOT NULL,\n\t\t\t\t\tapplied INTEGER NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_fuzz_item_events_key_seq ON fuzz_item_events(item_key, seq)\",\n\t\t\t);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_accounts (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tbalance INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_transfer_events (\n\t\t\t\t\tseq INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tphase INTEGER NOT NULL,\n\t\t\t\t\tlocal_index INTEGER NOT NULL,\n\t\t\t\t\tfrom_account TEXT NOT NULL,\n\t\t\t\t\tto_account TEXT NOT NULL,\n\t\t\t\t\tamount INTEGER NOT NULL,\n\t\t\t\t\tbalance_sum_before INTEGER NOT NULL,\n\t\t\t\t\tbalance_sum_after INTEGER NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_edge_payloads (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tkind TEXT NOT NULL,\n\t\t\t\t\tpayload TEXT NOT NULL,\n\t\t\t\t\tpayload_checksum INTEGER NOT NULL,\n\t\t\t\t\tpayload_bytes INTEGER NOT NULL,\n\t\t\t\t\tupdated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_edge_expectations (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tpresent INTEGER NOT NULL,\n\t\t\t\t\tpayload_checksum INTEGER NOT NULL,\n\t\t\t\t\tpayload_bytes INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_trigger_audit (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tpayload_id TEXT NOT NULL,\n\t\t\t\t\told_checksum INTEGER NOT NULL,\n\t\t\t\t\tnew_checksum INTEGER NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_fuzz_edge_kind_size ON fuzz_edge_payloads(kind, payload_bytes)\",\n\t\t\t);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_indexed (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\ttenant TEXT NOT NULL,\n\t\t\t\t\tbucket INTEGER NOT NULL,\n\t\t\t\t\tscore INTEGER NOT NULL,\n\t\t\t\t\tlabel TEXT NOT NULL,\n\t\t\t\t\tpayload TEXT NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_fuzz_indexed_tenant_bucket_score ON fuzz_indexed(tenant, bucket, score)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_fuzz_indexed_score_label ON fuzz_indexed(score, label)\",\n\t\t\t);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_rel_users (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tname TEXT NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_rel_products (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tprice INTEGER NOT NULL CHECK (price >= 0)\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_orders (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tuser_id TEXT NOT NULL,\n\t\t\t\t\ttotal INTEGER NOT NULL,\n\t\t\t\t\tstatus TEXT NOT NULL,\n\t\t\t\t\tFOREIGN KEY (user_id) REFERENCES fuzz_rel_users(id)\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_order_items (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\torder_id TEXT NOT NULL,\n\t\t\t\t\tproduct_id TEXT NOT NULL,\n\t\t\t\t\tquantity INTEGER NOT NULL CHECK (quantity > 0),\n\t\t\t\t\tprice INTEGER NOT NULL CHECK (price >= 0),\n\t\t\t\t\tFOREIGN KEY (order_id) REFERENCES fuzz_orders(id),\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES fuzz_rel_products(id)\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_payments (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\torder_id TEXT NOT NULL,\n\t\t\t\t\tamount INTEGER NOT NULL,\n\t\t\t\t\tstatus TEXT NOT NULL,\n\t\t\t\t\tFOREIGN KEY (order_id) REFERENCES fuzz_orders(id)\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_inventory (\n\t\t\t\t\tproduct_id TEXT PRIMARY KEY,\n\t\t\t\t\tinitial_qty INTEGER NOT NULL,\n\t\t\t\t\tsold_qty INTEGER NOT NULL,\n\t\t\t\t\tstock_qty INTEGER NOT NULL,\n\t\t\t\t\tFOREIGN KEY (product_id) REFERENCES fuzz_rel_products(id)\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_constraints (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tmust_not_null TEXT NOT NULL,\n\t\t\t\t\tqty INTEGER NOT NULL CHECK (qty >= 0),\n\t\t\t\t\tuniq TEXT NOT NULL UNIQUE\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_constraint_attempts (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tname TEXT NOT NULL,\n\t\t\t\t\texpected_failed INTEGER NOT NULL,\n\t\t\t\t\tactually_failed INTEGER NOT NULL,\n\t\t\t\t\tbefore_count INTEGER NOT NULL,\n\t\t\t\t\tafter_count INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_fk_parent (\n\t\t\t\t\tid TEXT PRIMARY KEY\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_fk_child (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tparent_id TEXT NOT NULL,\n\t\t\t\t\tFOREIGN KEY (parent_id) REFERENCES fuzz_fk_parent(id) ON DELETE CASCADE\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_savepoints (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tvalue INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_savepoint_expectations (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tpresent INTEGER NOT NULL,\n\t\t\t\t\tvalue INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_idempotent_ops (\n\t\t\t\t\top_id TEXT PRIMARY KEY,\n\t\t\t\t\ttarget_id TEXT NOT NULL,\n\t\t\t\t\tamount INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_idempotent_targets (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tvalue INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_schema_registry (\n\t\t\t\t\tname TEXT PRIMARY KEY,\n\t\t\t\t\ttype TEXT NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_probe_results (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tphase INTEGER NOT NULL,\n\t\t\t\t\tscenario TEXT NOT NULL,\n\t\t\t\t\tname TEXT NOT NULL,\n\t\t\t\t\texpected TEXT NOT NULL,\n\t\t\t\t\tactual TEXT NOT NULL,\n\t\t\t\t\tmismatch INTEGER NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_prepared_churn (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tvalue INTEGER NOT NULL,\n\t\t\t\t\tpayload TEXT NOT NULL,\n\t\t\t\t\tpayload_checksum INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_prepared_expectations (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tvalue INTEGER NOT NULL,\n\t\t\t\t\tpayload_checksum INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_shadow_checksums (\n\t\t\t\t\tname TEXT PRIMARY KEY,\n\t\t\t\t\tvalue INTEGER NOT NULL,\n\t\t\t\t\trow_count INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_nasty_counter (\n\t\t\t\t\tid TEXT PRIMARY KEY,\n\t\t\t\t\tvalue INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS fuzz_nasty_rows (\n\t\t\t\t\tgroup_id TEXT NOT NULL,\n\t\t\t\t\tn INTEGER NOT NULL,\n\t\t\t\t\tpayload TEXT NOT NULL,\n\t\t\t\t\tPRIMARY KEY (group_id, n)\n\t\t\t\t)\n\t\t\t`);\n\t\t},\n\t}),\n\tactions: {\n\t\treset: async (c) => {\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_nasty_rows\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_nasty_counter\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_shadow_checksums\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_prepared_expectations\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_prepared_churn\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_probe_results\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_schema_registry\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_idempotent_targets\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_idempotent_ops\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_savepoint_expectations\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_savepoints\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_fk_child\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_fk_parent\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_constraint_attempts\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_constraints\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_payments\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_order_items\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_orders\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_inventory\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_rel_products\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_rel_users\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_indexed\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_trigger_audit\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_edge_expectations\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_edge_payloads\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_transfer_events\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_accounts\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_item_events\");\n\t\t\tawait c.db.execute(\"DELETE FROM fuzz_items\");\n\t\t\tawait ensureAccounts(c.db);\n\t\t\treturn await validate(c.db);\n\t\t},\n\n\t\trunPhase: async (c, input: RunPhaseInput): Promise => {\n\t\t\tconst mode = input.mode ?? \"balanced\";\n\t\t\tconst iterations = Math.max(1, Math.floor(input.iterations));\n\t\t\tconst keySpace = Math.max(1, Math.floor(input.keySpace ?? DEFAULT_KEY_SPACE));\n\t\t\tconst maxPayloadBytes = Math.max(\n\t\t\t\t1,\n\t\t\t\tMath.floor(input.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES),\n\t\t\t);\n\t\t\tconst growthTargetBytes = Math.max(\n\t\t\t\t1,\n\t\t\t\tMath.floor(input.growthTargetBytes ?? DEFAULT_GROWTH_TARGET_BYTES),\n\t\t\t);\n\t\t\tconst rng = makeRng(`${input.seed}:${input.phase}:${mode}`);\n\t\t\tconst ops: Record = {};\n\t\t\tlet stage = \"ensureAccounts\";\n\n\t\t\ttry {\n\t\t\t\tawait ensureAccounts(c.db);\n\n\t\t\t\tfor (let i = 0; i < iterations; i += 1) {\n\t\t\t\t\tconst kind = chooseKind(mode, rng);\n\t\t\t\t\tops[kind] = (ops[kind] ?? 0) + 1;\n\t\t\t\t\tstage = `base:${kind}:iteration:${i}`;\n\n\t\t\t\t\tif (kind === \"transfer\") {\n\t\t\t\t\t\tconst fromIndex = intBetween(rng, 0, ACCOUNT_COUNT - 1);\n\t\t\t\t\t\tlet toIndex = intBetween(rng, 0, ACCOUNT_COUNT - 1);\n\t\t\t\t\t\tif (toIndex === fromIndex) toIndex = (toIndex + 1) % ACCOUNT_COUNT;\n\t\t\t\t\t\tconst fromAccount = `acct-${fromIndex}`;\n\t\t\t\t\t\tconst toAccount = `acct-${toIndex}`;\n\t\t\t\t\t\ttry {\n\t\t\t\t\t\t\tawait applyTransfer(c.db, {\n\t\t\t\t\t\t\t\tphase: input.phase,\n\t\t\t\t\t\t\t\tlocalIndex: i,\n\t\t\t\t\t\t\t\tfromAccount,\n\t\t\t\t\t\t\t\ttoAccount,\n\t\t\t\t\t\t\t\tamount: intBetween(rng, 1, 500),\n\t\t\t\t\t\t\t});\n\t\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\t\tthrow new Error(\n\t\t\t\t\t\t\t\t`base operation transfer failed at iteration ${i} from ${fromAccount} to ${toAccount}`,\n\t\t\t\t\t\t\t\t{ cause: error },\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if (kind === \"hot\") {\n\t\t\t\t\t\tconst itemKey = `hot-${intBetween(rng, 0, 3)}`;\n\t\t\t\t\t\tconst updates = intBetween(rng, 2, mode === \"hot\" ? 12 : 5);\n\t\t\t\t\t\ttry {\n\t\t\t\t\t\t\tawait applyHotUpdates(c.db, {\n\t\t\t\t\t\t\t\tseed: input.seed,\n\t\t\t\t\t\t\t\tphase: input.phase,\n\t\t\t\t\t\t\t\tlocalIndex: i,\n\t\t\t\t\t\t\t\titemKey,\n\t\t\t\t\t\t\t\tupdates,\n\t\t\t\t\t\t\t\tpayloadBytes: intBetween(rng, 1, maxPayloadBytes),\n\t\t\t\t\t\t\t});\n\t\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\t\tthrow new Error(\n\t\t\t\t\t\t\t\t`base operation hot failed at iteration ${i} for ${itemKey} with ${updates} updates`,\n\t\t\t\t\t\t\t\t{ cause: error },\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tconst itemKey =\n\t\t\t\t\t\t\tmode === \"hot\" && rng() < 0.6\n\t\t\t\t\t\t\t\t? `hot-${intBetween(rng, 0, 3)}`\n\t\t\t\t\t\t\t\t: `item-${intBetween(rng, 0, keySpace - 1)}`;\n\t\t\t\t\t\tconst payloadBytes =\n\t\t\t\t\t\t\tmode === \"payloads\"\n\t\t\t\t\t\t\t\t? intBetween(rng, Math.min(256, maxPayloadBytes), maxPayloadBytes)\n\t\t\t\t\t\t\t\t: intBetween(rng, 1, maxPayloadBytes);\n\t\t\t\t\t\ttry {\n\t\t\t\t\t\t\tawait applyItemOperation(c.db, {\n\t\t\t\t\t\t\t\tseed: input.seed,\n\t\t\t\t\t\t\t\tphase: input.phase,\n\t\t\t\t\t\t\t\tlocalIndex: i,\n\t\t\t\t\t\t\t\tkind,\n\t\t\t\t\t\t\t\titemKey,\n\t\t\t\t\t\t\t\tpayloadBytes,\n\t\t\t\t\t\t\t});\n\t\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\t\tthrow new Error(\n\t\t\t\t\t\t\t\t`base operation ${kind} failed at iteration ${i} for ${JSON.stringify(itemKey)} with payloadBytes ${payloadBytes}`,\n\t\t\t\t\t\t\t\t{ cause: error },\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tstage = \"deep-scenarios\";\n\t\t\t\tawait applyDeepScenarios(c.db, {\n\t\t\t\t\tseed: input.seed,\n\t\t\t\t\tphase: input.phase,\n\t\t\t\t\tmode,\n\t\t\t\t\titerations,\n\t\t\t\t\trng,\n\t\t\t\t\tmaxPayloadBytes,\n\t\t\t\t\tgrowthTargetBytes,\n\t\t\t\t\tops,\n\t\t\t\t});\n\n\t\t\t\tstage = \"validate\";\n\t\t\t\treturn {\n\t\t\t\t\tseed: input.seed,\n\t\t\t\t\tphase: input.phase,\n\t\t\t\t\tmode,\n\t\t\t\t\titerations,\n\t\t\t\t\tops,\n\t\t\t\t\tvalidation: await validate(c.db),\n\t\t\t\t};\n\t\t\t} catch (error) {\n\t\t\t\tconst detail =\n\t\t\t\t\terror instanceof Error ? error.message : typeof error === \"string\" ? error : String(error);\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`runPhase failed during ${stage} for mode ${mode} phase ${input.phase} seed ${input.seed}: ${detail}`,\n\t\t\t\t\t{ cause: error },\n\t\t\t\t);\n\t\t\t}\n\t\t},\n\n\t\tvalidate: async (c) => {\n\t\t\tawait ensureAccounts(c.db);\n\t\t\treturn await validate(c.db);\n\t\t},\n\n\t\tdebugItemMismatches: async (c, limit?: number) => {\n\t\t\tawait ensureAccounts(c.db);\n\t\t\treturn await debugItemMismatches(c.db, limit ?? 5);\n\t\t},\n\n\t\tgoToSleep: (c) => {\n\t\t\tc.sleep();\n\t\t\treturn { ok: true };\n\t\t},\n\t},\n});\n","import { actor } from \"rivetkit\";\nimport type { SqliteNativeMetrics } from \"rivetkit/db\";\nimport { db } from \"rivetkit/db\";\n\ninterface RunCycleInput {\n\tseed: string;\n\tcycle: number;\n\tinsertRows?: number;\n\trowBytes?: number;\n\tdeleteRows?: number;\n\tretainRows?: number;\n\tscanRows?: number;\n}\n\ninterface CountRow {\n\tcount: number;\n}\n\ninterface StorageRow {\n\tpage_count: number;\n\tfreelist_count: number;\n\tpage_size: number;\n\tvfs: SqliteNativeMetrics | null;\n}\n\nconst DEFAULT_INSERT_ROWS = 128;\nconst DEFAULT_ROW_BYTES = 16 * 1024;\n// const DEFAULT_DELETE_ROWS = 64;\n// const DEFAULT_RETAIN_ROWS = 1024;\nconst DEFAULT_SCAN_ROWS = 512;\nconst INSERT_BATCH_ROWS = 32;\n\nfunction finiteInt(value: number | undefined, fallback: number): number {\n\tif (value === undefined) return fallback;\n\tif (!Number.isFinite(value) || value < 0) {\n\t\tthrow new Error(`expected a non-negative finite number, got ${value}`);\n\t}\n\treturn Math.floor(value);\n}\n\nfunction copyNativeMetrics(\n\tmetrics: SqliteNativeMetrics | null | undefined,\n): SqliteNativeMetrics | null {\n\tif (!metrics) return null;\n\tconst raw = metrics as unknown as Record;\n\tconst numberField = (camel: string, snake: string) =>\n\t\tNumber(raw[camel] ?? raw[snake] ?? 0);\n\treturn {\n\t\trequestBuildNs: numberField(\"requestBuildNs\", \"request_build_ns\"),\n\t\tserializeNs: numberField(\"serializeNs\", \"serialize_ns\"),\n\t\ttransportNs: numberField(\"transportNs\", \"transport_ns\"),\n\t\tstateUpdateNs: numberField(\"stateUpdateNs\", \"state_update_ns\"),\n\t\ttotalNs: numberField(\"totalNs\", \"total_ns\"),\n\t\tcommitCount: numberField(\"commitCount\", \"commit_count\"),\n\t\tpageCacheEntries: numberField(\"pageCacheEntries\", \"page_cache_entries\"),\n\t\tpageCacheWeightedSize: numberField(\n\t\t\t\"pageCacheWeightedSize\",\n\t\t\t\"page_cache_weighted_size\",\n\t\t),\n\t\tpageCacheCapacityPages: numberField(\n\t\t\t\"pageCacheCapacityPages\",\n\t\t\t\"page_cache_capacity_pages\",\n\t\t),\n\t\twriteBufferDirtyPages: numberField(\n\t\t\t\"writeBufferDirtyPages\",\n\t\t\t\"write_buffer_dirty_pages\",\n\t\t),\n\t\tdbSizePages: numberField(\"dbSizePages\", \"db_size_pages\"),\n\t};\n}\n\nasync function queryOne(\n\tdatabase: { execute: (sql: string, ...args: unknown[]) => Promise },\n\tsql: string,\n\t...args: unknown[]\n): Promise {\n\tconst rows = await database.execute(sql, ...args);\n\tif (!rows[0]) throw new Error(`query returned no rows: ${sql}`);\n\treturn rows[0] as T;\n}\n\nasync function storageStats(database: {\n\texecute: (sql: string, ...args: unknown[]) => Promise;\n\tnativeMetrics?: () =>\n\t\t| SqliteNativeMetrics\n\t\t| Promise\n\t\t| null;\n}): Promise {\n\tconst [pageCount, freelistCount, pageSize] = await Promise.all([\n\t\tqueryOne<{ page_count: number }>(database, \"PRAGMA page_count\"),\n\t\tqueryOne<{ freelist_count: number }>(database, \"PRAGMA freelist_count\"),\n\t\tqueryOne<{ page_size: number }>(database, \"PRAGMA page_size\"),\n\t]);\n\n\tconst nativeMetrics = await database.nativeMetrics?.();\n\tconst copiedMetrics = copyNativeMetrics(nativeMetrics);\n\n\treturn {\n\t\tpage_count: pageCount.page_count,\n\t\tfreelist_count: freelistCount.freelist_count,\n\t\tpage_size: pageSize.page_size,\n\t\tvfs: copiedMetrics,\n\t};\n}\n\nexport const sqliteMemoryPressure = actor({\n\toptions: {\n\t\tactionTimeout: 300_000,\n\t},\n\tstate: {\n\t\tsleepCount: 0,\n\t},\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS pressure_rows (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tseed TEXT NOT NULL,\n\t\t\t\t\tcycle INTEGER NOT NULL,\n\t\t\t\t\tbucket INTEGER NOT NULL,\n\t\t\t\t\tpayload BLOB NOT NULL,\n\t\t\t\t\ttouched_count INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_pressure_rows_seed_cycle ON pressure_rows(seed, cycle)\",\n\t\t\t);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_pressure_rows_bucket ON pressure_rows(bucket)\",\n\t\t\t);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS pressure_cycles (\n\t\t\t\t\tcycle INTEGER PRIMARY KEY,\n\t\t\t\t\tseed TEXT NOT NULL,\n\t\t\t\t\tinserted_rows INTEGER NOT NULL,\n\t\t\t\t\tdeleted_rows INTEGER NOT NULL,\n\t\t\t\t\tactive_rows INTEGER NOT NULL,\n\t\t\t\t\tactive_bytes INTEGER NOT NULL,\n\t\t\t\t\tduration_ms REAL NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t},\n\t}),\n\tonSleep: (c) => {\n\t\tc.state.sleepCount += 1;\n\t\tconsole.log(\n\t\t\tJSON.stringify({\n\t\t\t\tkind: \"sqlite_memory_pressure_on_sleep\",\n\t\t\t\tactorId: c.actorId,\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t\ttimestamp: new Date().toISOString(),\n\t\t\t}),\n\t\t);\n\t},\n\tactions: {\n\t\treset: async (c) => {\n\t\t\tawait c.db.execute(\"DELETE FROM pressure_cycles\");\n\t\t\tawait c.db.execute(\"DELETE FROM pressure_rows\");\n\t\t\tawait c.db.execute(\"VACUUM\");\n\t\t\treturn {\n\t\t\t\tok: true,\n\t\t\t\tstorage: await storageStats(c.db),\n\t\t\t};\n\t\t},\n\n\t\tgoToSleep: (c) => {\n\t\t\tc.sleep();\n\t\t\treturn { ok: true };\n\t\t},\n\n\t\treleaseStorage: async (c) => {\n\t\t\tconst before = await storageStats(c.db);\n\t\t\t// Keep the remote DB large for the sleep reclamation soak.\n\t\t\t// await c.db.execute(\"DELETE FROM pressure_cycles\");\n\t\t\t// await c.db.execute(\"DELETE FROM pressure_rows\");\n\t\t\t// await c.db.execute(\"VACUUM\");\n\t\t\treturn {\n\t\t\t\tok: true,\n\t\t\t\tbefore,\n\t\t\t\tafter: await storageStats(c.db),\n\t\t\t};\n\t\t},\n\n\t\tstats: async (c) => {\n\t\t\tconst rowStats = await queryOne<{\n\t\t\t\tactive_rows: number;\n\t\t\t\tactive_bytes: number | null;\n\t\t\t\ttouched_sum: number | null;\n\t\t\t}>(\n\t\t\t\tc.db,\n\t\t\t\t\"SELECT COUNT(*) AS active_rows, COALESCE(SUM(length(payload)), 0) AS active_bytes, COALESCE(SUM(touched_count), 0) AS touched_sum FROM pressure_rows\",\n\t\t\t);\n\t\t\tconst cycles = await queryOne(\n\t\t\t\tc.db,\n\t\t\t\t\"SELECT COUNT(*) AS count FROM pressure_cycles\",\n\t\t\t);\n\t\t\tconst integrity = await queryOne<{ integrity_check: string }>(\n\t\t\t\tc.db,\n\t\t\t\t\"PRAGMA integrity_check\",\n\t\t\t);\n\n\t\t\treturn {\n\t\t\t\tactiveRows: rowStats.active_rows,\n\t\t\t\tactiveBytes: rowStats.active_bytes ?? 0,\n\t\t\t\ttouchedCount: rowStats.touched_sum ?? 0,\n\t\t\t\tcycles: cycles.count,\n\t\t\t\tintegrityCheck: integrity.integrity_check,\n\t\t\t\tstorage: await storageStats(c.db),\n\t\t\t};\n\t\t},\n\n\t\trunCycle: async (c, input: RunCycleInput) => {\n\t\t\tconst startedAt = performance.now();\n\t\t\tconst insertRows = finiteInt(input.insertRows, DEFAULT_INSERT_ROWS);\n\t\t\tconst rowBytes = finiteInt(input.rowBytes, DEFAULT_ROW_BYTES);\n\t\t\t// const deleteRows = finiteInt(input.deleteRows, DEFAULT_DELETE_ROWS);\n\t\t\t// const retainRows = Math.max(\n\t\t\t// \t1,\n\t\t\t// \tfiniteInt(input.retainRows, DEFAULT_RETAIN_ROWS),\n\t\t\t// );\n\t\t\tconst scanRows = Math.max(1, finiteInt(input.scanRows, DEFAULT_SCAN_ROWS));\n\t\t\tconst now = Date.now();\n\t\t\tlet insertedRows = 0;\n\t\t\tconst logStage = (\n\t\t\t\tstage: string,\n\t\t\t\tphase: \"start\" | \"end\" | \"error\",\n\t\t\t\tfields: Record = {},\n\t\t\t) => {\n\t\t\t\tconsole.log(\n\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\tkind: \"sqlite_memory_pressure_run_cycle_stage\",\n\t\t\t\t\t\tactorId: c.actorId,\n\t\t\t\t\t\tseed: input.seed,\n\t\t\t\t\t\tcycle: input.cycle,\n\t\t\t\t\t\tstage,\n\t\t\t\t\t\tphase,\n\t\t\t\t\t\telapsedMs: performance.now() - startedAt,\n\t\t\t\t\t\ttimestamp: new Date().toISOString(),\n\t\t\t\t\t\t...fields,\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t};\n\t\t\tconst executeTimed = async (\n\t\t\t\tstage: string,\n\t\t\t\tsql: string,\n\t\t\t\t...args: unknown[]\n\t\t\t) => {\n\t\t\t\tconst stageStartedAt = performance.now();\n\t\t\t\tlogStage(stage, \"start\", { argCount: args.length });\n\t\t\t\ttry {\n\t\t\t\t\tconst rows = await c.db.execute(sql, ...args);\n\t\t\t\t\tlogStage(stage, \"end\", {\n\t\t\t\t\t\tdurationMs: performance.now() - stageStartedAt,\n\t\t\t\t\t\trowCount: rows.length,\n\t\t\t\t\t});\n\t\t\t\t\treturn rows;\n\t\t\t\t} catch (err) {\n\t\t\t\t\tlogStage(stage, \"error\", {\n\t\t\t\t\t\tdurationMs: performance.now() - stageStartedAt,\n\t\t\t\t\t\terror: err instanceof Error ? err.message : String(err),\n\t\t\t\t\t});\n\t\t\t\t\tthrow err;\n\t\t\t\t}\n\t\t\t};\n\t\t\tlogStage(\"run_cycle\", \"start\", {\n\t\t\t\tinsertRows,\n\t\t\t\trowBytes,\n\t\t\t\tscanRows,\n\t\t\t});\n\n\t\t\tawait executeTimed(\"begin\", \"BEGIN\");\n\t\t\ttry {\n\t\t\t\twhile (insertedRows < insertRows) {\n\t\t\t\t\tconst batchRows = Math.min(\n\t\t\t\t\t\tINSERT_BATCH_ROWS,\n\t\t\t\t\t\tinsertRows - insertedRows,\n\t\t\t\t\t);\n\t\t\t\t\tconst placeholders: string[] = [];\n\t\t\t\t\tconst args: unknown[] = [];\n\n\t\t\t\t\tfor (let i = 0; i < batchRows; i += 1) {\n\t\t\t\t\t\tconst rowIndex = insertedRows + i;\n\t\t\t\t\t\tplaceholders.push(\"(?, ?, ?, randomblob(?), 0, ?)\");\n\t\t\t\t\t\targs.push(\n\t\t\t\t\t\t\tinput.seed,\n\t\t\t\t\t\t\tinput.cycle,\n\t\t\t\t\t\t\t(input.cycle + rowIndex) % 32,\n\t\t\t\t\t\t\trowBytes,\n\t\t\t\t\t\t\tnow + rowIndex,\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\n\t\t\t\t\tawait executeTimed(\n\t\t\t\t\t\t\"insert_batch\",\n\t\t\t\t\t\t`INSERT INTO pressure_rows (seed, cycle, bucket, payload, touched_count, created_at) VALUES ${placeholders.join(\", \")}`,\n\t\t\t\t\t\t...args,\n\t\t\t\t\t);\n\t\t\t\t\tinsertedRows += batchRows;\n\t\t\t\t\tlogStage(\"insert_batch_progress\", \"end\", {\n\t\t\t\t\t\tinsertedRows,\n\t\t\t\t\t\tbatchRows,\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t\tawait executeTimed(\"commit\", \"COMMIT\");\n\t\t\t} catch (err) {\n\t\t\t\tawait executeTimed(\"rollback\", \"ROLLBACK\").catch(() => undefined);\n\t\t\t\tthrow err;\n\t\t\t}\n\n\t\t\tconst scan = await executeTimed(\n\t\t\t\t\"scan_recent\",\n\t\t\t\t\"SELECT id, length(payload) AS payload_bytes FROM pressure_rows ORDER BY id DESC LIMIT ?\",\n\t\t\t\tscanRows,\n\t\t\t);\n\t\t\tconst bucketAgg = await executeTimed(\n\t\t\t\t\"bucket_agg\",\n\t\t\t\t\"SELECT bucket, COUNT(*) AS rows, SUM(length(payload)) AS bytes FROM pressure_rows WHERE bucket BETWEEN ? AND ? GROUP BY bucket ORDER BY bucket\",\n\t\t\t\tinput.cycle % 16,\n\t\t\t\t(input.cycle % 16) + 15,\n\t\t\t);\n\t\t\tawait executeTimed(\n\t\t\t\t\"touch_recent\",\n\t\t\t\t\"UPDATE pressure_rows SET touched_count = touched_count + 1 WHERE id IN (SELECT id FROM pressure_rows ORDER BY id DESC LIMIT ?)\",\n\t\t\t\tMath.min(scanRows, insertRows),\n\t\t\t);\n\n\t\t\tlet deletedRows = 0;\n\t\t\t// const beforeDelete = await queryOne(\n\t\t\t// \tc.db,\n\t\t\t// \t\"SELECT COUNT(*) AS count FROM pressure_rows\",\n\t\t\t// );\n\t\t\t// const overRetainRows = Math.max(0, beforeDelete.count - retainRows);\n\t\t\t// const deleteLimit = Math.max(deleteRows, overRetainRows);\n\t\t\t// if (deleteLimit > 0) {\n\t\t\t// \tawait c.db.execute(\n\t\t\t// \t\t\"DELETE FROM pressure_rows WHERE id IN (SELECT id FROM pressure_rows ORDER BY id ASC LIMIT ?)\",\n\t\t\t// \t\tdeleteLimit,\n\t\t\t// \t);\n\t\t\t// \tconst afterDelete = await queryOne(\n\t\t\t// \t\tc.db,\n\t\t\t// \t\t\"SELECT changes() AS count\",\n\t\t\t// \t);\n\t\t\t// \tdeletedRows = afterDelete.count;\n\t\t\t// }\n\n\t\t\tconst rowStatsRows = await executeTimed(\n\t\t\t\t\"row_stats\",\n\t\t\t\t\"SELECT COUNT(*) AS active_rows, COALESCE(SUM(length(payload)), 0) AS active_bytes FROM pressure_rows\",\n\t\t\t);\n\t\t\tconst rowStats = rowStatsRows[0] as\n\t\t\t\t| {\n\t\t\t\t\t\tactive_rows: number;\n\t\t\t\t\t\tactive_bytes: number | null;\n\t\t\t\t }\n\t\t\t\t| undefined;\n\t\t\tif (!rowStats) throw new Error(\"query returned no rows: row_stats\");\n\t\t\tconst integrityRows = await executeTimed(\n\t\t\t\t\"integrity_check\",\n\t\t\t\t\"PRAGMA integrity_check\",\n\t\t\t);\n\t\t\tconst integrity = integrityRows[0] as\n\t\t\t\t| { integrity_check: string }\n\t\t\t\t| undefined;\n\t\t\tif (!integrity) {\n\t\t\t\tthrow new Error(\"query returned no rows: integrity_check\");\n\t\t\t}\n\t\t\tconst durationMs = performance.now() - startedAt;\n\n\t\t\tawait executeTimed(\n\t\t\t\t\"record_cycle\",\n\t\t\t\t\"INSERT OR REPLACE INTO pressure_cycles (cycle, seed, inserted_rows, deleted_rows, active_rows, active_bytes, duration_ms, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)\",\n\t\t\t\tinput.cycle,\n\t\t\t\tinput.seed,\n\t\t\t\tinsertedRows,\n\t\t\t\tdeletedRows,\n\t\t\t\trowStats.active_rows,\n\t\t\t\trowStats.active_bytes ?? 0,\n\t\t\t\tdurationMs,\n\t\t\t\tnow,\n\t\t\t);\n\n\t\t\tconst storageStartedAt = performance.now();\n\t\t\tlogStage(\"storage_stats\", \"start\");\n\t\t\tconst storage = await storageStats(c.db);\n\t\t\tlogStage(\"storage_stats\", \"end\", {\n\t\t\t\tdurationMs: performance.now() - storageStartedAt,\n\t\t\t\tpageCount: storage.page_count,\n\t\t\t\tdbSizePages: storage.vfs?.dbSizePages ?? null,\n\t\t\t\tpageCacheEntries: storage.vfs?.pageCacheEntries ?? null,\n\t\t\t});\n\t\t\tlogStage(\"run_cycle\", \"end\", {\n\t\t\t\tdurationMs,\n\t\t\t\tactiveRows: rowStats.active_rows,\n\t\t\t\tactiveBytes: rowStats.active_bytes ?? 0,\n\t\t\t\tpageCount: storage.page_count,\n\t\t\t});\n\n\t\t\treturn {\n\t\t\t\tseed: input.seed,\n\t\t\t\tcycle: input.cycle,\n\t\t\t\tinsertedRows,\n\t\t\t\tdeletedRows,\n\t\t\t\tactiveRows: rowStats.active_rows,\n\t\t\t\tactiveBytes: rowStats.active_bytes ?? 0,\n\t\t\t\tscannedRows: scan.length,\n\t\t\t\tbucketsRead: bucketAgg.length,\n\t\t\t\tintegrityCheck: integrity.integrity_check,\n\t\t\t\tstorage,\n\t\t\t\tdurationMs,\n\t\t\t};\n\t\t},\n\t},\n});\n","import {\n\tactor,\n\ttype RivetMessageEvent,\n\ttype UniversalWebSocket,\n} from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nconst DEFAULT_SLEEP_GRACE_PERIOD_MS = 120_000;\nconst DEFAULT_ON_SLEEP_DELAY_MS = 0;\n\ntype EntryRow = {\n\trequest_id: string;\n\tidx: number;\n\tcreated_at: number;\n};\n\ntype CountRow = {\n\tcount: number;\n};\n\ntype SleepStateRow = {\n\tsleep_started_at: number;\n};\n\ntype DebugEventRow = {\n\tevent_id: string;\n\tname: string;\n\tactor_id: string;\n\tconnection_id: string | null;\n\trequest_id: string | null;\n\tdetails_json: string;\n\tcreated_at: number;\n};\n\ntype ExpectedRequest = {\n\trequestId: string;\n\tseconds: number;\n};\n\ntype DebugEventInput = {\n\tname: string;\n\tconnectionId?: string;\n\trequestId?: string;\n\tdetails?: Record;\n\tcreatedAt?: number;\n};\n\ntype DebugContext = {\n\tactorId: string;\n\tdb: {\n\t\texecute: (query: string, ...params: unknown[]) => Promise;\n\t};\n\tlog: {\n\t\twarn: (payload: unknown) => void;\n\t};\n};\n\nconst debugSocketsByActorId = new Map>();\n\nfunction sleep(ms: number): Promise {\n\treturn new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nfunction positiveInteger(value: unknown, name: string) {\n\tif (!Number.isInteger(value) || (value as number) < 1) {\n\t\tthrow new Error(`${name} must be a positive integer`);\n\t}\n\n\treturn value as number;\n}\n\nfunction stringValue(value: unknown, name: string) {\n\tif (typeof value !== \"string\" || value.length === 0) {\n\t\tthrow new Error(`${name} must be a non-empty string`);\n\t}\n\n\treturn value;\n}\n\nfunction typedRows(rows: unknown[]): T[] {\n\treturn rows as T[];\n}\n\nfunction numberFromEnv(name: string, fallback: number): number {\n\tconst value = process.env[name];\n\tif (value === undefined || value === \"\") return fallback;\n\n\tconst parsed = Number(value);\n\tif (!Number.isFinite(parsed) || parsed < 0) {\n\t\tthrow new Error(`${name} must be a finite non-negative number`);\n\t}\n\n\treturn parsed;\n}\n\nfunction send(websocket: UniversalWebSocket, payload: unknown) {\n\tif (websocket.readyState !== 1) return;\n\twebsocket.send(JSON.stringify(payload));\n}\n\nfunction debugPayload(row: DebugEventRow, replayed: boolean) {\n\treturn {\n\t\ttype: \"debugEvent\",\n\t\teventId: row.event_id,\n\t\tname: row.name,\n\t\tactorId: row.actor_id,\n\t\tconnectionId: row.connection_id,\n\t\trequestId: row.request_id,\n\t\tdetails: JSON.parse(row.details_json) as Record,\n\t\tcreatedAt: row.created_at,\n\t\treplayed,\n\t};\n}\n\nfunction publishDebugEvent(row: DebugEventRow) {\n\tconst sockets = debugSocketsByActorId.get(row.actor_id);\n\tif (!sockets) return;\n\n\tfor (const socket of sockets) {\n\t\tsend(socket, debugPayload(row, false));\n\t}\n}\n\nfunction addDebugSocket(actorId: string, websocket: UniversalWebSocket) {\n\tconst sockets = debugSocketsByActorId.get(actorId) ?? new Set();\n\tsockets.add(websocket);\n\tdebugSocketsByActorId.set(actorId, sockets);\n\n\treturn () => {\n\t\tsockets.delete(websocket);\n\t\tif (sockets.size === 0) {\n\t\t\tdebugSocketsByActorId.delete(actorId);\n\t\t}\n\t};\n}\n\nasync function recordDebugEvent(c: DebugContext, input: DebugEventInput) {\n\tconst row: DebugEventRow = {\n\t\tevent_id: crypto.randomUUID(),\n\t\tname: input.name,\n\t\tactor_id: c.actorId,\n\t\tconnection_id: input.connectionId ?? null,\n\t\trequest_id: input.requestId ?? null,\n\t\tdetails_json: JSON.stringify(input.details ?? {}),\n\t\tcreated_at: input.createdAt ?? Date.now(),\n\t};\n\n\ttry {\n\t\tawait c.db.execute(\n\t\t\t\"INSERT INTO mock_agentic_debug_events (event_id, name, actor_id, connection_id, request_id, details_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)\",\n\t\t\trow.event_id,\n\t\t\trow.name,\n\t\t\trow.actor_id,\n\t\t\trow.connection_id,\n\t\t\trow.request_id,\n\t\t\trow.details_json,\n\t\t\trow.created_at,\n\t\t);\n\t\tpublishDebugEvent(row);\n\t} catch (error) {\n\t\tc.log.warn({\n\t\t\tmsg: \"mock agentic debug event failed\",\n\t\t\tname: input.name,\n\t\t\terr: error instanceof Error ? error.message : String(error),\n\t\t});\n\t}\n}\n\nasync function replayDebugEvents(\n\tdatabase: DebugContext[\"db\"],\n\twebsocket: UniversalWebSocket,\n) {\n\tconst rows = typedRows(\n\t\tawait database.execute(`\n\t\t\tSELECT event_id, name, actor_id, connection_id, request_id, details_json, created_at\n\t\t\tFROM (\n\t\t\t\tSELECT event_id, name, actor_id, connection_id, request_id, details_json, created_at\n\t\t\t\tFROM mock_agentic_debug_events\n\t\t\t\tORDER BY created_at DESC\n\t\t\t\tLIMIT 200\n\t\t\t)\n\t\t\tORDER BY created_at ASC\n\t\t`),\n\t);\n\n\tfor (const row of rows) {\n\t\tsend(websocket, debugPayload(row, true));\n\t}\n}\n\nfunction verifyEntryRows(rows: EntryRow[], expectedSeconds: number) {\n\tconst seen = new Set();\n\tconst indexes = rows.map((row) => row.idx).sort((a, b) => a - b);\n\tfor (const idx of indexes) seen.add(idx);\n\n\tconst missing: number[] = [];\n\tfor (let idx = 1; idx <= expectedSeconds; idx += 1) {\n\t\tif (!seen.has(idx)) missing.push(idx);\n\t}\n\n\tconst contiguous =\n\t\trows.length === expectedSeconds &&\n\t\tmissing.length === 0 &&\n\t\tindexes.every((idx, offset) => idx === offset + 1);\n\n\treturn {\n\t\texpectedSeconds,\n\t\tcount: rows.length,\n\t\tcontiguous,\n\t\tmissing,\n\t\tindexes,\n\t\tok: contiguous,\n\t};\n}\n\nfunction verifyAllRows(rows: EntryRow[], expectedRequests: ExpectedRequest[]) {\n\tconst expectedByRequest = new Map(\n\t\texpectedRequests.map((request) => [request.requestId, request.seconds]),\n\t);\n\tconst rowsByRequest = new Map();\n\n\tfor (const row of rows) {\n\t\tconst requestRows = rowsByRequest.get(row.request_id) ?? [];\n\t\trequestRows.push(row);\n\t\trowsByRequest.set(row.request_id, requestRows);\n\t}\n\n\tconst requests = expectedRequests.map((request) => {\n\t\tconst result = verifyEntryRows(\n\t\t\trowsByRequest.get(request.requestId) ?? [],\n\t\t\trequest.seconds,\n\t\t);\n\t\treturn {\n\t\t\trequestId: request.requestId,\n\t\t\t...result,\n\t\t};\n\t});\n\n\tconst unexpectedRequestIds = [...rowsByRequest.keys()]\n\t\t.filter((requestId) => !expectedByRequest.has(requestId))\n\t\t.sort();\n\tconst expectedTotalRows = expectedRequests.reduce(\n\t\t(total, request) => total + request.seconds,\n\t\t0,\n\t);\n\tconst ok =\n\t\tunexpectedRequestIds.length === 0 &&\n\t\trows.length === expectedTotalRows &&\n\t\trequests.every((request) => request.ok);\n\n\treturn {\n\t\ttype: \"verifiedAll\",\n\t\texpectedRequests: expectedRequests.length,\n\t\texpectedTotalRows,\n\t\ttotalRows: rows.length,\n\t\trows,\n\t\tunexpectedRequestIds,\n\t\trequests,\n\t\tok,\n\t};\n}\n\nexport const mockAgenticLoop = actor({\n\toptions: {\n\t\tcanHibernateWebSocket: false,\n\t\tsleepGracePeriod: DEFAULT_SLEEP_GRACE_PERIOD_MS,\n\t},\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS mock_agentic_entries (\n\t\t\t\t\trequest_id TEXT NOT NULL,\n\t\t\t\t\tidx INTEGER NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL,\n\t\t\t\t\tPRIMARY KEY (request_id, idx)\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_mock_agentic_entries_created_at ON mock_agentic_entries(created_at)\",\n\t\t\t);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS mock_agentic_sleep_state (\n\t\t\t\t\tid INTEGER PRIMARY KEY CHECK (id = 1),\n\t\t\t\t\tsleep_started_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS mock_agentic_debug_events (\n\t\t\t\t\tevent_id TEXT PRIMARY KEY,\n\t\t\t\t\tname TEXT NOT NULL,\n\t\t\t\t\tactor_id TEXT NOT NULL,\n\t\t\t\t\tconnection_id TEXT,\n\t\t\t\t\trequest_id TEXT,\n\t\t\t\t\tdetails_json TEXT NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait database.execute(\n\t\t\t\t\"CREATE INDEX IF NOT EXISTS idx_mock_agentic_debug_events_created_at ON mock_agentic_debug_events(created_at)\",\n\t\t\t);\n\t\t},\n\t}),\n\tasync onWake(c) {\n\t\tawait recordDebugEvent(c, {\n\t\t\tname: \"onWake\",\n\t\t\tdetails: {\n\t\t\t\tkey: c.key,\n\t\t\t\tname: c.name,\n\t\t\t},\n\t\t});\n\t},\n\tasync onSleep(c) {\n\t\tconst delayMs = numberFromEnv(\n\t\t\t\"MOCK_AGENTIC_ON_SLEEP_DELAY_MS\",\n\t\t\tDEFAULT_ON_SLEEP_DELAY_MS,\n\t\t);\n\t\tconst sleepStartedAt = Date.now();\n\t\tawait recordDebugEvent(c, {\n\t\t\tname: \"onSleepStart\",\n\t\t\tcreatedAt: sleepStartedAt,\n\t\t\tdetails: {\n\t\t\t\tdelayMs,\n\t\t\t},\n\t\t});\n\t\tawait c.db.execute(\n\t\t\t\"INSERT OR REPLACE INTO mock_agentic_sleep_state (id, sleep_started_at) VALUES (1, ?)\",\n\t\t\tsleepStartedAt,\n\t\t);\n\t\tc.log.info({\n\t\t\tmsg: \"mock agentic loop onSleep delay\",\n\t\t\tdelayMs,\n\t\t\tsleepStartedAt,\n\t\t});\n\t\tawait sleep(delayMs);\n\t\tawait recordDebugEvent(c, {\n\t\t\tname: \"onSleepEnd\",\n\t\t\tdetails: {\n\t\t\t\tdelayMs,\n\t\t\t\tsleepStartedAt,\n\t\t\t\telapsedMs: Date.now() - sleepStartedAt,\n\t\t\t},\n\t\t});\n\t},\n\tasync onRequest(c, request) {\n\t\tconst url = new URL(request.url);\n\t\tif (url.pathname === \"/bypass\" || url.pathname === \"/request/bypass\") {\n\t\t\tconst [sleepState] = typedRows(\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"SELECT sleep_started_at FROM mock_agentic_sleep_state WHERE id = 1\",\n\t\t\t\t),\n\t\t\t);\n\t\t\treturn new Response(JSON.stringify({\n\t\t\t\ttype: \"bypass\",\n\t\t\t\ttransport: \"http\",\n\t\t\t\tsleepStarted: sleepState !== undefined,\n\t\t\t\tsleepStartedAt: sleepState?.sleep_started_at ?? null,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t}), {\n\t\t\t\theaders: {\n\t\t\t\t\t\"content-type\": \"application/json\",\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\treturn new Response(\"not found\", { status: 404 });\n\t},\n\tonWebSocket(c, websocket: UniversalWebSocket) {\n\t\tconst connectionId = crypto.randomUUID();\n\t\tlet activeInference: Promise | undefined;\n\t\tconst removeDebugSocket = addDebugSocket(c.actorId, websocket);\n\n\t\tsend(websocket, {\n\t\t\ttype: \"hello\",\n\t\t\tconnectionId,\n\t\t\ttimestamp: Date.now(),\n\t\t});\n\t\tvoid (async () => {\n\t\t\ttry {\n\t\t\t\tawait replayDebugEvents(c.db, websocket);\n\t\t\t} catch (error) {\n\t\t\t\tc.log.warn({\n\t\t\t\t\tmsg: \"mock agentic debug replay failed\",\n\t\t\t\t\terr: error instanceof Error ? error.message : String(error),\n\t\t\t\t});\n\t\t\t}\n\t\t\tawait recordDebugEvent(c, {\n\t\t\t\tname: \"webSocketOpen\",\n\t\t\t\tconnectionId,\n\t\t\t});\n\t\t})();\n\n\t\tconst verify = async (requestId: string, expectedSeconds: number) => {\n\t\t\tconst rows = typedRows(\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"SELECT request_id, idx, created_at FROM mock_agentic_entries WHERE request_id = ? ORDER BY idx ASC\",\n\t\t\t\t\trequestId,\n\t\t\t\t),\n\t\t\t);\n\t\t\treturn {\n\t\t\t\ttype: \"verified\",\n\t\t\t\trequestId,\n\t\t\t\t...verifyEntryRows(rows, expectedSeconds),\n\t\t\t};\n\t\t};\n\n\t\tconst sleepStatus = async () => {\n\t\t\tconst [sleepState] = typedRows(\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"SELECT sleep_started_at FROM mock_agentic_sleep_state WHERE id = 1\",\n\t\t\t\t),\n\t\t\t);\n\t\t\treturn {\n\t\t\t\tsleepStarted: sleepState !== undefined,\n\t\t\t\tsleepStartedAt: sleepState?.sleep_started_at ?? null,\n\t\t\t};\n\t\t};\n\n\t\tconst runInference = async (requestId: string, seconds: number) => {\n\t\t\tsend(websocket, {\n\t\t\t\ttype: \"started\",\n\t\t\t\trequestId,\n\t\t\t\tseconds,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t});\n\n\t\t\tawait c.db.execute(\n\t\t\t\t\"DELETE FROM mock_agentic_entries WHERE request_id = ?\",\n\t\t\t\trequestId,\n\t\t\t);\n\n\t\t\tfor (let idx = 1; idx <= seconds; idx += 1) {\n\t\t\t\tawait sleep(1_000);\n\t\t\t\tconst createdAt = Date.now();\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO mock_agentic_entries (request_id, idx, created_at) VALUES (?, ?, ?)\",\n\t\t\t\t\trequestId,\n\t\t\t\t\tidx,\n\t\t\t\t\tcreatedAt,\n\t\t\t\t);\n\t\t\t\tsend(websocket, {\n\t\t\t\t\ttype: \"progress\",\n\t\t\t\t\trequestId,\n\t\t\t\t\tidx,\n\t\t\t\t\tseconds,\n\t\t\t\t\tcreatedAt,\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tconst verification = await verify(requestId, seconds);\n\t\t\tsend(websocket, {\n\t\t\t\ttype: \"done\",\n\t\t\t\trequestId,\n\t\t\t\tseconds,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t\tverification,\n\t\t\t});\n\t\t};\n\n\t\twebsocket.addEventListener(\"message\", async (event: RivetMessageEvent) => {\n\t\t\ttry {\n\t\t\t\tif (typeof event.data !== \"string\") {\n\t\t\t\t\tthrow new Error(\"message data must be a JSON string\");\n\t\t\t\t}\n\n\t\t\t\tconst message = JSON.parse(event.data) as Record;\n\t\t\t\tconst type = stringValue(message.type, \"type\");\n\n\t\t\t\tif (type === \"history\") {\n\t\t\t\t\tconst rows = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\"SELECT request_id, idx, created_at FROM mock_agentic_entries ORDER BY created_at ASC, request_id ASC, idx ASC\",\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tconst [count] = typedRows(\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\"SELECT COUNT(*) AS count FROM mock_agentic_entries\",\n\t\t\t\t\t\t),\n\t\t\t\t\t);\n\t\t\t\t\tsend(websocket, {\n\t\t\t\t\t\ttype: \"history\",\n\t\t\t\t\t\ttotalRows: count?.count ?? rows.length,\n\t\t\t\t\t\tentries: rows,\n\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t});\n\t\t\t\t\treturn;\n\t\t\t\t}\n\n\t\t\t\tif (type === \"ping\") {\n\t\t\t\t\tsend(websocket, {\n\t\t\t\t\t\ttype: \"pong\",\n\t\t\t\t\t\tprobeId: stringValue(message.probeId, \"probeId\"),\n\t\t\t\t\t\t...(await sleepStatus()),\n\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t});\n\t\t\t\t\treturn;\n\t\t\t\t}\n\n\t\t\t\tif (type === \"verify\") {\n\t\t\t\t\tconst requestId = stringValue(message.requestId, \"requestId\");\n\t\t\t\t\tconst expectedSeconds = positiveInteger(\n\t\t\t\t\t\tmessage.expectedSeconds,\n\t\t\t\t\t\t\"expectedSeconds\",\n\t\t\t\t\t);\n\t\t\t\t\tsend(websocket, await verify(requestId, expectedSeconds));\n\t\t\t\t\treturn;\n\t\t\t\t}\n\n\t\t\t\tif (type === \"infer\") {\n\t\t\t\t\tconst requestId = stringValue(message.requestId, \"requestId\");\n\t\t\t\t\tconst seconds = positiveInteger(message.seconds, \"seconds\");\n\t\t\t\t\tawait recordDebugEvent(c, {\n\t\t\t\t\t\tname: \"inferenceRequested\",\n\t\t\t\t\t\tconnectionId,\n\t\t\t\t\t\trequestId,\n\t\t\t\t\t\tdetails: {\n\t\t\t\t\t\t\tseconds,\n\t\t\t\t\t\t},\n\t\t\t\t\t});\n\t\t\t\t\tconst previousInference = activeInference;\n\t\t\t\t\tconst inference = (async () => {\n\t\t\t\t\t\tawait previousInference?.catch(() => undefined);\n\t\t\t\t\t\tawait runInference(requestId, seconds);\n\t\t\t\t\t})();\n\t\t\t\t\tactiveInference = inference;\n\t\t\t\t\tawait c.keepAwake(inference);\n\t\t\t\t\tif (activeInference === inference) {\n\t\t\t\t\t\tactiveInference = undefined;\n\t\t\t\t\t}\n\t\t\t\t\treturn;\n\t\t\t\t}\n\n\t\t\t\tthrow new Error(`unknown message type: ${type}`);\n\t\t\t} catch (error) {\n\t\t\t\tsend(websocket, {\n\t\t\t\t\ttype: \"error\",\n\t\t\t\t\tmessage:\n\t\t\t\t\t\terror instanceof Error\n\t\t\t\t\t\t\t? error.message\n\t\t\t\t\t\t\t: \"unknown websocket error\",\n\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t});\n\t\t\t}\n\t\t});\n\n\t\twebsocket.addEventListener(\"close\", async () => {\n\t\t\tremoveDebugSocket();\n\t\t\tawait recordDebugEvent(c, {\n\t\t\t\tname: \"webSocketClose\",\n\t\t\t\tconnectionId,\n\t\t\t});\n\t\t});\n\t},\n\tactions: {\n\t\tverify: async (c, requestId: string, expectedSeconds: number) => {\n\t\t\tconst rows = typedRows(\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"SELECT request_id, idx, created_at FROM mock_agentic_entries WHERE request_id = ? ORDER BY idx ASC\",\n\t\t\t\t\trequestId,\n\t\t\t\t),\n\t\t\t);\n\t\t\treturn {\n\t\t\t\trequestId,\n\t\t\t\texpectedSeconds,\n\t\t\t\tcount: rows.length,\n\t\t\t\tindexes: rows.map((row) => row.idx),\n\t\t\t};\n\t\t},\n\t\tverifyAll: async (c, expectedRequests: ExpectedRequest[]) => {\n\t\t\tif (!Array.isArray(expectedRequests)) {\n\t\t\t\tthrow new Error(\"expectedRequests must be an array\");\n\t\t\t}\n\n\t\t\tfor (const request of expectedRequests) {\n\t\t\t\tstringValue(request.requestId, \"requestId\");\n\t\t\t\tpositiveInteger(request.seconds, \"seconds\");\n\t\t\t}\n\n\t\t\tconst rows = typedRows(\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"SELECT request_id, idx, created_at FROM mock_agentic_entries ORDER BY request_id ASC, idx ASC\",\n\t\t\t\t),\n\t\t\t);\n\t\t\treturn verifyAllRows(rows, expectedRequests);\n\t\t},\n\t},\n});\n","import { actor, type RivetMessageEvent, type UniversalWebSocket } from \"rivetkit\";\n\n// Minimal non-hibernatable WebSocket actor for fuzz-testing the\n// force-sleep → gateway close path. Keeps state intentionally tiny so the\n// only thing being exercised is the close lifecycle, not user code.\nexport const sleepCloseFuzz = actor({\n\toptions: {\n\t\tcanHibernateWebSocket: false,\n\t},\n\tstate: {\n\t\tconnectionCount: 0,\n\t\tmessageCount: 0,\n\t},\n\tonWebSocket(c, websocket: UniversalWebSocket) {\n\t\tc.state.connectionCount += 1;\n\t\tconst connectionId = crypto.randomUUID();\n\n\t\twebsocket.send(\n\t\t\tJSON.stringify({\n\t\t\t\ttype: \"welcome\",\n\t\t\t\tconnectionId,\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t}),\n\t\t);\n\n\t\tconst interval = setInterval(() => {\n\t\t\tif (websocket.readyState !== 1) return;\n\t\t\twebsocket.send(\n\t\t\t\tJSON.stringify({\n\t\t\t\t\ttype: \"tick\",\n\t\t\t\t\tconnectionId,\n\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t}),\n\t\t\t);\n\t\t}, 500);\n\n\t\twebsocket.addEventListener(\"message\", (event: RivetMessageEvent) => {\n\t\t\tc.state.messageCount += 1;\n\t\t\twebsocket.send(\n\t\t\t\tJSON.stringify({\n\t\t\t\t\ttype: \"echo\",\n\t\t\t\t\tconnectionId,\n\t\t\t\t\treceived: event.data,\n\t\t\t\t}),\n\t\t\t);\n\t\t});\n\n\t\twebsocket.addEventListener(\"close\", () => {\n\t\t\tclearInterval(interval);\n\t\t\tc.state.connectionCount -= 1;\n\t\t});\n\t},\n\tactions: {\n\t\tgetStats(c) {\n\t\t\treturn {\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t\tmessageCount: c.state.messageCount,\n\t\t\t};\n\t\t},\n\t},\n});\n","import { actor, type RivetMessageEvent, type UniversalWebSocket } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nconst DEFAULT_TOKENS_PER_SECOND = 20;\nconst DEFAULT_DURATION_MS = 5_000;\n\nfunction send(websocket: UniversalWebSocket, payload: unknown): void {\n\tif (websocket.readyState !== 1) return;\n\twebsocket.send(JSON.stringify(payload));\n}\n\nfunction parsePositiveNumber(\n\tvalue: unknown,\n\tname: string,\n\tfallback: number,\n): number {\n\tif (value === undefined || value === null) return fallback;\n\tconst parsed = Number(value);\n\tif (!Number.isFinite(parsed) || parsed <= 0) {\n\t\tthrow new Error(`${name} must be a positive number`);\n\t}\n\treturn parsed;\n}\n\nfunction sleep(ms: number, signal: AbortSignal): Promise {\n\tif (signal.aborted) return Promise.resolve();\n\treturn new Promise((resolve) => {\n\t\tconst timeout = setTimeout(resolve, ms);\n\t\tsignal.addEventListener(\n\t\t\t\"abort\",\n\t\t\t() => {\n\t\t\t\tclearTimeout(timeout);\n\t\t\t\tresolve();\n\t\t\t},\n\t\t\t{ once: true },\n\t\t);\n\t});\n}\n\nexport const loadTestAgent = actor({\n\toptions: {\n\t\tcanHibernateWebSocket: false,\n\t\tsleepGracePeriod: 30_000,\n\t},\n\tdb: db({\n\t\tonMigrate: async (db) => {\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS messages (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tconnection_id TEXT NOT NULL,\n\t\t\t\t\trequest_id TEXT NOT NULL,\n\t\t\t\t\ttoken_index INTEGER NOT NULL,\n\t\t\t\t\ttoken TEXT NOT NULL,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t\tawait db.execute(`\n\t\t\t\tCREATE INDEX IF NOT EXISTS messages_request_idx\n\t\t\t\tON messages (request_id, token_index)\n\t\t\t`);\n\t\t},\n\t}),\n\tstate: {\n\t\tconnectionCount: 0,\n\t\tinferenceCount: 0,\n\t\ttokenCount: 0,\n\t},\n\tonWebSocket(c, websocket: UniversalWebSocket) {\n\t\tc.state.connectionCount += 1;\n\t\tconst connectionId = crypto.randomUUID();\n\n\t\tsend(websocket, {\n\t\t\ttype: \"connected\",\n\t\t\tconnectionId,\n\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\ttimestamp: Date.now(),\n\t\t});\n\n\t\twebsocket.addEventListener(\"message\", async (event: RivetMessageEvent) => {\n\t\t\ttry {\n\t\t\t\tconst message =\n\t\t\t\t\ttypeof event.data === \"string\"\n\t\t\t\t\t\t? JSON.parse(event.data)\n\t\t\t\t\t\t: undefined;\n\n\t\t\t\t// Fast-path ping: echo back without touching SQLite so the client can measure raw\n\t\t\t\t// RTT without the per-message storage write. Used by the counter-latency client's\n\t\t\t\t// first two probes after WS open.\n\t\t\t\tif (message && message.type === \"ping\") {\n\t\t\t\t\tsend(websocket, {\n\t\t\t\t\t\ttype: \"pong\",\n\t\t\t\t\t\tconnectionId,\n\t\t\t\t\t\tid: message.id,\n\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t});\n\t\t\t\t\treturn;\n\t\t\t\t}\n\n\t\t\t\tif (!message || message.type !== \"inference\") {\n\t\t\t\t\tthrow new Error(\"expected inference message\");\n\t\t\t\t}\n\n\t\t\t\tconst requestId =\n\t\t\t\t\ttypeof message.requestId === \"string\" && message.requestId\n\t\t\t\t\t\t? message.requestId\n\t\t\t\t\t\t: crypto.randomUUID();\n\t\t\t\tconst tokensPerSecond = parsePositiveNumber(\n\t\t\t\t\tmessage.tokensPerSecond,\n\t\t\t\t\t\"tokensPerSecond\",\n\t\t\t\t\tDEFAULT_TOKENS_PER_SECOND,\n\t\t\t\t);\n\t\t\t\tconst durationMs = parsePositiveNumber(\n\t\t\t\t\tmessage.durationMs,\n\t\t\t\t\t\"durationMs\",\n\t\t\t\t\tDEFAULT_DURATION_MS,\n\t\t\t\t);\n\t\t\t\tconst intervalMs = 1_000 / tokensPerSecond;\n\t\t\t\tconst targetTokens = Math.max(\n\t\t\t\t\t1,\n\t\t\t\t\tMath.floor((durationMs / 1_000) * tokensPerSecond),\n\t\t\t\t);\n\n\t\t\t\tconst inference = (async () => {\n\t\t\t\t\tc.state.inferenceCount += 1;\n\t\t\t\t\tsend(websocket, {\n\t\t\t\t\t\ttype: \"inference-start\",\n\t\t\t\t\t\tconnectionId,\n\t\t\t\t\t\trequestId,\n\t\t\t\t\t\ttokensPerSecond,\n\t\t\t\t\t\tdurationMs,\n\t\t\t\t\t\ttargetTokens,\n\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t});\n\n\t\t\t\t\tconst startedAt = performance.now();\n\t\t\t\t\tfor (let i = 0; i < targetTokens; i++) {\n\t\t\t\t\t\tif (c.abortSignal.aborted || websocket.readyState !== 1) {\n\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tconst tokenIndex = i + 1;\n\t\t\t\t\t\tconst token = `token-${tokenIndex}`;\n\t\t\t\t\t\tconst createdAt = Date.now();\n\t\t\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\t\t\"INSERT INTO messages (connection_id, request_id, token_index, token, created_at) VALUES (?, ?, ?, ?, ?)\",\n\t\t\t\t\t\t\tconnectionId,\n\t\t\t\t\t\t\trequestId,\n\t\t\t\t\t\t\ttokenIndex,\n\t\t\t\t\t\t\ttoken,\n\t\t\t\t\t\t\tcreatedAt,\n\t\t\t\t\t\t);\n\t\t\t\t\t\tc.state.tokenCount += 1;\n\n\t\t\t\t\t\tsend(websocket, {\n\t\t\t\t\t\t\ttype: \"token\",\n\t\t\t\t\t\t\tconnectionId,\n\t\t\t\t\t\t\trequestId,\n\t\t\t\t\t\t\ttokenIndex,\n\t\t\t\t\t\t\ttoken,\n\t\t\t\t\t\t\ttimestamp: createdAt,\n\t\t\t\t\t\t});\n\n\t\t\t\t\t\tconst nextAt = startedAt + tokenIndex * intervalMs;\n\t\t\t\t\t\tconst delayMs = Math.max(0, nextAt - performance.now());\n\t\t\t\t\t\tif (delayMs > 0) {\n\t\t\t\t\t\t\tawait sleep(delayMs, c.abortSignal);\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tsend(websocket, {\n\t\t\t\t\t\ttype: \"inference-complete\",\n\t\t\t\t\t\tconnectionId,\n\t\t\t\t\t\trequestId,\n\t\t\t\t\t\ttokenCount: targetTokens,\n\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t});\n\t\t\t\t})();\n\n\t\t\t\tawait c.keepAwake(inference);\n\t\t\t} catch (error) {\n\t\t\t\tsend(websocket, {\n\t\t\t\t\ttype: \"error\",\n\t\t\t\t\tmessage:\n\t\t\t\t\t\terror instanceof Error\n\t\t\t\t\t\t\t? error.message\n\t\t\t\t\t\t\t: \"unknown websocket error\",\n\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t});\n\t\t\t}\n\t\t});\n\n\t\twebsocket.addEventListener(\"close\", () => {\n\t\t\tc.state.connectionCount -= 1;\n\t\t});\n\t},\n\tactions: {\n\t\tgetStats(c) {\n\t\t\treturn {\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t\tinferenceCount: c.state.inferenceCount,\n\t\t\t\ttokenCount: c.state.tokenCount,\n\t\t\t};\n\t\t},\n\t},\n});\n","import { actor, type RivetMessageEvent, type UniversalWebSocket } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\ntype AgentConcurrent2Request =\n\t| { type: \"agent2_resume\"; version: number }\n\t| { type: \"agent2_connect\"; clientId: string; staggerHandleMs?: number }\n\t| { type: \"force_sleep\" }\n\t| { type: \"ping\"; id?: number };\n\ninterface AgentConcurrent2Step {\n\tname: string;\n\tdurationMs: number;\n\trowCount: number;\n}\n\ninterface AgentConcurrent2QueryStats {\n\ttotal: number;\n\treads: number;\n\tmutations: number;\n\ttx: number;\n\tother: number;\n\trows: number;\n\terrors: number;\n\tslow: number;\n\tmaxMs: number;\n\tmaxStep: string;\n\tbyOperation: Record;\n\tbyTable: Record;\n}\n\ninterface AgentConcurrent2StatsSnapshot {\n\twakeIndex: number;\n\tactorIteration: number;\n\twakeIteration: number;\n\tcycle: AgentConcurrent2QueryStats;\n\twake: AgentConcurrent2QueryStats;\n\tactor: AgentConcurrent2QueryStats;\n}\n\ninterface AgentConcurrent2WorkloadResult {\n\tname: string;\n\ttotalMs: number;\n\tsteps: AgentConcurrent2Step[];\n}\n\ninterface AgentConcurrent2ResultMessage {\n\ttype: \"agent2_result\";\n\ttrigger: AgentConcurrent2Request[\"type\"];\n\ttotalMs: number;\n\tresults: AgentConcurrent2WorkloadResult[];\n\tstats: AgentConcurrent2StatsSnapshot;\n}\n\ninterface AgentConcurrent2ErrorMessage {\n\ttype: \"agent2_error\";\n\ttrigger: AgentConcurrent2Request[\"type\"] | \"unknown\";\n\terror: string;\n\tstats?: AgentConcurrent2StatsSnapshot;\n}\n\ninterface AgentConcurrent2Vars {\n\tsql: AgentConcurrent2Db | null;\n\twakeStats: AgentConcurrent2QueryStats | null;\n\twakeStartedAt: number | null;\n\twakeIteration: number;\n}\n\ninterface RawRivetDB {\n\texecute: (\n\t\tquery: string,\n\t\t...args: unknown[]\n\t) => Promise[]>;\n}\n\ntype SQLPrimitive = string | number | boolean | null;\n\ninterface AgentConcurrent2State {\n\trunCount: number;\n\twakeCount: number;\n\tqueryStats: AgentConcurrent2QueryStats;\n}\n\ninterface AgentConcurrent2QueryStatsSet {\n\tcycle: AgentConcurrent2QueryStats;\n\twake: AgentConcurrent2QueryStats;\n\tactor: AgentConcurrent2QueryStats;\n}\n\ninterface AgentConcurrent2Runtime {\n\tsql: AgentConcurrent2Db;\n\twakeStats: AgentConcurrent2QueryStats;\n\tvars: AgentConcurrent2Vars;\n}\n\ntype AgentConcurrent2Db = (>(\n\tquery: string,\n\t...values: SQLPrimitive[]\n) => Promise) & {\n\twithTransaction(\n\t\tstats: AgentConcurrent2QueryStatsSet,\n\t\tfn: (tx: AgentConcurrent2Db) => Promise,\n\t): Promise;\n};\n\nclass AsyncMutex {\n\tprivate locked = false;\n\tprivate waiters: Array<() => void> = [];\n\n\tasync acquire(): Promise {\n\t\tif (!this.locked) {\n\t\t\tthis.locked = true;\n\t\t\treturn;\n\t\t}\n\t\tawait new Promise((resolve) => this.waiters.push(resolve));\n\t\tthis.locked = true;\n\t}\n\n\trelease(): void {\n\t\tconst next = this.waiters.shift();\n\t\tif (next) {\n\t\t\tnext();\n\t\t\treturn;\n\t\t}\n\t\tthis.locked = false;\n\t}\n}\n\nfunction createSerializedDb(\n\texecute: >(\n\t\tquery: string,\n\t\t...values: SQLPrimitive[]\n\t) => Promise,\n): AgentConcurrent2Db {\n\tconst mutex = new AsyncMutex();\n\tlet activeTransaction: AgentConcurrent2Db | null = null;\n\n\tconst createTransactionDb = (): AgentConcurrent2Db => {\n\t\tconst tx = Object.assign(\n\t\t\t>(\n\t\t\t\tquery: string,\n\t\t\t\t...values: SQLPrimitive[]\n\t\t\t) => execute(query, ...values),\n\t\t\t{\n\t\t\t\twithTransaction: async (\n\t\t\t\t\t_stats: AgentConcurrent2QueryStatsSet,\n\t\t\t\t\tfn: (tx: AgentConcurrent2Db) => Promise,\n\t\t\t\t): Promise => fn(tx),\n\t\t\t},\n\t\t);\n\t\treturn tx;\n\t};\n\n\tconst queryWithMutex = async >(\n\t\tquery: string,\n\t\t...values: SQLPrimitive[]\n\t): Promise => {\n\t\tawait mutex.acquire();\n\t\ttry {\n\t\t\treturn await execute(query, ...values);\n\t\t} finally {\n\t\t\tmutex.release();\n\t\t}\n\t};\n\n\treturn Object.assign(queryWithMutex, {\n\t\twithTransaction: async (\n\t\t\tstats: AgentConcurrent2QueryStatsSet,\n\t\t\tfn: (tx: AgentConcurrent2Db) => Promise,\n\t\t): Promise => {\n\t\t\tif (activeTransaction) {\n\t\t\t\treturn fn(activeTransaction);\n\t\t\t}\n\t\t\tawait mutex.acquire();\n\t\t\tconst tx = createTransactionDb();\n\t\t\ttry {\n\t\t\t\tawait executeTrackedQuery(execute, stats, \"transaction-begin\", \"BEGIN\");\n\t\t\t\tactiveTransaction = tx;\n\t\t\t\ttry {\n\t\t\t\t\tconst result = await fn(tx);\n\t\t\t\t\tactiveTransaction = null;\n\t\t\t\t\tawait executeTrackedQuery(execute, stats, \"transaction-commit\", \"COMMIT\");\n\t\t\t\t\treturn result;\n\t\t\t\t} catch (error) {\n\t\t\t\t\tactiveTransaction = null;\n\t\t\t\t\tawait executeTrackedQuery(\n\t\t\t\t\t\texecute,\n\t\t\t\t\t\tstats,\n\t\t\t\t\t\t\"transaction-rollback\",\n\t\t\t\t\t\t\"ROLLBACK\",\n\t\t\t\t\t);\n\t\t\t\t\tthrow error;\n\t\t\t\t}\n\t\t\t} finally {\n\t\t\t\tactiveTransaction = null;\n\t\t\t\tmutex.release();\n\t\t\t}\n\t\t},\n\t});\n}\n\nconst MESSAGE_COUNT = 84;\nconst MESSAGE_TOOL_REF_COUNT = 122;\nconst TOOL_CALL_COUNT = 61;\nconst EXECUTOR_TOOL_COUNT = 42;\nconst THREAD_EVENT_COUNT = 233;\n\nconst MESSAGE_CONTENT_BYTES = 2_600;\nconst THREAD_EVENT_PAYLOAD_BYTES = 1_000;\nconst TOOL_CALL_RESULT_BYTES = 2_700;\nconst EXECUTOR_TOOL_SCHEMA_BYTES = 550;\nconst SLOW_QUERY_MS = 1_000;\n\nfunction send(\n\twebsocket: UniversalWebSocket,\n\tmessage: AgentConcurrent2ResultMessage | AgentConcurrent2ErrorMessage | object,\n): void {\n\tif (websocket.readyState === 1) {\n\t\twebsocket.send(JSON.stringify(message));\n\t}\n}\n\nexport const loadTestAgent2 = actor({\n\toptions: {\n\t\tcanHibernateWebSocket: false,\n\t\tsleepGracePeriod: 1_000,\n\t},\n\tstate: {\n\t\trunCount: 0,\n\t\twakeCount: 0,\n\t\tqueryStats: createAgentConcurrent2QueryStats(),\n\t} as AgentConcurrent2State,\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait createAgentConcurrent2Schema(database);\n\t\t\tawait seedAgentConcurrent2Data(database);\n\t\t},\n\t}),\n\tvars: {\n\t\tsql: null,\n\t\twakeStats: null,\n\t\twakeStartedAt: null,\n\t\twakeIteration: 0,\n\t} as AgentConcurrent2Vars,\n\tonWebSocket: (c, websocket: UniversalWebSocket) => {\n\t\tsend(websocket, {\n\t\t\ttype: \"connected\",\n\t\t\ttimestamp: Date.now(),\n\t\t});\n\n\t\twebsocket.addEventListener(\"message\", (event: RivetMessageEvent) => {\n\t\t\tconst promise = handleAgentConcurrent2Message(c, websocket, event.data);\n\t\t\tvoid c.keepAwake(promise);\n\t\t});\n\t},\n\tactions: {\n\t\trun: async (c, clientId?: string) => {\n\t\t\tconst runtime = ensureAgentConcurrent2Runtime(c);\n\t\t\tc.state.runCount++;\n\t\t\truntime.vars.wakeIteration++;\n\t\t\tconst cycleStats = createAgentConcurrent2QueryStats();\n\t\t\tconst stats = createAgentConcurrent2StatsSet(\n\t\t\t\tcycleStats,\n\t\t\t\truntime.wakeStats,\n\t\t\t\tc.state.queryStats,\n\t\t\t);\n\t\t\tconst result = await runAgentConcurrent2Workload(\n\t\t\t\truntime.sql,\n\t\t\t\tclientId ?? `agent2-action-${c.state.runCount}`,\n\t\t\t\t0,\n\t\t\t\tstats,\n\t\t\t);\n\t\t\treturn {\n\t\t\t\t...result,\n\t\t\t\tstats: snapshotAgentConcurrent2Stats(c, cycleStats),\n\t\t\t};\n\t\t},\n\t\tgetRunCount: (c) => c.state.runCount,\n\t\tsleep: (c) => {\n\t\t\tc.sleep();\n\t\t\treturn true;\n\t\t},\n\t},\n});\n\nasync function handleAgentConcurrent2Message(\n\tc: {\n\t\tdb: RawRivetDB;\n\t\tvars: AgentConcurrent2Vars;\n\t\tstate: AgentConcurrent2State;\n\t\tsleep: () => void;\n\t},\n\twebsocket: UniversalWebSocket,\n\tdata: unknown,\n): Promise {\n\tlet trigger: AgentConcurrent2Request[\"type\"] | \"unknown\" = \"unknown\";\n\tlet cycleStats: AgentConcurrent2QueryStats | null = null;\n\ttry {\n\t\tconst request = parseAgentConcurrent2Request(data);\n\t\ttrigger = request.type;\n\n\t\tif (request.type === \"ping\") {\n\t\t\tsend(websocket, {\n\t\t\t\ttype: \"pong\",\n\t\t\t\tid: request.id,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t});\n\t\t\treturn;\n\t\t}\n\n\t\tif (request.type === \"force_sleep\") {\n\t\t\tsend(websocket, { type: \"sleeping\", timestamp: Date.now() });\n\t\t\tc.sleep();\n\t\t\treturn;\n\t\t}\n\n\t\tconst runtime = ensureAgentConcurrent2Runtime(c);\n\t\tc.state.runCount++;\n\t\truntime.vars.wakeIteration++;\n\t\tcycleStats = createAgentConcurrent2QueryStats();\n\t\tconst stats = createAgentConcurrent2StatsSet(\n\t\t\tcycleStats,\n\t\t\truntime.wakeStats,\n\t\t\tc.state.queryStats,\n\t\t);\n\n\t\tif (request.type === \"agent2_resume\") {\n\t\t\tconst startedAt = performance.now();\n\t\t\tconst result = await runCatchupSnapshot(\n\t\t\t\truntime.sql,\n\t\t\t\trequest.version,\n\t\t\t\tstats,\n\t\t\t);\n\t\t\tsend(websocket, {\n\t\t\t\ttype: \"agent2_result\",\n\t\t\t\ttrigger: request.type,\n\t\t\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\t\t\tresults: [result],\n\t\t\t\tstats: snapshotAgentConcurrent2Stats(c, cycleStats),\n\t\t\t});\n\t\t\treturn;\n\t\t}\n\n\t\tconst result = await runAgentConcurrent2Workload(\n\t\t\truntime.sql,\n\t\t\trequest.clientId,\n\t\t\trequest.staggerHandleMs ?? 0,\n\t\t\tstats,\n\t\t);\n\t\tsend(websocket, {\n\t\t\ttype: \"agent2_result\",\n\t\t\ttrigger: request.type,\n\t\t\t...result,\n\t\t\tstats: snapshotAgentConcurrent2Stats(c, cycleStats),\n\t\t});\n\t} catch (error) {\n\t\tsend(websocket, {\n\t\t\ttype: \"agent2_error\",\n\t\t\ttrigger,\n\t\t\terror: error instanceof Error ? error.message : String(error),\n\t\t\t...(cycleStats ? { stats: snapshotAgentConcurrent2Stats(c, cycleStats) } : {}),\n\t\t});\n\t}\n}\n\nfunction parseAgentConcurrent2Request(data: unknown): AgentConcurrent2Request {\n\tif (typeof data !== \"string\") {\n\t\tthrow new Error(\"agent concurrent 2 request must be a string\");\n\t}\n\tconst parsed = JSON.parse(data) as unknown;\n\tif (!parsed || typeof parsed !== \"object\") {\n\t\tthrow new Error(\"agent concurrent 2 request must be an object\");\n\t}\n\tconst request = parsed as Record;\n\n\tif (request.type === \"ping\") {\n\t\treturn {\n\t\t\ttype: \"ping\",\n\t\t\t...(typeof request.id === \"number\" ? { id: request.id } : {}),\n\t\t};\n\t}\n\tif (request.type === \"force_sleep\") {\n\t\treturn { type: \"force_sleep\" };\n\t}\n\tif (request.type === \"agent2_resume\") {\n\t\treturn { type: \"agent2_resume\", version: numberField(request, \"version\") };\n\t}\n\tif (request.type === \"agent2_connect\") {\n\t\treturn {\n\t\t\ttype: \"agent2_connect\",\n\t\t\tclientId: stringField(request, \"clientId\"),\n\t\t\t...(typeof request.staggerHandleMs === \"number\"\n\t\t\t\t? { staggerHandleMs: request.staggerHandleMs }\n\t\t\t\t: {}),\n\t\t};\n\t}\n\tthrow new Error(`unknown agent concurrent 2 request type: ${String(request.type)}`);\n}\n\nfunction stringField(record: Record, field: string): string {\n\tconst value = record[field];\n\tif (typeof value !== \"string\" || value.length === 0) {\n\t\tthrow new Error(`agent concurrent 2 request ${field} must be a string`);\n\t}\n\treturn value;\n}\n\nfunction numberField(record: Record, field: string): number {\n\tconst value = record[field];\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) {\n\t\tthrow new Error(`agent concurrent 2 request ${field} must be a finite number`);\n\t}\n\treturn value;\n}\n\nfunction createAgentConcurrent2Db(db: RawRivetDB): AgentConcurrent2Db {\n\treturn createSerializedDb(async >(\n\t\tquery: string,\n\t\t...values: SQLPrimitive[]\n\t): Promise => {\n\t\tconst converted = values.map((value) =>\n\t\t\ttypeof value === \"boolean\" ? (value ? 1 : 0) : value,\n\t\t);\n\t\treturn (await db.execute(query, ...converted)) as T[];\n\t});\n}\n\nfunction ensureAgentConcurrent2Runtime(c: {\n\tdb: RawRivetDB;\n\tvars: AgentConcurrent2Vars;\n\tstate: AgentConcurrent2State;\n}): AgentConcurrent2Runtime {\n\tc.vars.sql ??= createAgentConcurrent2Db(c.db);\n\tc.state.queryStats ??= createAgentConcurrent2QueryStats();\n\tc.state.wakeCount ??= 0;\n\tif (!c.vars.wakeStats) {\n\t\tc.vars.wakeStats = createAgentConcurrent2QueryStats();\n\t\tc.vars.wakeStartedAt = Date.now();\n\t\tc.vars.wakeIteration = 0;\n\t\tc.state.wakeCount++;\n\t}\n\treturn {\n\t\tsql: c.vars.sql,\n\t\twakeStats: c.vars.wakeStats,\n\t\tvars: c.vars,\n\t};\n}\n\nfunction createAgentConcurrent2QueryStats(): AgentConcurrent2QueryStats {\n\treturn {\n\t\ttotal: 0,\n\t\treads: 0,\n\t\tmutations: 0,\n\t\ttx: 0,\n\t\tother: 0,\n\t\trows: 0,\n\t\terrors: 0,\n\t\tslow: 0,\n\t\tmaxMs: 0,\n\t\tmaxStep: \"\",\n\t\tbyOperation: {},\n\t\tbyTable: {},\n\t};\n}\n\nfunction createAgentConcurrent2StatsSet(\n\tcycle: AgentConcurrent2QueryStats,\n\twake: AgentConcurrent2QueryStats,\n\tactor: AgentConcurrent2QueryStats,\n): AgentConcurrent2QueryStatsSet {\n\treturn { cycle, wake, actor };\n}\n\nfunction snapshotAgentConcurrent2Stats(\n\tc: { vars: AgentConcurrent2Vars; state: AgentConcurrent2State },\n\tcycle: AgentConcurrent2QueryStats,\n): AgentConcurrent2StatsSnapshot {\n\treturn {\n\t\twakeIndex: c.state.wakeCount,\n\t\tactorIteration: c.state.runCount,\n\t\twakeIteration: c.vars.wakeIteration,\n\t\tcycle: cloneAgentConcurrent2QueryStats(cycle),\n\t\twake: cloneAgentConcurrent2QueryStats(\n\t\t\tc.vars.wakeStats ?? createAgentConcurrent2QueryStats(),\n\t\t),\n\t\tactor: cloneAgentConcurrent2QueryStats(c.state.queryStats),\n\t};\n}\n\nfunction cloneAgentConcurrent2QueryStats(\n\tstats: AgentConcurrent2QueryStats,\n): AgentConcurrent2QueryStats {\n\treturn {\n\t\ttotal: stats.total,\n\t\treads: stats.reads,\n\t\tmutations: stats.mutations,\n\t\ttx: stats.tx,\n\t\tother: stats.other,\n\t\trows: stats.rows,\n\t\terrors: stats.errors,\n\t\tslow: stats.slow,\n\t\tmaxMs: stats.maxMs,\n\t\tmaxStep: stats.maxStep,\n\t\tbyOperation: { ...stats.byOperation },\n\t\tbyTable: { ...stats.byTable },\n\t};\n}\n\nasync function runAgentConcurrent2Workload(\n\tsql: AgentConcurrent2Db,\n\tclientId: string,\n\tstaggerHandleMs: number,\n\tstats: AgentConcurrent2QueryStatsSet,\n): Promise> {\n\tconst startedAt = performance.now();\n\tconst buildToolPlanContext = runBuildToolPlanContext(sql, stats);\n\tconst catchupSnapshot = runCatchupSnapshot(sql, 0, stats);\n\tconst recoverToolCalls = runRecoverToolCalls(sql, stats);\n\tconst mutationMix = runMutationMix(sql, clientId, stats);\n\tconst handleExecutorConnect = delay(staggerHandleMs).then(() =>\n\t\trunHandleClientConnect(sql, clientId, stats),\n\t);\n\n\tconst results = await Promise.all([\n\t\thandleExecutorConnect,\n\t\tbuildToolPlanContext,\n\t\tcatchupSnapshot,\n\t\trecoverToolCalls,\n\t\tmutationMix,\n\t]);\n\treturn {\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tresults,\n\t};\n}\n\nasync function runHandleClientConnect(\n\tsql: AgentConcurrent2Db,\n\tclientId: string,\n\tstats: AgentConcurrent2QueryStatsSet,\n): Promise {\n\tconst startedAt = performance.now();\n\tconst steps: AgentConcurrent2Step[] = [];\n\tconst nextSeq = await sql.withTransaction(stats, async (tx) => {\n\t\tconst latestExecutor = await timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"load-latest-executor-id\",\n\t\t\t`SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1`,\n\t\t);\n\t\tconst latestExecutorId = String(\n\t\t\tlatestExecutor[0]?.executor_id ?? \"seed-executor\",\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"select-cached-executor-tools\",\n\t\t\t`SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`,\n\t\t\tlatestExecutorId,\n\t\t);\n\t\tconst executorType = await timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"select-executor-type\",\n\t\t\t`SELECT value FROM thread_meta_kv WHERE key = 'executor_type'`,\n\t\t);\n\t\tif (!executorType[0]?.value) {\n\t\t\tawait timedQuery(\n\t\t\t\ttx,\n\t\t\t\tstats,\n\t\t\t\tsteps,\n\t\t\t\t\"set-executor-type\",\n\t\t\t\t`INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES ('executor_type', ?, ?)`,\n\t\t\t\t\"local-client\",\n\t\t\t\tnew Date().toISOString(),\n\t\t\t);\n\t\t}\n\t\tconst sandboxIntent = await timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"select-workspace-intent\",\n\t\t\t`SELECT value FROM thread_meta_kv WHERE key = 'workspace_intent'`,\n\t\t);\n\t\tif (hasPendingLaunch(sandboxIntent[0]?.value)) {\n\t\t\tawait timedQuery(\n\t\t\t\ttx,\n\t\t\t\tstats,\n\t\t\t\tsteps,\n\t\t\t\t\"clear-pending-launch\",\n\t\t\t\t`UPDATE thread_meta_kv SET value = ?, updated_at = ? WHERE key = 'workspace_intent'`,\n\t\t\t\tJSON.stringify({ spec: null, pendingLaunch: null }),\n\t\t\t\tnew Date().toISOString(),\n\t\t\t);\n\t\t}\n\t\tconst seqRows = await timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"select-next-thread-event-seq\",\n\t\t\t`SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events`,\n\t\t);\n\t\tconst seq = Number(seqRows[0]?.seq ?? 1);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"insert-client-connected-event\",\n\t\t\t`INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`,\n\t\t\tseq,\n\t\t\t\"client_connected\",\n\t\t\tJSON.stringify({ type: \"client_connected\", clientId }),\n\t\t\tnew Date().toISOString(),\n\t\t);\n\t\treturn seq;\n\t});\n\tsteps.push({\n\t\tname: \"transaction-total\",\n\t\tdurationMs: Math.round(performance.now() - startedAt),\n\t\trowCount: nextSeq,\n\t});\n\treturn {\n\t\tname: \"handle-client-connect\",\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tsteps,\n\t};\n}\n\nasync function runBuildToolPlanContext(\n\tsql: AgentConcurrent2Db,\n\tstats: AgentConcurrent2QueryStatsSet,\n): Promise {\n\tconst startedAt = performance.now();\n\tconst steps: AgentConcurrent2Step[] = [];\n\tconst latestExecutor = await timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"load-latest-executor-id\",\n\t\t`SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1`,\n\t);\n\tconst latestExecutorId = String(latestExecutor[0]?.executor_id ?? \"seed-executor\");\n\tawait timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"select-executor-tools\",\n\t\t`SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`,\n\t\tlatestExecutorId,\n\t);\n\tawait timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"count-uncancelled-top-level\",\n\t\t`SELECT COUNT(*) as count FROM messages WHERE cancelled = 0 AND parent_tool_use_id IS NULL`,\n\t);\n\tconst unresolvedRows = await timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"find-unresolved-assistant-message\",\n\t\t`SELECT m.*\n\t\t\tFROM message_tool_refs AS tool_use\n\t\t\tJOIN messages AS m\n\t\t\t\tON m.message_id = tool_use.assistant_message_id\n\t\t\tWHERE tool_use.block_type = 'tool_use'\n\t\t\t\tAND tool_use.cancelled = 0\n\t\t\t\tAND m.cancelled = 0\n\t\t\t\tAND m.role = 'assistant'\n\t\t\t\tAND m.parent_tool_use_id IS NULL\n\t\t\t\tAND NOT EXISTS (\n\t\t\t\t\tSELECT 1\n\t\t\t\t\tFROM message_tool_refs AS tool_result\n\t\t\t\t\tJOIN messages AS tool_result_message\n\t\t\t\t\t\tON tool_result_message.message_id = tool_result.source_message_id\n\t\t\t\t\tWHERE tool_result.assistant_message_id = tool_use.assistant_message_id\n\t\t\t\t\t\tAND tool_result.block_type = 'tool_result'\n\t\t\t\t\t\tAND tool_result.cancelled = 0\n\t\t\t\t\t\tAND tool_result.tool_use_id = tool_use.tool_use_id\n\t\t\t\t\t\tAND tool_result_message.parent_tool_use_id IS NULL\n\t\t\t\t)\n\t\t\tGROUP BY m.message_id\n\t\t\tORDER BY m.created_at DESC\n\t\t\tLIMIT 1`,\n\t);\n\tconst unresolvedMessageId = unresolvedRows[0]?.message_id;\n\tif (typeof unresolvedMessageId === \"string\") {\n\t\tawait timedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"get-persisted-tool-result-ids\",\n\t\t\t`SELECT tool_result.tool_use_id\n\t\t\t\tFROM message_tool_refs AS tool_result\n\t\t\t\tJOIN messages AS tool_result_message\n\t\t\t\t\tON tool_result_message.message_id = tool_result.source_message_id\n\t\t\t\tWHERE tool_result.assistant_message_id = ?\n\t\t\t\t\tAND tool_result.block_type = 'tool_result'\n\t\t\t\t\tAND tool_result.cancelled = 0\n\t\t\t\t\tAND tool_result_message.parent_tool_use_id IS NULL`,\n\t\t\tunresolvedMessageId,\n\t\t);\n\t\tawait timedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"get-tool-calls-by-message-id\",\n\t\t\t`SELECT * FROM tool_calls WHERE message_id = ?`,\n\t\t\tunresolvedMessageId,\n\t\t);\n\t}\n\tawait timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"is-last-message-cancelled-assistant\",\n\t\t`SELECT role, cancelled FROM messages\n\t\t\tWHERE parent_tool_use_id IS NULL\n\t\t\tORDER BY created_at DESC\n\t\t\tLIMIT 1`,\n\t);\n\tawait timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"get-last-uncancelled\",\n\t\t`SELECT m.* FROM messages m\n\t\t\tWHERE m.cancelled = 0 AND m.parent_tool_use_id IS NULL\n\t\t\tORDER BY m.created_at DESC\n\t\t\tLIMIT 1`,\n\t);\n\treturn {\n\t\tname: \"build-tool-plan-context\",\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tsteps,\n\t};\n}\n\nasync function runCatchupSnapshot(\n\tsql: AgentConcurrent2Db,\n\tversion: number,\n\tstats: AgentConcurrent2QueryStatsSet,\n): Promise {\n\tconst startedAt = performance.now();\n\tconst steps: AgentConcurrent2Step[] = [];\n\tawait Promise.all([\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"thread-events-list-since-version\",\n\t\t\t`SELECT seq, event_type, payload, created_at FROM thread_events WHERE seq > ? ORDER BY seq ASC`,\n\t\t\tversion,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"environment-snapshot\",\n\t\t\t`SELECT snapshot FROM environment_snapshot WHERE id = 1`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"thread-settings-snapshot\",\n\t\t\t`SELECT settings FROM thread_settings_snapshot WHERE id = 1`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"retry-state\",\n\t\t\t`SELECT * FROM retry_state WHERE id = 1`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"queued-messages\",\n\t\t\t`SELECT * FROM queued_messages ORDER BY created_at ASC`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"executor-artifacts\",\n\t\t\t`SELECT artifact_key, data_type, length(content_base64) AS bytes, tool_call_id, updated_at FROM executor_artifacts ORDER BY updated_at ASC`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"tool-approvals\",\n\t\t\t`SELECT * FROM tool_approvals ORDER BY timestamp ASC`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"compaction-summaries\",\n\t\t\t`SELECT cut_message_id, created_at FROM compaction_summaries ORDER BY created_at ASC`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"executor-status\",\n\t\t\t`SELECT value FROM thread_meta_kv WHERE key = 'executor_status'`,\n\t\t),\n\t]);\n\tsteps.sort((a, b) => b.durationMs - a.durationMs);\n\treturn {\n\t\tname: \"catchup-snapshot\",\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tsteps,\n\t};\n}\n\nasync function runRecoverToolCalls(\n\tsql: AgentConcurrent2Db,\n\tstats: AgentConcurrent2QueryStatsSet,\n): Promise {\n\tconst startedAt = performance.now();\n\tconst steps: AgentConcurrent2Step[] = [];\n\tawait timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"hydrate-tool-progress\",\n\t\t`SELECT id, progress\n\t\t\tFROM tool_calls\n\t\t\tWHERE progress IS NOT NULL\n\t\t\t\tAND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`,\n\t);\n\tawait timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"get-pending-tool-calls\",\n\t\t`SELECT * FROM tool_calls\n\t\t\tWHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')\n\t\t\tORDER BY issued_at ASC`,\n\t);\n\tawait timedQuery(\n\t\tsql,\n\t\tstats,\n\t\tsteps,\n\t\t\"get-next-tool-expiry\",\n\t\t`SELECT MIN(expires_at) AS expires_at\n\t\t\tFROM tool_calls\n\t\t\tWHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`,\n\t);\n\treturn {\n\t\tname: \"recover-tool-calls\",\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tsteps,\n\t};\n}\n\nasync function runMutationMix(\n\tsql: AgentConcurrent2Db,\n\tclientId: string,\n\tstats: AgentConcurrent2QueryStatsSet,\n): Promise {\n\tconst startedAt = performance.now();\n\tconst steps: AgentConcurrent2Step[] = [];\n\tconst writeCount = await sql.withTransaction(stats, async (tx) => {\n\t\tconst now = new Date().toISOString();\n\t\tconst suffix = safeId(clientId);\n\t\tconst seqRows = await timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"select-max-thread-event-seq\",\n\t\t\t`SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events`,\n\t\t);\n\t\tconst seq = Number(seqRows[0]?.seq ?? 1);\n\t\tconst lastMessageRows = await timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"select-last-message-created-at\",\n\t\t\t`SELECT MAX(created_at) AS created_at FROM messages`,\n\t\t);\n\t\tconst latestToolRows = await timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"select-existing-tool-call\",\n\t\t\t`SELECT id FROM tool_calls ORDER BY issued_at DESC LIMIT 1`,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"select-sandbox-row\",\n\t\t\t`SELECT sandbox_id, restart_attempts, traffic_access_token, project_id, repository_url, additional_repositories, setup\n\t\t\t\tFROM e2b_sandbox\n\t\t\t\tWHERE id = 1`,\n\t\t);\n\n\t\tconst messageIdValue = `agent2-message-${suffix}-${seq}`;\n\t\tconst toolUseIdValue = `agent2-tool-${suffix}-${seq}`;\n\t\tconst toolCallIdValue = `agent2-call-${suffix}-${seq}`;\n\t\tconst latestToolCallId = String(latestToolRows[0]?.id ?? toolUseID(1));\n\t\tconst lastCreatedAt = String(lastMessageRows[0]?.created_at ?? now);\n\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"upsert-agent-state\",\n\t\t\t`INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES (?, ?, ?)`,\n\t\t\t\"last_agent_state\",\n\t\t\tJSON.stringify({ status: \"working\", clientId, lastCreatedAt }),\n\t\t\tnow,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"insert-work-event\",\n\t\t\t`INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`,\n\t\t\tseq,\n\t\t\t\"message_added\",\n\t\t\tJSON.stringify({ type: \"message_added\", messageId: messageIdValue }),\n\t\t\tnow,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"insert-message\",\n\t\t\t`INSERT INTO messages (role, content, meta, user_state, message_id, created_at, cancelled, parent_tool_use_id, tool_result_for_message_id)\n\t\t\t\tVALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n\t\t\t\"assistant\",\n\t\t\t\"agent concurrent 2 mutation payload\",\n\t\t\tJSON.stringify({ clientId, seq }),\n\t\t\tnull,\n\t\t\tmessageIdValue,\n\t\t\tnow,\n\t\t\t0,\n\t\t\tnull,\n\t\t\tnull,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"delete-message-tool-refs\",\n\t\t\t`DELETE FROM message_tool_refs WHERE source_message_id = ?`,\n\t\t\tmessageIdValue,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"insert-message-added-event\",\n\t\t\t`INSERT OR IGNORE INTO message_added_events (message_id, seq) VALUES (?, ?)`,\n\t\t\tmessageIdValue,\n\t\t\tseq,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"insert-message-tool-ref\",\n\t\t\t`INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled)\n\t\t\t\tVALUES (?, ?, ?, ?, ?)`,\n\t\t\tmessageIdValue,\n\t\t\tmessageIdValue,\n\t\t\ttoolUseIdValue,\n\t\t\t\"tool_use\",\n\t\t\t0,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"insert-tool-call\",\n\t\t\t`INSERT OR IGNORE INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at)\n\t\t\t\tVALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n\t\t\ttoolCallIdValue,\n\t\t\t`provider-${toolCallIdValue}`,\n\t\t\t\"tool_1\",\n\t\t\tJSON.stringify({ path: `/tmp/${toolCallIdValue}` }),\n\t\t\t\"seed-executor\",\n\t\t\tmessageIdValue,\n\t\t\tnow,\n\t\t\tnull,\n\t\t\t\"running\",\n\t\t\tnull,\n\t\t\tJSON.stringify({ pct: 0.5, clientId }),\n\t\t\tnull,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"update-tool-call-progress\",\n\t\t\t`UPDATE tool_calls SET progress = ? WHERE id = ? AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`,\n\t\t\tJSON.stringify({ pct: 0.75, clientId, updatedAt: now }),\n\t\t\ttoolCallIdValue,\n\t\t);\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tstats,\n\t\t\tsteps,\n\t\t\t\"update-existing-tool-call-progress\",\n\t\t\t`UPDATE tool_calls SET progress = ? WHERE id = ? AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`,\n\t\t\tJSON.stringify({ pct: 0.25, clientId, updatedAt: now }),\n\t\t\tlatestToolCallId,\n\t\t);\n\n\t\treturn seq;\n\t});\n\tsteps.push({\n\t\tname: \"transaction-total\",\n\t\tdurationMs: Math.round(performance.now() - startedAt),\n\t\trowCount: writeCount,\n\t});\n\treturn {\n\t\tname: \"mutation-mix\",\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tsteps,\n\t};\n}\n\nasync function timedQuery>(\n\tsql: AgentConcurrent2Db,\n\tstats: AgentConcurrent2QueryStatsSet,\n\tsteps: AgentConcurrent2Step[],\n\tname: string,\n\tquery: string,\n\t...values: SQLPrimitive[]\n): Promise {\n\tconst startedAt = performance.now();\n\ttry {\n\t\tconst rows = await sql(query, ...values);\n\t\tconst durationMs = Math.round(performance.now() - startedAt);\n\t\trecordAgentConcurrent2Query(stats, name, query, durationMs, rows.length, false);\n\t\tsteps.push({\n\t\t\tname,\n\t\t\tdurationMs,\n\t\t\trowCount: rows.length,\n\t\t});\n\t\treturn rows;\n\t} catch (error) {\n\t\tconst durationMs = Math.round(performance.now() - startedAt);\n\t\trecordAgentConcurrent2Query(stats, name, query, durationMs, 0, true);\n\t\tthrow error;\n\t}\n}\n\nasync function executeTrackedQuery>(\n\texecute: >(\n\t\tquery: string,\n\t\t...values: SQLPrimitive[]\n\t) => Promise,\n\tstats: AgentConcurrent2QueryStatsSet,\n\tname: string,\n\tquery: string,\n\t...values: SQLPrimitive[]\n): Promise {\n\tconst startedAt = performance.now();\n\ttry {\n\t\tconst rows = await execute(query, ...values);\n\t\trecordAgentConcurrent2Query(\n\t\t\tstats,\n\t\t\tname,\n\t\t\tquery,\n\t\t\tMath.round(performance.now() - startedAt),\n\t\t\trows.length,\n\t\t\tfalse,\n\t\t);\n\t\treturn rows;\n\t} catch (error) {\n\t\trecordAgentConcurrent2Query(\n\t\t\tstats,\n\t\t\tname,\n\t\t\tquery,\n\t\t\tMath.round(performance.now() - startedAt),\n\t\t\t0,\n\t\t\ttrue,\n\t\t);\n\t\tthrow error;\n\t}\n}\n\nfunction recordAgentConcurrent2Query(\n\tstats: AgentConcurrent2QueryStatsSet,\n\tname: string,\n\tquery: string,\n\tdurationMs: number,\n\trowCount: number,\n\tfailed: boolean,\n): void {\n\tconst classification = classifyAgentConcurrent2Query(query);\n\tfor (const target of [stats.cycle, stats.wake, stats.actor]) {\n\t\ttarget.total++;\n\t\ttarget.rows += rowCount;\n\t\tif (failed) target.errors++;\n\t\tif (durationMs >= SLOW_QUERY_MS) target.slow++;\n\t\tif (durationMs > target.maxMs) {\n\t\t\ttarget.maxMs = durationMs;\n\t\t\ttarget.maxStep = `${name}:${classification.table}`;\n\t\t}\n\t\ttarget.byOperation[classification.operation] =\n\t\t\t(target.byOperation[classification.operation] ?? 0) + 1;\n\t\ttarget.byTable[classification.table] =\n\t\t\t(target.byTable[classification.table] ?? 0) + 1;\n\t\tif (classification.kind === \"read\") {\n\t\t\ttarget.reads++;\n\t\t} else if (classification.kind === \"mutation\") {\n\t\t\ttarget.mutations++;\n\t\t} else if (classification.kind === \"tx\") {\n\t\t\ttarget.tx++;\n\t\t} else {\n\t\t\ttarget.other++;\n\t\t}\n\t}\n}\n\nfunction classifyAgentConcurrent2Query(query: string): {\n\toperation: string;\n\tkind: \"read\" | \"mutation\" | \"tx\" | \"other\";\n\ttable: string;\n} {\n\tconst normalized = query.trim().replace(/\\s+/g, \" \");\n\tconst operation = normalized.match(/^([a-z]+)/i)?.[1]?.toLowerCase() ?? \"other\";\n\tconst table = extractAgentConcurrent2Table(normalized, operation);\n\tif (operation === \"select\") {\n\t\treturn { operation, kind: \"read\", table };\n\t}\n\tif (\n\t\toperation === \"insert\" ||\n\t\toperation === \"update\" ||\n\t\toperation === \"delete\" ||\n\t\toperation === \"replace\"\n\t) {\n\t\treturn { operation, kind: \"mutation\", table };\n\t}\n\tif (operation === \"begin\" || operation === \"commit\" || operation === \"rollback\") {\n\t\treturn { operation, kind: \"tx\", table };\n\t}\n\treturn { operation, kind: \"other\", table };\n}\n\nfunction extractAgentConcurrent2Table(query: string, operation: string): string {\n\tconst lower = query.toLowerCase();\n\tif (operation === \"select\") {\n\t\treturn firstMatch(lower, /\\bfrom\\s+([a-z0-9_]+)/) ?? \"unknown\";\n\t}\n\tif (operation === \"insert\" || operation === \"replace\") {\n\t\treturn firstMatch(lower, /\\binto\\s+([a-z0-9_]+)/) ?? \"unknown\";\n\t}\n\tif (operation === \"update\") {\n\t\treturn firstMatch(lower, /\\bupdate\\s+([a-z0-9_]+)/) ?? \"unknown\";\n\t}\n\tif (operation === \"delete\") {\n\t\treturn firstMatch(lower, /\\bfrom\\s+([a-z0-9_]+)/) ?? \"unknown\";\n\t}\n\tif (operation === \"begin\" || operation === \"commit\" || operation === \"rollback\") {\n\t\treturn \"transaction\";\n\t}\n\treturn \"unknown\";\n}\n\nfunction firstMatch(value: string, pattern: RegExp): string | null {\n\treturn pattern.exec(value)?.[1] ?? null;\n}\n\nfunction hasPendingLaunch(value: unknown): boolean {\n\tif (typeof value !== \"string\" || value.length === 0) {\n\t\treturn false;\n\t}\n\ttry {\n\t\tconst parsed = JSON.parse(value) as { pendingLaunch?: unknown };\n\t\treturn parsed.pendingLaunch !== null && parsed.pendingLaunch !== undefined;\n\t} catch {\n\t\treturn false;\n\t}\n}\n\nfunction delay(ms: number): Promise {\n\treturn new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));\n}\n\nasync function createAgentConcurrent2Schema(database: RawRivetDB): Promise {\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS executor_tools (\n\t\texecutor_id TEXT NOT NULL,\n\t\ttool_name TEXT NOT NULL,\n\t\tschema TEXT NOT NULL,\n\t\tupdated_at TEXT NOT NULL,\n\t\tPRIMARY KEY (executor_id, tool_name)\n\t)`);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_executor_tools_executor ON executor_tools(executor_id)`,\n\t);\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS thread_meta_kv (\n\t\tkey TEXT PRIMARY KEY,\n\t\tvalue TEXT,\n\t\tupdated_at TEXT NOT NULL\n\t)`);\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS thread_events (\n\t\tseq INTEGER PRIMARY KEY,\n\t\tevent_type TEXT NOT NULL,\n\t\tpayload TEXT NOT NULL,\n\t\tcreated_at TEXT NOT NULL DEFAULT (datetime('now'))\n\t)`);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_thread_events_seq ON thread_events(seq)`,\n\t);\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS message_added_events (\n\t\tmessage_id TEXT PRIMARY KEY,\n\t\tseq INTEGER NOT NULL UNIQUE\n\t)`);\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS messages (\n\t\tmessage_id TEXT PRIMARY KEY,\n\t\trole TEXT NOT NULL CHECK(role IN ('user', 'assistant', 'info')),\n\t\tcontent TEXT NOT NULL,\n\t\tmeta TEXT,\n\t\tuser_state TEXT,\n\t\tcreated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),\n\t\tcancelled INTEGER NOT NULL DEFAULT 0,\n\t\tread_at TEXT,\n\t\tparent_tool_use_id TEXT,\n\t\ttool_result_for_message_id TEXT\n\t)`);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_messages_parent_role_cancelled_created_at ON messages(parent_tool_use_id, role, cancelled, created_at DESC)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_messages_parent_cancelled_created_at ON messages(parent_tool_use_id, cancelled, created_at DESC)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_messages_parent_created_at ON messages(parent_tool_use_id, created_at DESC)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_messages_role_created_at ON messages(role, created_at)`,\n\t);\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS message_tool_refs (\n\t\tsource_message_id TEXT NOT NULL,\n\t\tassistant_message_id TEXT NOT NULL,\n\t\ttool_use_id TEXT NOT NULL,\n\t\tblock_type TEXT NOT NULL CHECK(block_type IN ('tool_use', 'tool_result')),\n\t\tcancelled INTEGER NOT NULL DEFAULT 0,\n\t\tPRIMARY KEY (source_message_id, block_type, tool_use_id)\n\t)`);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_message_tool_refs_assistant_lookup ON message_tool_refs(assistant_message_id, block_type, cancelled, tool_use_id)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE UNIQUE INDEX IF NOT EXISTS idx_message_tool_refs_live_tool_result ON message_tool_refs(assistant_message_id, tool_use_id) WHERE block_type = 'tool_result' AND cancelled = 0`,\n\t);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_message_tool_refs_source_message ON message_tool_refs(source_message_id)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_message_tool_refs_tool_use_lookup ON message_tool_refs(tool_use_id, assistant_message_id) WHERE block_type = 'tool_use' AND cancelled = 0`,\n\t);\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS tool_calls (\n\t\tid TEXT PRIMARY KEY,\n\t\tprovider_tool_use_id TEXT NOT NULL,\n\t\ttool_name TEXT NOT NULL,\n\t\targs TEXT NOT NULL,\n\t\texecutor_id TEXT,\n\t\tmessage_id TEXT NOT NULL,\n\t\tissued_at TEXT NOT NULL,\n\t\texpires_at TEXT,\n\t\tstate TEXT NOT NULL CHECK(state IN ('queued', 'pending_reconnect', 'pending_ack', 'running', 'completed', 'expired', 'revoked')),\n\t\tresult TEXT,\n\t\tprogress TEXT,\n\t\tcompleted_at TEXT\n\t)`);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_tool_calls_message_id ON tool_calls(message_id)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_tool_calls_state ON tool_calls(state)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_tool_calls_expires_at ON tool_calls(expires_at) WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS environment_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), snapshot TEXT NOT NULL, updated_at TEXT NOT NULL)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS thread_settings_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), settings TEXT NOT NULL, updated_at TEXT NOT NULL)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS retry_state (id INTEGER PRIMARY KEY CHECK (id = 1), attempt INTEGER NOT NULL DEFAULT 0, scheduled_at INTEGER NOT NULL, reason TEXT NOT NULL)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS queued_messages (message_id TEXT PRIMARY KEY, content TEXT NOT NULL, user_state TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), steer INTEGER NOT NULL DEFAULT 0, user_meta TEXT)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS executor_artifacts (artifact_key TEXT PRIMARY KEY, data_type TEXT NOT NULL, content_base64 TEXT NOT NULL, tool_call_id TEXT, updated_at TEXT NOT NULL)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS e2b_sandbox (\n\t\t\tid INTEGER PRIMARY KEY CHECK (id = 1),\n\t\t\tsandbox_id TEXT,\n\t\t\trestart_attempts INTEGER NOT NULL DEFAULT 0,\n\t\t\ttraffic_access_token TEXT,\n\t\t\tproject_id TEXT,\n\t\t\trepository_url TEXT,\n\t\t\tadditional_repositories TEXT,\n\t\t\tsetup TEXT,\n\t\t\tcreated_at TEXT NOT NULL,\n\t\t\tupdated_at TEXT NOT NULL\n\t\t)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS tool_approvals (id TEXT PRIMARY KEY, tool_call_id TEXT NOT NULL UNIQUE, tool_name TEXT NOT NULL, args TEXT NOT NULL, reason TEXT, to_allow TEXT, context TEXT NOT NULL CHECK(context IN ('thread', 'subagent')), subagent_tool_name TEXT, parent_tool_call_id TEXT, timestamp INTEGER NOT NULL, matched_rule TEXT, rule_source TEXT CHECK(rule_source IN ('user', 'built-in')))`,\n\t);\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_tool_approvals_timestamp ON tool_approvals(timestamp)`,\n\t);\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS compaction_summaries (summary_id TEXT PRIMARY KEY, summary_text TEXT NOT NULL, cut_message_id TEXT NOT NULL, created_at TEXT NOT NULL)`,\n\t);\n}\n\nasync function seedAgentConcurrent2Data(database: RawRivetDB): Promise {\n\tconst existing = await database.execute(`SELECT COUNT(*) AS count FROM messages`);\n\tif (Number(existing[0]?.count ?? 0) > 0) {\n\t\treturn;\n\t}\n\n\tconst now = new Date(\"2026-05-16T03:58:18.661Z\").getTime();\n\tconst text = (size: number) => \"x\".repeat(size);\n\tconst isoAt = (index: number) => new Date(now + index * 1_000).toISOString();\n\n\tawait batchInsert(database, `INSERT INTO thread_meta_kv (key, value, updated_at)`, [\n\t\t[\"executor_type\", \"local-client\", isoAt(0)],\n\t\t[\"workspace_intent\", JSON.stringify({ spec: null, pendingLaunch: null }), isoAt(0)],\n\t\t[\"executor_status\", JSON.stringify({ available: true, message: \"ready\" }), isoAt(0)],\n\t]);\n\n\tconst messageRows: unknown[][] = [];\n\tfor (let index = 1; index <= MESSAGE_COUNT; index++) {\n\t\tconst role = index % 2 === 0 ? \"assistant\" : \"user\";\n\t\tmessageRows.push([\n\t\t\tmessageId(index),\n\t\t\trole,\n\t\t\ttext(MESSAGE_CONTENT_BYTES),\n\t\t\tnull,\n\t\t\tnull,\n\t\t\tisoAt(index),\n\t\t\t0,\n\t\t\tnull,\n\t\t\tnull,\n\t\t\tnull,\n\t\t]);\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO messages (message_id, role, content, meta, user_state, created_at, cancelled, read_at, parent_tool_use_id, tool_result_for_message_id)`,\n\t\tmessageRows,\n\t\t20,\n\t);\n\n\tconst messageToolRefRows: unknown[][] = [];\n\tfor (let index = 0; index < MESSAGE_TOOL_REF_COUNT / 2; index++) {\n\t\tconst assistantIndex = 2 + (index % 42) * 2;\n\t\tconst sourceIndex = Math.max(1, assistantIndex - 1);\n\t\tconst resultIndex = Math.min(MESSAGE_COUNT, assistantIndex + 1);\n\t\tconst toolUseId = toolUseID(index + 1);\n\t\tmessageToolRefRows.push([\n\t\t\tmessageId(sourceIndex),\n\t\t\tmessageId(assistantIndex),\n\t\t\ttoolUseId,\n\t\t\t\"tool_use\",\n\t\t\t0,\n\t\t]);\n\t\tmessageToolRefRows.push([\n\t\t\tmessageId(resultIndex),\n\t\t\tmessageId(assistantIndex),\n\t\t\ttoolUseId,\n\t\t\t\"tool_result\",\n\t\t\t0,\n\t\t]);\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled)`,\n\t\tmessageToolRefRows,\n\t\t50,\n\t);\n\n\tconst toolCallRows: unknown[][] = [];\n\tfor (let index = 1; index <= TOOL_CALL_COUNT; index++) {\n\t\tconst assistantIndex = 2 + ((index - 1) % 42) * 2;\n\t\ttoolCallRows.push([\n\t\t\ttoolUseID(index),\n\t\t\t`provider-${index}`,\n\t\t\t`tool_${index % 21}`,\n\t\t\tJSON.stringify({ path: `/tmp/file-${index}` }),\n\t\t\t\"seed-executor\",\n\t\t\tmessageId(assistantIndex),\n\t\t\tisoAt(index),\n\t\t\tnull,\n\t\t\t\"completed\",\n\t\t\tJSON.stringify({\n\t\t\t\tok: true,\n\t\t\t\trun: { status: \"done\", result: text(TOOL_CALL_RESULT_BYTES) },\n\t\t\t}),\n\t\t\tnull,\n\t\t\tisoAt(index + 100),\n\t\t]);\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at)`,\n\t\ttoolCallRows,\n\t\t20,\n\t);\n\n\tconst executorToolRows: unknown[][] = [];\n\tfor (let index = 1; index <= EXECUTOR_TOOL_COUNT; index++) {\n\t\tconst schema = JSON.stringify({\n\t\t\tname: `tool_${index}`,\n\t\t\tdescription: text(EXECUTOR_TOOL_SCHEMA_BYTES),\n\t\t\tinput_schema: { type: \"object\", properties: {} },\n\t\t});\n\t\texecutorToolRows.push([\"seed-executor\", `tool_${index}`, schema, isoAt(index)]);\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO executor_tools (executor_id, tool_name, schema, updated_at)`,\n\t\texecutorToolRows,\n\t\t42,\n\t);\n\n\tconst threadEventRows: unknown[][] = [];\n\tfor (let index = 1; index <= THREAD_EVENT_COUNT; index++) {\n\t\tthreadEventRows.push([\n\t\t\tindex,\n\t\t\tindex % 3 === 0 ? \"message_added\" : \"agent_state_changed\",\n\t\t\tJSON.stringify({ type: \"seed_event\", body: text(THREAD_EVENT_PAYLOAD_BYTES) }),\n\t\t\tisoAt(index),\n\t\t]);\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO thread_events (seq, event_type, payload, created_at)`,\n\t\tthreadEventRows,\n\t\t25,\n\t);\n\n\tconst messageAddedRows: unknown[][] = [];\n\tfor (let index = 1; index <= MESSAGE_COUNT; index++) {\n\t\tmessageAddedRows.push([messageId(index), index]);\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO message_added_events (message_id, seq)`,\n\t\tmessageAddedRows,\n\t\t50,\n\t);\n\n\tawait database.execute(\n\t\t`INSERT INTO environment_snapshot (id, snapshot, updated_at) VALUES (1, ?, ?)`,\n\t\tJSON.stringify({ cwd: \"/workspace\", body: text(3_620) }),\n\t\tisoAt(0),\n\t);\n\tawait database.execute(\n\t\t`INSERT INTO thread_settings_snapshot (id, settings, updated_at) VALUES (1, ?, ?)`,\n\t\tJSON.stringify({ maxTokens: 20_000, body: text(55) }),\n\t\tisoAt(0),\n\t);\n\tawait database.execute(\n\t\t`INSERT INTO e2b_sandbox (id, sandbox_id, restart_attempts, traffic_access_token, project_id, repository_url, additional_repositories, setup, created_at, updated_at)\n\t\t\tVALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n\t\t\"sandbox-seed\",\n\t\t0,\n\t\t\"token-seed\",\n\t\t\"project-seed\",\n\t\t\"https://example.invalid/repo.git\",\n\t\tJSON.stringify([]),\n\t\tJSON.stringify({ commands: [] }),\n\t\tisoAt(0),\n\t\tisoAt(0),\n\t);\n}\n\nasync function batchInsert(\n\tdatabase: RawRivetDB,\n\tinsertPrefix: string,\n\trows: unknown[][],\n\tbatchSize = 100,\n): Promise {\n\tif (rows.length === 0) {\n\t\treturn;\n\t}\n\tconst columnCount = rows[0]?.length ?? 0;\n\tif (columnCount === 0) {\n\t\treturn;\n\t}\n\tconst rowPlaceholder = `(${\"?,\".repeat(columnCount).slice(0, -1)})`;\n\tfor (let index = 0; index < rows.length; index += batchSize) {\n\t\tconst chunk = rows.slice(index, index + batchSize);\n\t\tconst values = chunk.map(() => rowPlaceholder).join(\",\");\n\t\tconst bindings = chunk.flat();\n\t\tawait database.execute(`${insertPrefix} VALUES ${values}`, ...bindings);\n\t}\n}\n\nfunction messageId(index: number): string {\n\treturn `M-${String(index).padStart(22, \"0\")}`;\n}\n\nfunction toolUseID(index: number): string {\n\treturn `toolu_${String(index).padStart(22, \"0\")}`;\n}\n\nfunction safeId(value: string): string {\n\treturn value.replace(/[^a-zA-Z0-9_-]/g, \"-\").slice(0, 80);\n}\n","import { actor, type RivetMessageEvent, type UniversalWebSocket } from \"rivetkit\";\nimport { db } from \"rivetkit/db\";\n\nexport const DEFAULT_ON_SLEEP_DURATION_MS = 5_000;\nexport const DEFAULT_ON_SLEEP_TICK_MS = 1_000;\nconst SLEEP_TIMEOUT_MS = 10 * 60 * 1000;\nconst SLEEP_GRACE_PERIOD_MS = 30 * 60 * 1000;\nconst ACTOR_STOPPED_CLOSE_CODE = 1000;\nconst ACTOR_STOPPED_CLOSE_REASON = \"actor stopped\";\n\nfunction sleep(ms: number): Promise {\n\treturn new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nfunction formatError(error: unknown): string {\n\tif (error instanceof Error) return error.stack ?? error.message;\n\treturn String(error);\n}\n\nexport const sigtermSleepProbe = actor({\n\tstate: {\n\t\tlabel: \"unprepared\",\n\t\twakeCount: 0,\n\t\tsleepCount: 0,\n\t\tonSleepDurationMs: DEFAULT_ON_SLEEP_DURATION_MS,\n\t\tonSleepTickMs: DEFAULT_ON_SLEEP_TICK_MS,\n\t\tconnectionCount: 0,\n\t\tmessageCount: 0,\n\t\tonSleepStartedAt: null as number | null,\n\t\tonSleepAsyncFinishedAt: null as number | null,\n\t\tonSleepFinishedAt: null as number | null,\n\t\tonSleepLastError: null as string | null,\n\t},\n\tcreateVars: () => ({\n\t\twebsockets: new Set(),\n\t}),\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait database.execute(`\n\t\t\t\tCREATE TABLE IF NOT EXISTS sigterm_sleep_log (\n\t\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\t\tevent TEXT NOT NULL,\n\t\t\t\t\tsleep_count INTEGER NOT NULL,\n\t\t\t\t\tdetail TEXT,\n\t\t\t\t\tcreated_at INTEGER NOT NULL\n\t\t\t\t)\n\t\t\t`);\n\t\t},\n\t}),\n\tonWebSocket: (c, websocket: UniversalWebSocket) => {\n\t\tc.vars.websockets.add(websocket);\n\t\tc.state.connectionCount += 1;\n\t\tconst connectionId = crypto.randomUUID();\n\n\t\tc.log.info({\n\t\t\tmsg: \"sigterm sleep probe websocket connected\",\n\t\t\tlabel: c.state.label,\n\t\t\tconnectionId,\n\t\t\tconnectionCount: c.state.connectionCount,\n\t\t});\n\n\t\twebsocket.send(\n\t\t\tJSON.stringify({\n\t\t\t\ttype: \"welcome\",\n\t\t\t\tconnectionId,\n\t\t\t\tlabel: c.state.label,\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t}),\n\t\t);\n\n\t\twebsocket.addEventListener(\"message\", (event: RivetMessageEvent) => {\n\t\t\tc.state.messageCount += 1;\n\t\t\tconst data = event.data;\n\t\t\tif (typeof data !== \"string\") return;\n\n\t\t\ttry {\n\t\t\t\tconst parsed = JSON.parse(data);\n\t\t\t\tif (parsed.type === \"ping\") {\n\t\t\t\t\twebsocket.send(\n\t\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\t\ttype: \"pong\",\n\t\t\t\t\t\t\tconnectionId,\n\t\t\t\t\t\t\tmessageCount: c.state.messageCount,\n\t\t\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t} catch {}\n\n\t\t\twebsocket.send(\n\t\t\t\tJSON.stringify({\n\t\t\t\t\ttype: \"echo\",\n\t\t\t\t\tconnectionId,\n\t\t\t\t\treceived: data,\n\t\t\t\t\tmessageCount: c.state.messageCount,\n\t\t\t\t\ttimestamp: Date.now(),\n\t\t\t\t}),\n\t\t\t);\n\t\t});\n\n\t\twebsocket.addEventListener(\"close\", (event) => {\n\t\t\tc.vars.websockets.delete(websocket);\n\t\t\tc.state.connectionCount -= 1;\n\t\t\tc.log.info({\n\t\t\t\tmsg: \"sigterm sleep probe websocket closed\",\n\t\t\t\tlabel: c.state.label,\n\t\t\t\tconnectionId,\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t\tcode: event.code,\n\t\t\t\treason: event.reason,\n\t\t\t});\n\t\t});\n\t},\n\tonWake: async (c) => {\n\t\tc.state.wakeCount += 1;\n\t\tc.log.info({\n\t\t\tmsg: \"sigterm sleep probe onWake\",\n\t\t\tlabel: c.state.label,\n\t\t\twakeCount: c.state.wakeCount,\n\t\t\tsleepCount: c.state.sleepCount,\n\t\t});\n\t\tawait c.db.execute(\n\t\t\t\"INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\"wake\",\n\t\t\tc.state.sleepCount,\n\t\t\t`wake-${c.state.wakeCount}`,\n\t\t\tDate.now(),\n\t\t);\n\t},\n\tonSleep: async (c) => {\n\t\tconst sleepCount = c.state.sleepCount + 1;\n\t\tconst startedAt = Date.now();\n\t\tc.state.sleepCount = sleepCount;\n\t\tc.state.onSleepStartedAt = startedAt;\n\t\tc.state.onSleepAsyncFinishedAt = null;\n\t\tc.state.onSleepFinishedAt = null;\n\t\tc.state.onSleepLastError = null;\n\n\t\tc.log.info({\n\t\t\tmsg: \"sigterm sleep probe onSleep start\",\n\t\t\tlabel: c.state.label,\n\t\t\tsleepCount,\n\t\t\tonSleepDurationMs: c.state.onSleepDurationMs,\n\t\t\tonSleepTickMs: c.state.onSleepTickMs,\n\t\t});\n\n\t\ttry {\n\t\t\tfor (const websocket of c.vars.websockets) {\n\t\t\t\tif (websocket.readyState !== 1) continue;\n\t\t\t\twebsocket.send(\n\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\ttype: \"onSleepStarted\",\n\t\t\t\t\t\tsleepCount,\n\t\t\t\t\t\tonSleepDurationMs: c.state.onSleepDurationMs,\n\t\t\t\t\t\tonSleepTickMs: c.state.onSleepTickMs,\n\t\t\t\t\t\ttimestamp: startedAt,\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\"on-sleep-start\",\n\t\t\t\tsleepCount,\n\t\t\t\tc.state.label,\n\t\t\t\tstartedAt,\n\t\t\t);\n\n\t\t\tconst deadline = startedAt + c.state.onSleepDurationMs;\n\t\t\tlet tickIndex = 0;\n\t\t\twhile (Date.now() < deadline) {\n\t\t\t\tconst waitMs = Math.min(\n\t\t\t\t\tc.state.onSleepTickMs,\n\t\t\t\t\tMath.max(0, deadline - Date.now()),\n\t\t\t\t);\n\t\t\t\tif (waitMs > 0) await sleep(waitMs);\n\n\t\t\t\ttickIndex += 1;\n\t\t\t\tconst tickAt = Date.now();\n\t\t\t\tconst detail = `tick=${tickIndex} elapsed-ms=${tickAt - startedAt}`;\n\t\t\t\tawait c.db.execute(\n\t\t\t\t\t\"INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\t\"on-sleep-tick\",\n\t\t\t\t\tsleepCount,\n\t\t\t\t\tdetail,\n\t\t\t\t\ttickAt,\n\t\t\t\t);\n\t\t\t\tc.log.info({\n\t\t\t\t\tmsg: \"sigterm sleep probe onSleep tick\",\n\t\t\t\t\tlabel: c.state.label,\n\t\t\t\t\tsleepCount,\n\t\t\t\t\ttickIndex,\n\t\t\t\t\telapsedMs: tickAt - startedAt,\n\t\t\t\t});\n\n\t\t\t\tfor (const websocket of c.vars.websockets) {\n\t\t\t\t\tif (websocket.readyState !== 1) continue;\n\t\t\t\t\twebsocket.send(\n\t\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\t\ttype: \"onSleepTick\",\n\t\t\t\t\t\t\tsleepCount,\n\t\t\t\t\t\t\ttickIndex,\n\t\t\t\t\t\t\telapsedMs: tickAt - startedAt,\n\t\t\t\t\t\t\ttimestamp: tickAt,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tconst asyncFinishedAt = Date.now();\n\t\t\tc.state.onSleepAsyncFinishedAt = asyncFinishedAt;\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\"on-sleep-after-await\",\n\t\t\t\tsleepCount,\n\t\t\t\t`delay-ms=${asyncFinishedAt - startedAt}`,\n\t\t\t\tasyncFinishedAt,\n\t\t\t);\n\n\t\t\tconst finishedAt = Date.now();\n\t\t\tc.state.onSleepFinishedAt = finishedAt;\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\"on-sleep-finish\",\n\t\t\t\tsleepCount,\n\t\t\t\tc.state.label,\n\t\t\t\tfinishedAt,\n\t\t\t);\n\n\t\t\tfor (const websocket of c.vars.websockets) {\n\t\t\t\tif (websocket.readyState !== 1) continue;\n\t\t\t\twebsocket.send(\n\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\ttype: \"onSleepFinished\",\n\t\t\t\t\t\tsleepCount,\n\t\t\t\t\t\telapsedMs: finishedAt - startedAt,\n\t\t\t\t\t\ttimestamp: finishedAt,\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t\twebsocket.close(\n\t\t\t\t\tACTOR_STOPPED_CLOSE_CODE,\n\t\t\t\t\tACTOR_STOPPED_CLOSE_REASON,\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tc.log.info({\n\t\t\t\tmsg: \"sigterm sleep probe onSleep finish\",\n\t\t\t\tlabel: c.state.label,\n\t\t\t\tsleepCount,\n\t\t\t\telapsedMs: finishedAt - startedAt,\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tconst message = formatError(error);\n\t\t\tc.state.onSleepLastError = message;\n\t\t\tc.log.error({\n\t\t\t\tmsg: \"sigterm sleep probe onSleep error\",\n\t\t\t\tlabel: c.state.label,\n\t\t\t\tsleepCount,\n\t\t\t\terror: message,\n\t\t\t});\n\t\t\tthrow error;\n\t\t}\n\t},\n\tactions: {\n\t\tprepare: async (\n\t\t\tc,\n\t\t\tlabel = `sigterm-sleep-probe-${Date.now()}`,\n\t\t\tonSleepDurationMs = DEFAULT_ON_SLEEP_DURATION_MS,\n\t\t\tonSleepTickMs = DEFAULT_ON_SLEEP_TICK_MS,\n\t\t) => {\n\t\t\tif (!Number.isFinite(onSleepDurationMs) || onSleepDurationMs < 0) {\n\t\t\t\tthrow new Error(\"onSleepDurationMs must be a finite non-negative number\");\n\t\t\t}\n\t\t\tif (!Number.isFinite(onSleepTickMs) || onSleepTickMs <= 0) {\n\t\t\t\tthrow new Error(\"onSleepTickMs must be a finite positive number\");\n\t\t\t}\n\t\t\tc.state.label = label;\n\t\t\tc.state.onSleepDurationMs = onSleepDurationMs;\n\t\t\tc.state.onSleepTickMs = onSleepTickMs;\n\t\t\tawait c.db.execute(\n\t\t\t\t\"INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)\",\n\t\t\t\t\"prepared\",\n\t\t\t\tc.state.sleepCount,\n\t\t\t\tlabel,\n\t\t\t\tDate.now(),\n\t\t\t);\n\t\t\treturn {\n\t\t\t\tlabel: c.state.label,\n\t\t\t\tonSleepDurationMs: c.state.onSleepDurationMs,\n\t\t\t\tonSleepTickMs: c.state.onSleepTickMs,\n\t\t\t\twakeCount: c.state.wakeCount,\n\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t\tmessageCount: c.state.messageCount,\n\t\t\t};\n\t\t},\n\t\tgetProof: async (c) => {\n\t\t\tconst rows = await c.db.execute<{\n\t\t\t\tid: number;\n\t\t\t\tevent: string;\n\t\t\t\tsleep_count: number;\n\t\t\t\tdetail: string | null;\n\t\t\t\tcreated_at: number;\n\t\t\t}>(\"SELECT * FROM sigterm_sleep_log ORDER BY id\");\n\t\t\treturn {\n\t\t\t\tstate: {\n\t\t\t\t\tlabel: c.state.label,\n\t\t\t\t\twakeCount: c.state.wakeCount,\n\t\t\t\t\tsleepCount: c.state.sleepCount,\n\t\t\t\t\tonSleepDurationMs: c.state.onSleepDurationMs,\n\t\t\t\t\tonSleepTickMs: c.state.onSleepTickMs,\n\t\t\t\t\tconnectionCount: c.state.connectionCount,\n\t\t\t\t\tmessageCount: c.state.messageCount,\n\t\t\t\t\tonSleepStartedAt: c.state.onSleepStartedAt,\n\t\t\t\t\tonSleepAsyncFinishedAt: c.state.onSleepAsyncFinishedAt,\n\t\t\t\t\tonSleepFinishedAt: c.state.onSleepFinishedAt,\n\t\t\t\t\tonSleepLastError: c.state.onSleepLastError,\n\t\t\t\t},\n\t\t\t\trows,\n\t\t\t};\n\t\t},\n\t},\n\toptions: {\n\t\tcanHibernateWebSocket: false,\n\t\tsleepTimeout: SLEEP_TIMEOUT_MS,\n\t\tsleepGracePeriod: SLEEP_GRACE_PERIOD_MS,\n\t},\n});\n","import { actor, setup } from 'rivetkit'\nimport { db } from 'rivetkit/db'\n\nexport type SlowReconnectRequest =\n\t| { type: 'client_resume'; version: number }\n\t| {\n\t\t\ttype: 'executor_connect'\n\t\t\tclientId: string\n\t\t\texecutorType?: 'local-client' | 'sandbox' | 'virtual'\n\t }\n\t| {\n\t\t\ttype: 'repro_reconnect'\n\t\t\tclientId?: string\n\t\t\tstaggerHandleMs?: number\n\t }\n\nexport interface SlowReconnectStep {\n\tname: string\n\tdurationMs: number\n\trowCount: number\n}\n\nexport interface SlowReconnectWorkloadResult {\n\tname: string\n\ttotalMs: number\n\tsteps: SlowReconnectStep[]\n}\n\nexport interface SlowReconnectResultMessage {\n\ttype: 'slow_reconnect_result'\n\ttrigger: SlowReconnectRequest['type']\n\ttotalMs: number\n\tresults: SlowReconnectWorkloadResult[]\n}\n\nexport interface SlowReconnectErrorMessage {\n\ttype: 'slow_reconnect_error'\n\ttrigger: SlowReconnectRequest['type'] | 'unknown'\n\terror: string\n}\n\nexport interface SlowReconnectVars {\n\tsql: Db | null\n}\n\ninterface RawRivetDB {\n\texecute: (query: string, ...args: unknown[]) => Promise[]>\n}\n\ntype SQLPrimitive = string | number | boolean | null\n\ntype Db = (>(\n\tquery: string,\n\t...values: SQLPrimitive[]\n) => Promise) & {\n\twithTransaction(fn: (tx: Db) => Promise): Promise\n}\n\nclass AsyncMutex {\n\tprivate locked = false\n\tprivate waiters: Array<() => void> = []\n\n\tasync acquire(): Promise {\n\t\tif (!this.locked) {\n\t\t\tthis.locked = true\n\t\t\treturn\n\t\t}\n\t\tawait new Promise((resolve) => this.waiters.push(resolve))\n\t\tthis.locked = true\n\t}\n\n\trelease(): void {\n\t\tconst next = this.waiters.shift()\n\t\tif (next) {\n\t\t\tnext()\n\t\t\treturn\n\t\t}\n\t\tthis.locked = false\n\t}\n}\n\nfunction createDb(execute: >(\n\tquery: string,\n\t...values: SQLPrimitive[]\n) => Promise): Db {\n\tconst mutex = new AsyncMutex()\n\tlet activeTransaction: Db | null = null\n\n\tconst createTransactionDb = (): Db => {\n\t\tconst tx = Object.assign(\n\t\t\t>(query: string, ...values: SQLPrimitive[]) =>\n\t\t\t\texecute(query, ...values),\n\t\t\t{\n\t\t\t\twithTransaction: async (fn: (tx: Db) => Promise): Promise => fn(tx),\n\t\t\t},\n\t\t)\n\t\treturn tx\n\t}\n\n\tconst queryWithMutex = async >(\n\t\tquery: string,\n\t\t...values: SQLPrimitive[]\n\t): Promise => {\n\t\tif (activeTransaction) {\n\t\t\treturn activeTransaction(query, ...values)\n\t\t}\n\t\tawait mutex.acquire()\n\t\ttry {\n\t\t\treturn await execute(query, ...values)\n\t\t} finally {\n\t\t\tmutex.release()\n\t\t}\n\t}\n\n\tconst sql = Object.assign(queryWithMutex, {\n\t\twithTransaction: async (fn: (tx: Db) => Promise): Promise => {\n\t\t\tif (activeTransaction) {\n\t\t\t\treturn fn(activeTransaction)\n\t\t\t}\n\t\t\tawait mutex.acquire()\n\t\t\tconst tx = createTransactionDb()\n\t\t\ttry {\n\t\t\t\tawait execute('BEGIN')\n\t\t\t\tactiveTransaction = tx\n\t\t\t\ttry {\n\t\t\t\t\tconst result = await fn(tx)\n\t\t\t\t\tactiveTransaction = null\n\t\t\t\t\tawait execute('COMMIT')\n\t\t\t\t\treturn result\n\t\t\t\t} catch (error) {\n\t\t\t\t\tactiveTransaction = null\n\t\t\t\t\tawait execute('ROLLBACK')\n\t\t\t\t\tthrow error\n\t\t\t\t}\n\t\t\t} finally {\n\t\t\t\tactiveTransaction = null\n\t\t\t\tmutex.release()\n\t\t\t}\n\t\t},\n\t})\n\treturn sql\n}\n\nconst MESSAGE_COUNT = 84\nconst MESSAGE_TOOL_REF_COUNT = 122\nconst TOOL_CALL_COUNT = 61\nconst EXECUTOR_TOOL_COUNT = 42\nconst THREAD_EVENT_COUNT = 233\n\nconst MESSAGE_CONTENT_BYTES = 10_620\nconst THREAD_EVENT_PAYLOAD_BYTES = 4_036\nconst TOOL_CALL_RESULT_BYTES = 10_975\nconst EXECUTOR_TOOL_SCHEMA_BYTES = 2_235\n\nexport const slowReconnectActor = actor({\n\tstate: { runCount: 0 },\n\tdb: db({\n\t\tonMigrate: async (database) => {\n\t\t\tawait createSlowReconnectSchema(database)\n\t\t},\n\t}),\n\tvars: { sql: null } as SlowReconnectVars,\n\tonWebSocket: (c, ws) => {\n\t\tconst sock = ws as unknown as WebSocket\n\t\tif (sock.readyState === WebSocket.OPEN) {\n\t\t\tsock.send('pong')\n\t\t}\n\n\t\tws.addEventListener('message', (event) => {\n\t\t\tconst promise = handleSlowReconnectWebSocketMessage(c, sock, event.data)\n\t\t\tvoid c.keepAwake(promise)\n\t\t})\n\t},\n\tactions: {\n\t\tprepare: async (c) => {\n\t\t\tawait createSlowReconnectSchema(c.db)\n\t\t\treturn await seedSlowReconnectData(c.db)\n\t\t},\n\t\treproReconnect: async (c, clientId?: string) => {\n\t\t\tc.vars.sql ??= createSlowReconnectDb(c.db)\n\t\t\tc.state.runCount++\n\t\t\treturn await runReconnectRepro(c.vars.sql, clientId ?? `action-${c.state.runCount}`, 0)\n\t\t},\n\t\tgetRunCount: (c) => c.state.runCount,\n\t\tsleep: (c) => {\n\t\t\tc.sleep()\n\t\t\treturn true\n\t\t},\n\t},\n})\n\nasync function handleSlowReconnectWebSocketMessage(\n\tc: { db: RawRivetDB; vars: SlowReconnectVars; state: { runCount: number } },\n\tsock: WebSocket,\n\tdata: unknown,\n): Promise {\n\tif (data === 'ping') {\n\t\tif (sock.readyState === WebSocket.OPEN) {\n\t\t\tsock.send('pong')\n\t\t}\n\t\treturn\n\t}\n\n\tlet trigger: SlowReconnectRequest['type'] | 'unknown' = 'unknown'\n\ttry {\n\t\tconst request = parseSlowReconnectRequest(data)\n\t\ttrigger = request.type\n\t\tc.vars.sql ??= createSlowReconnectDb(c.db)\n\t\tc.state.runCount++\n\n\t\tif (request.type === 'client_resume') {\n\t\t\tconst startedAt = performance.now()\n\t\t\tconst result = await runCatchupSnapshot(c.vars.sql, request.version)\n\t\t\tsendJSON(sock, {\n\t\t\t\ttype: 'slow_reconnect_result',\n\t\t\t\ttrigger: request.type,\n\t\t\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\t\t\tresults: [result],\n\t\t\t})\n\t\t\treturn\n\t\t}\n\n\t\tconst clientId =\n\t\t\trequest.type === 'executor_connect'\n\t\t\t\t? request.clientId\n\t\t\t\t: (request.clientId ?? `slow-reconnect-${c.state.runCount}`)\n\t\tconst staggerHandleMs = request.type === 'repro_reconnect' ? (request.staggerHandleMs ?? 0) : 0\n\t\tconst result = await runReconnectRepro(c.vars.sql, clientId, staggerHandleMs)\n\n\t\tif (request.type === 'executor_connect') {\n\t\t\tsendJSON(sock, {\n\t\t\t\ttype: 'executor_connected',\n\t\t\t\texecutorId: clientId,\n\t\t\t\tregisteredToolCount: EXECUTOR_TOOL_COUNT,\n\t\t\t\tguidanceInventory: [],\n\t\t\t\tresumeBootstrap: true,\n\t\t\t})\n\t\t}\n\n\t\tsendJSON(sock, {\n\t\t\ttype: 'slow_reconnect_result',\n\t\t\ttrigger: request.type,\n\t\t\t...result,\n\t\t})\n\t} catch (error) {\n\t\tsendJSON(sock, {\n\t\t\ttype: 'slow_reconnect_error',\n\t\t\ttrigger,\n\t\t\terror: error instanceof Error ? error.message : String(error),\n\t\t})\n\t}\n}\n\nfunction parseSlowReconnectRequest(data: unknown): SlowReconnectRequest {\n\tif (typeof data !== 'string') {\n\t\tthrow new Error('slowReconnectActor request must be a string')\n\t}\n\tconst parsed = JSON.parse(data) as unknown\n\tif (!parsed || typeof parsed !== 'object') {\n\t\tthrow new Error('slowReconnectActor request must be an object')\n\t}\n\tconst request = parsed as Record\n\tif (request.type === 'client_resume') {\n\t\treturn { type: 'client_resume', version: numberField(request, 'version') }\n\t}\n\tif (request.type === 'executor_connect') {\n\t\tconst executorType = request.executorType\n\t\treturn {\n\t\t\ttype: 'executor_connect',\n\t\t\tclientId: stringField(request, 'clientId'),\n\t\t\t...(executorType === 'local-client' ||\n\t\t\texecutorType === 'sandbox' ||\n\t\t\texecutorType === 'virtual'\n\t\t\t\t? { executorType }\n\t\t\t\t: {}),\n\t\t}\n\t}\n\tif (request.type === 'repro_reconnect') {\n\t\treturn {\n\t\t\ttype: 'repro_reconnect',\n\t\t\t...(typeof request.clientId === 'string' ? { clientId: request.clientId } : {}),\n\t\t\t...(typeof request.staggerHandleMs === 'number'\n\t\t\t\t? { staggerHandleMs: request.staggerHandleMs }\n\t\t\t\t: {}),\n\t\t}\n\t}\n\tthrow new Error(`Unknown slowReconnectActor request type: ${String(request.type)}`)\n}\n\nfunction stringField(record: Record, field: string): string {\n\tconst value = record[field]\n\tif (typeof value !== 'string' || value.length === 0) {\n\t\tthrow new Error(`slowReconnectActor request ${field} must be a non-empty string`)\n\t}\n\treturn value\n}\n\nfunction numberField(record: Record, field: string): number {\n\tconst value = record[field]\n\tif (typeof value !== 'number' || !Number.isFinite(value)) {\n\t\tthrow new Error(`slowReconnectActor request ${field} must be a finite number`)\n\t}\n\treturn value\n}\n\nfunction sendJSON(\n\tsock: WebSocket,\n\tmessage: SlowReconnectResultMessage | SlowReconnectErrorMessage | object,\n): void {\n\tif (sock.readyState === WebSocket.OPEN) {\n\t\tsock.send(JSON.stringify(message))\n\t}\n}\n\nfunction createSlowReconnectDb(db: RawRivetDB): Db {\n\treturn createDb(async >(\n\t\tquery: string,\n\t\t...values: SQLPrimitive[]\n\t): Promise => {\n\t\tconst converted = values.map((value) =>\n\t\t\ttypeof value === 'boolean' ? (value ? 1 : 0) : value,\n\t\t)\n\t\treturn (await db.execute(query, ...converted)) as T[]\n\t})\n}\n\nasync function runReconnectRepro(\n\tsql: Db,\n\tclientId: string,\n\tstaggerHandleMs: number,\n): Promise> {\n\tconst startedAt = performance.now()\n\tconst buildToolPlanContext = runBuildToolPlanContext(sql)\n\tconst catchupSnapshot = runCatchupSnapshot(sql, 0)\n\tconst recoverToolCalls = runRecoverToolCalls(sql)\n\tconst handleExecutorConnect = delay(staggerHandleMs).then(() =>\n\t\trunHandleExecutorConnect(sql, clientId),\n\t)\n\n\tconst results = await Promise.all([\n\t\thandleExecutorConnect,\n\t\tbuildToolPlanContext,\n\t\tcatchupSnapshot,\n\t\trecoverToolCalls,\n\t])\n\treturn {\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tresults,\n\t}\n}\n\nasync function runHandleExecutorConnect(\n\tsql: Db,\n\tclientId: string,\n): Promise {\n\tconst startedAt = performance.now()\n\tconst steps: SlowReconnectStep[] = []\n\tconst nextSeq = await sql.withTransaction(async (tx) => {\n\t\tconst latestExecutor = await timedQuery(\n\t\t\ttx,\n\t\t\tsteps,\n\t\t\t'load-latest-executor-id',\n\t\t\t`SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1`,\n\t\t)\n\t\tconst latestExecutorId = String(latestExecutor[0]?.executor_id ?? 'seed-executor')\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tsteps,\n\t\t\t'select-cached-executor-tools',\n\t\t\t`SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`,\n\t\t\tlatestExecutorId,\n\t\t)\n\t\tconst executorType = await timedQuery(\n\t\t\ttx,\n\t\t\tsteps,\n\t\t\t'select-executor-type',\n\t\t\t`SELECT value FROM thread_meta_kv WHERE key = 'executor_type'`,\n\t\t)\n\t\tif (!executorType[0]?.value) {\n\t\t\tawait timedQuery(\n\t\t\t\ttx,\n\t\t\t\tsteps,\n\t\t\t\t'set-executor-type',\n\t\t\t\t`INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES ('executor_type', ?, ?)`,\n\t\t\t\t'local-client',\n\t\t\t\tnew Date().toISOString(),\n\t\t\t)\n\t\t}\n\t\tconst sandboxIntent = await timedQuery(\n\t\t\ttx,\n\t\t\tsteps,\n\t\t\t'select-sandbox-intent',\n\t\t\t`SELECT value FROM thread_meta_kv WHERE key = 'sandbox_intent'`,\n\t\t)\n\t\tif (hasPendingLaunch(sandboxIntent[0]?.value)) {\n\t\t\tawait timedQuery(\n\t\t\t\ttx,\n\t\t\t\tsteps,\n\t\t\t\t'clear-pending-launch',\n\t\t\t\t`UPDATE thread_meta_kv SET value = ?, updated_at = ? WHERE key = 'sandbox_intent'`,\n\t\t\t\tJSON.stringify({ spec: null, pendingLaunch: null }),\n\t\t\t\tnew Date().toISOString(),\n\t\t\t)\n\t\t}\n\t\tconst seqRows = await timedQuery(\n\t\t\ttx,\n\t\t\tsteps,\n\t\t\t'select-next-thread-event-seq',\n\t\t\t`SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events`,\n\t\t)\n\t\tconst seq = Number(seqRows[0]?.seq ?? 1)\n\t\tawait timedQuery(\n\t\t\ttx,\n\t\t\tsteps,\n\t\t\t'insert-executor-connected-event',\n\t\t\t`INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`,\n\t\t\tseq,\n\t\t\t'executor_connected',\n\t\t\tJSON.stringify({ type: 'executor_connected', executorId: clientId }),\n\t\t\tnew Date().toISOString(),\n\t\t)\n\t\treturn seq\n\t})\n\tsteps.push({\n\t\tname: 'transaction-total',\n\t\tdurationMs: Math.round(performance.now() - startedAt),\n\t\trowCount: nextSeq,\n\t})\n\treturn {\n\t\tname: 'handle-executor-connect',\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tsteps,\n\t}\n}\n\nasync function runBuildToolPlanContext(sql: Db): Promise {\n\tconst startedAt = performance.now()\n\tconst steps: SlowReconnectStep[] = []\n\tconst latestExecutor = await timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'load-latest-executor-id',\n\t\t`SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1`,\n\t)\n\tconst latestExecutorId = String(latestExecutor[0]?.executor_id ?? 'seed-executor')\n\tawait timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'select-executor-tools',\n\t\t`SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`,\n\t\tlatestExecutorId,\n\t)\n\tawait timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'count-uncancelled-top-level',\n\t\t`SELECT COUNT(*) as count FROM messages WHERE cancelled = 0 AND parent_tool_use_id IS NULL`,\n\t)\n\tconst unresolvedRows = await timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'find-unresolved-assistant-message',\n\t\t`SELECT m.*\n\t\t\tFROM message_tool_refs AS tool_use\n\t\t\tJOIN messages AS m\n\t\t\t\tON m.message_id = tool_use.assistant_message_id\n\t\t\tWHERE tool_use.block_type = 'tool_use'\n\t\t\t\tAND tool_use.cancelled = 0\n\t\t\t\tAND m.cancelled = 0\n\t\t\t\tAND m.role = 'assistant'\n\t\t\t\tAND m.parent_tool_use_id IS NULL\n\t\t\t\tAND NOT EXISTS (\n\t\t\t\t\tSELECT 1\n\t\t\t\t\tFROM message_tool_refs AS tool_result\n\t\t\t\t\tJOIN messages AS tool_result_message\n\t\t\t\t\t\tON tool_result_message.message_id = tool_result.source_message_id\n\t\t\t\t\tWHERE tool_result.assistant_message_id = tool_use.assistant_message_id\n\t\t\t\t\t\tAND tool_result.block_type = 'tool_result'\n\t\t\t\t\t\tAND tool_result.cancelled = 0\n\t\t\t\t\t\tAND tool_result.tool_use_id = tool_use.tool_use_id\n\t\t\t\t\t\tAND tool_result_message.parent_tool_use_id IS NULL\n\t\t\t\t)\n\t\t\tGROUP BY m.message_id\n\t\t\tORDER BY m.created_at DESC\n\t\t\tLIMIT 1`,\n\t)\n\tconst unresolvedMessageId = unresolvedRows[0]?.message_id\n\tif (typeof unresolvedMessageId === 'string') {\n\t\tawait timedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'get-persisted-tool-result-ids',\n\t\t\t`SELECT tool_result.tool_use_id\n\t\t\t\tFROM message_tool_refs AS tool_result\n\t\t\t\tJOIN messages AS tool_result_message\n\t\t\t\t\tON tool_result_message.message_id = tool_result.source_message_id\n\t\t\t\tWHERE tool_result.assistant_message_id = ?\n\t\t\t\t\tAND tool_result.block_type = 'tool_result'\n\t\t\t\t\tAND tool_result.cancelled = 0\n\t\t\t\t\tAND tool_result_message.parent_tool_use_id IS NULL`,\n\t\t\tunresolvedMessageId,\n\t\t)\n\t\tawait timedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'get-tool-calls-by-message-id',\n\t\t\t`SELECT * FROM tool_calls WHERE message_id = ?`,\n\t\t\tunresolvedMessageId,\n\t\t)\n\t}\n\tawait timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'is-last-message-cancelled-assistant',\n\t\t`SELECT role, cancelled FROM messages\n\t\t\tWHERE parent_tool_use_id IS NULL\n\t\t\tORDER BY created_at DESC\n\t\t\tLIMIT 1`,\n\t)\n\tawait timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'get-last-uncancelled',\n\t\t`SELECT m.* FROM messages m\n\t\t\tWHERE m.cancelled = 0 AND m.parent_tool_use_id IS NULL\n\t\t\tORDER BY m.created_at DESC\n\t\t\tLIMIT 1`,\n\t)\n\treturn {\n\t\tname: 'build-tool-plan-context',\n\t\ttotalMs: Math.round(performance.now() - startedAt),\n\t\tsteps,\n\t}\n}\n\nasync function runCatchupSnapshot(sql: Db, version: number): Promise {\n\tconst startedAt = performance.now()\n\tconst steps: SlowReconnectStep[] = []\n\tawait Promise.all([\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'thread-events-list-since-version',\n\t\t\t`SELECT seq, event_type, payload, created_at FROM thread_events WHERE seq > ? ORDER BY seq ASC`,\n\t\t\tversion,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'environment-snapshot',\n\t\t\t`SELECT snapshot FROM environment_snapshot WHERE id = 1`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'thread-settings-snapshot',\n\t\t\t`SELECT settings FROM thread_settings_snapshot WHERE id = 1`,\n\t\t),\n\t\ttimedQuery(sql, steps, 'retry-state', `SELECT * FROM retry_state WHERE id = 1`),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'queued-messages',\n\t\t\t`SELECT * FROM queued_messages ORDER BY created_at ASC`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'executor-artifacts',\n\t\t\t`SELECT artifact_key, data_type, length(content_base64) AS bytes, tool_call_id, updated_at FROM executor_artifacts ORDER BY updated_at ASC`,\n\t\t),\n\t\ttimedQuery(sql, steps, 'tool-approvals', `SELECT * FROM tool_approvals ORDER BY timestamp ASC`),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'compaction-summaries',\n\t\t\t`SELECT cut_message_id, created_at FROM compaction_summaries ORDER BY created_at ASC`,\n\t\t),\n\t\ttimedQuery(\n\t\t\tsql,\n\t\t\tsteps,\n\t\t\t'executor-status',\n\t\t\t`SELECT value FROM thread_meta_kv WHERE key = 'executor_status'`,\n\t\t),\n\t])\n\tsteps.sort((a, b) => b.durationMs - a.durationMs)\n\treturn { name: 'catchup-snapshot', totalMs: Math.round(performance.now() - startedAt), steps }\n}\n\nasync function runRecoverToolCalls(sql: Db): Promise {\n\tconst startedAt = performance.now()\n\tconst steps: SlowReconnectStep[] = []\n\tawait timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'hydrate-tool-progress',\n\t\t`SELECT id, progress\n\t\t\tFROM tool_calls\n\t\t\tWHERE progress IS NOT NULL\n\t\t\t\tAND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`,\n\t)\n\tawait timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'get-pending-tool-calls',\n\t\t`SELECT * FROM tool_calls\n\t\t\tWHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')\n\t\t\tORDER BY issued_at ASC`,\n\t)\n\tawait timedQuery(\n\t\tsql,\n\t\tsteps,\n\t\t'get-next-tool-expiry',\n\t\t`SELECT MIN(expires_at) AS expires_at\n\t\t\tFROM tool_calls\n\t\t\tWHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`,\n\t)\n\treturn { name: 'recover-tool-calls', totalMs: Math.round(performance.now() - startedAt), steps }\n}\n\nasync function timedQuery>(\n\tsql: Db,\n\tsteps: SlowReconnectStep[],\n\tname: string,\n\tquery: string,\n\t...values: SQLPrimitive[]\n): Promise {\n\tconst startedAt = performance.now()\n\tconst rows = await sql(query, ...values)\n\tsteps.push({\n\t\tname,\n\t\tdurationMs: Math.round(performance.now() - startedAt),\n\t\trowCount: rows.length,\n\t})\n\treturn rows\n}\n\nfunction hasPendingLaunch(value: unknown): boolean {\n\tif (typeof value !== 'string' || value.length === 0) {\n\t\treturn false\n\t}\n\ttry {\n\t\tconst parsed = JSON.parse(value) as { pendingLaunch?: unknown }\n\t\treturn parsed.pendingLaunch !== null && parsed.pendingLaunch !== undefined\n\t} catch {\n\t\treturn false\n\t}\n}\n\nfunction delay(ms: number): Promise {\n\treturn new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)))\n}\n\nasync function createSlowReconnectSchema(database: RawRivetDB): Promise {\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS executor_tools (\n\t\texecutor_id TEXT NOT NULL,\n\t\ttool_name TEXT NOT NULL,\n\t\tschema TEXT NOT NULL,\n\t\tupdated_at TEXT NOT NULL,\n\t\tPRIMARY KEY (executor_id, tool_name)\n\t)`)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_executor_tools_executor ON executor_tools(executor_id)`,\n\t)\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS thread_meta_kv (\n\t\tkey TEXT PRIMARY KEY,\n\t\tvalue TEXT,\n\t\tupdated_at TEXT NOT NULL\n\t)`)\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS thread_events (\n\t\tseq INTEGER PRIMARY KEY,\n\t\tevent_type TEXT NOT NULL,\n\t\tpayload TEXT NOT NULL,\n\t\tcreated_at TEXT NOT NULL DEFAULT (datetime('now'))\n\t)`)\n\tawait database.execute(`CREATE INDEX IF NOT EXISTS idx_thread_events_seq ON thread_events(seq)`)\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS message_added_events (\n\t\tmessage_id TEXT PRIMARY KEY,\n\t\tseq INTEGER NOT NULL UNIQUE\n\t)`)\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS messages (\n\t\tmessage_id TEXT PRIMARY KEY,\n\t\trole TEXT NOT NULL CHECK(role IN ('user', 'assistant', 'info')),\n\t\tcontent TEXT NOT NULL,\n\t\tmeta TEXT,\n\t\tuser_state TEXT,\n\t\tcreated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),\n\t\tcancelled INTEGER NOT NULL DEFAULT 0,\n\t\tread_at TEXT,\n\t\tparent_tool_use_id TEXT,\n\t\ttool_result_for_message_id TEXT\n\t)`)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_messages_parent_role_cancelled_created_at ON messages(parent_tool_use_id, role, cancelled, created_at DESC)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_messages_parent_cancelled_created_at ON messages(parent_tool_use_id, cancelled, created_at DESC)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_messages_parent_created_at ON messages(parent_tool_use_id, created_at DESC)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_messages_role_created_at ON messages(role, created_at)`,\n\t)\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS message_tool_refs (\n\t\tsource_message_id TEXT NOT NULL,\n\t\tassistant_message_id TEXT NOT NULL,\n\t\ttool_use_id TEXT NOT NULL,\n\t\tblock_type TEXT NOT NULL CHECK(block_type IN ('tool_use', 'tool_result')),\n\t\tcancelled INTEGER NOT NULL DEFAULT 0,\n\t\tPRIMARY KEY (source_message_id, block_type, tool_use_id)\n\t)`)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_message_tool_refs_assistant_lookup ON message_tool_refs(assistant_message_id, block_type, cancelled, tool_use_id)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE UNIQUE INDEX IF NOT EXISTS idx_message_tool_refs_live_tool_result ON message_tool_refs(assistant_message_id, tool_use_id) WHERE block_type = 'tool_result' AND cancelled = 0`,\n\t)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_message_tool_refs_source_message ON message_tool_refs(source_message_id)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_message_tool_refs_tool_use_lookup ON message_tool_refs(tool_use_id, assistant_message_id) WHERE block_type = 'tool_use' AND cancelled = 0`,\n\t)\n\tawait database.execute(`CREATE TABLE IF NOT EXISTS tool_calls (\n\t\tid TEXT PRIMARY KEY,\n\t\tprovider_tool_use_id TEXT NOT NULL,\n\t\ttool_name TEXT NOT NULL,\n\t\targs TEXT NOT NULL,\n\t\texecutor_id TEXT,\n\t\tmessage_id TEXT NOT NULL,\n\t\tissued_at TEXT NOT NULL,\n\t\texpires_at TEXT,\n\t\tstate TEXT NOT NULL CHECK(state IN ('queued', 'pending_reconnect', 'pending_ack', 'running', 'completed', 'expired', 'revoked')),\n\t\tresult TEXT,\n\t\tprogress TEXT,\n\t\tcompleted_at TEXT\n\t)`)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_tool_calls_message_id ON tool_calls(message_id)`,\n\t)\n\tawait database.execute(`CREATE INDEX IF NOT EXISTS idx_tool_calls_state ON tool_calls(state)`)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_tool_calls_expires_at ON tool_calls(expires_at) WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`,\n\t)\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS environment_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), snapshot TEXT NOT NULL, updated_at TEXT NOT NULL)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS thread_settings_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), settings TEXT NOT NULL, updated_at TEXT NOT NULL)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS retry_state (id INTEGER PRIMARY KEY CHECK (id = 1), attempt INTEGER NOT NULL DEFAULT 0, scheduled_at INTEGER NOT NULL, reason TEXT NOT NULL)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS queued_messages (message_id TEXT PRIMARY KEY, content TEXT NOT NULL, user_state TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), steer INTEGER NOT NULL DEFAULT 0, user_meta TEXT)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS executor_artifacts (artifact_key TEXT PRIMARY KEY, data_type TEXT NOT NULL, content_base64 TEXT NOT NULL, tool_call_id TEXT, updated_at TEXT NOT NULL)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS tool_approvals (id TEXT PRIMARY KEY, tool_call_id TEXT NOT NULL UNIQUE, tool_name TEXT NOT NULL, args TEXT NOT NULL, reason TEXT, to_allow TEXT, context TEXT NOT NULL CHECK(context IN ('thread', 'subagent')), subagent_tool_name TEXT, parent_tool_call_id TEXT, timestamp INTEGER NOT NULL, matched_rule TEXT, rule_source TEXT CHECK(rule_source IN ('user', 'built-in')))`,\n\t)\n\tawait database.execute(\n\t\t`CREATE INDEX IF NOT EXISTS idx_tool_approvals_timestamp ON tool_approvals(timestamp)`,\n\t)\n\tawait database.execute(\n\t\t`CREATE TABLE IF NOT EXISTS compaction_summaries (summary_id TEXT PRIMARY KEY, summary_text TEXT NOT NULL, cut_message_id TEXT NOT NULL, created_at TEXT NOT NULL)`,\n\t)\n}\n\nasync function seedSlowReconnectData(database: RawRivetDB): Promise<{\n\tseeded: boolean\n\tmessages: number\n\ttoolCalls: number\n\tthreadEvents: number\n}> {\n\tconst existing = await database.execute(`SELECT COUNT(*) AS count FROM messages`)\n\tif (Number(existing[0]?.count ?? 0) > 0) {\n\t\treturn {\n\t\t\tseeded: false,\n\t\t\tmessages: MESSAGE_COUNT,\n\t\t\ttoolCalls: TOOL_CALL_COUNT,\n\t\t\tthreadEvents: THREAD_EVENT_COUNT,\n\t\t}\n\t}\n\n\tconst now = new Date('2026-05-16T03:58:18.661Z').getTime()\n\tconst text = (size: number) => 'x'.repeat(size)\n\tconst isoAt = (index: number) => new Date(now + index * 1_000).toISOString()\n\n\tawait batchInsert(database, `INSERT INTO thread_meta_kv (key, value, updated_at)`, [\n\t\t['executor_type', 'local-client', isoAt(0)],\n\t\t['sandbox_intent', JSON.stringify({ spec: null, pendingLaunch: null }), isoAt(0)],\n\t\t['executor_status', JSON.stringify({ available: true, message: 'ready' }), isoAt(0)],\n\t])\n\n\tconst messageRows: unknown[][] = []\n\tfor (let index = 1; index <= MESSAGE_COUNT; index++) {\n\t\tconst role = index % 2 === 0 ? 'assistant' : 'user'\n\t\tmessageRows.push([\n\t\t\tmessageId(index),\n\t\t\trole,\n\t\t\ttext(MESSAGE_CONTENT_BYTES),\n\t\t\tnull,\n\t\t\tnull,\n\t\t\tisoAt(index),\n\t\t\t0,\n\t\t\tnull,\n\t\t\tnull,\n\t\t\tnull,\n\t\t])\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO messages (message_id, role, content, meta, user_state, created_at, cancelled, read_at, parent_tool_use_id, tool_result_for_message_id)`,\n\t\tmessageRows,\n\t\t20,\n\t)\n\n\tconst messageToolRefRows: unknown[][] = []\n\tfor (let index = 0; index < MESSAGE_TOOL_REF_COUNT / 2; index++) {\n\t\tconst assistantIndex = 2 + (index % 42) * 2\n\t\tconst sourceIndex = Math.max(1, assistantIndex - 1)\n\t\tconst resultIndex = Math.min(MESSAGE_COUNT, assistantIndex + 1)\n\t\tconst toolUseId = toolUseID(index + 1)\n\t\tmessageToolRefRows.push([\n\t\t\tmessageId(sourceIndex),\n\t\t\tmessageId(assistantIndex),\n\t\t\ttoolUseId,\n\t\t\t'tool_use',\n\t\t\t0,\n\t\t])\n\t\tmessageToolRefRows.push([\n\t\t\tmessageId(resultIndex),\n\t\t\tmessageId(assistantIndex),\n\t\t\ttoolUseId,\n\t\t\t'tool_result',\n\t\t\t0,\n\t\t])\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled)`,\n\t\tmessageToolRefRows,\n\t\t50,\n\t)\n\n\tconst toolCallRows: unknown[][] = []\n\tfor (let index = 1; index <= TOOL_CALL_COUNT; index++) {\n\t\tconst assistantIndex = 2 + ((index - 1) % 42) * 2\n\t\ttoolCallRows.push([\n\t\t\ttoolUseID(index),\n\t\t\t`provider-${index}`,\n\t\t\t`tool_${index % 21}`,\n\t\t\tJSON.stringify({ path: `/tmp/file-${index}` }),\n\t\t\t'seed-executor',\n\t\t\tmessageId(assistantIndex),\n\t\t\tisoAt(index),\n\t\t\tnull,\n\t\t\t'completed',\n\t\t\tJSON.stringify({\n\t\t\t\tok: true,\n\t\t\t\trun: { status: 'done', result: text(TOOL_CALL_RESULT_BYTES) },\n\t\t\t}),\n\t\t\tnull,\n\t\t\tisoAt(index + 100),\n\t\t])\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at)`,\n\t\ttoolCallRows,\n\t\t20,\n\t)\n\n\tconst executorToolRows: unknown[][] = []\n\tfor (let index = 1; index <= EXECUTOR_TOOL_COUNT; index++) {\n\t\tconst schema = JSON.stringify({\n\t\t\tname: `tool_${index}`,\n\t\t\tdescription: text(EXECUTOR_TOOL_SCHEMA_BYTES),\n\t\t\tinput_schema: { type: 'object', properties: {} },\n\t\t})\n\t\texecutorToolRows.push(['seed-executor', `tool_${index}`, schema, isoAt(index)])\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO executor_tools (executor_id, tool_name, schema, updated_at)`,\n\t\texecutorToolRows,\n\t\t42,\n\t)\n\n\tconst threadEventRows: unknown[][] = []\n\tfor (let index = 1; index <= THREAD_EVENT_COUNT; index++) {\n\t\tthreadEventRows.push([\n\t\t\tindex,\n\t\t\tindex % 3 === 0 ? 'message_added' : 'agent_state_changed',\n\t\t\tJSON.stringify({ type: 'seed_event', body: text(THREAD_EVENT_PAYLOAD_BYTES) }),\n\t\t\tisoAt(index),\n\t\t])\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO thread_events (seq, event_type, payload, created_at)`,\n\t\tthreadEventRows,\n\t\t25,\n\t)\n\n\tconst messageAddedRows: unknown[][] = []\n\tfor (let index = 1; index <= MESSAGE_COUNT; index++) {\n\t\tmessageAddedRows.push([messageId(index), index])\n\t}\n\tawait batchInsert(\n\t\tdatabase,\n\t\t`INSERT INTO message_added_events (message_id, seq)`,\n\t\tmessageAddedRows,\n\t\t50,\n\t)\n\n\tawait database.execute(\n\t\t`INSERT INTO environment_snapshot (id, snapshot, updated_at) VALUES (1, ?, ?)`,\n\t\tJSON.stringify({ cwd: '/workspace', body: text(3_620) }),\n\t\tisoAt(0),\n\t)\n\tawait database.execute(\n\t\t`INSERT INTO thread_settings_snapshot (id, settings, updated_at) VALUES (1, ?, ?)`,\n\t\tJSON.stringify({ maxTokens: 20_000, body: text(55) }),\n\t\tisoAt(0),\n\t)\n\treturn {\n\t\tseeded: true,\n\t\tmessages: MESSAGE_COUNT,\n\t\ttoolCalls: TOOL_CALL_COUNT,\n\t\tthreadEvents: THREAD_EVENT_COUNT,\n\t}\n}\n\nasync function batchInsert(\n\tdatabase: RawRivetDB,\n\tinsertPrefix: string,\n\trows: unknown[][],\n\tbatchSize = 100,\n): Promise {\n\tif (rows.length === 0) {\n\t\treturn\n\t}\n\tconst columnCount = rows[0]?.length ?? 0\n\tif (columnCount === 0) {\n\t\treturn\n\t}\n\tconst rowPlaceholder = `(${'?,'.repeat(columnCount).slice(0, -1)})`\n\tfor (let index = 0; index < rows.length; index += batchSize) {\n\t\tconst chunk = rows.slice(index, index + batchSize)\n\t\tconst values = chunk.map(() => rowPlaceholder).join(',')\n\t\tconst bindings = chunk.flat()\n\t\tawait database.execute(`${insertPrefix} VALUES ${values}`, ...bindings)\n\t}\n}\n\nfunction messageId(index: number): string {\n\treturn `M-${String(index).padStart(22, '0')}`\n}\n\nfunction toolUseID(index: number): string {\n\treturn `toolu_${String(index).padStart(22, '0')}`\n}\n\nexport const registry = setup({\n\tuse: { slowReconnectActor },\n\tmaxIncomingMessageSize: 5 * 1024 * 1024,\n\tmaxOutgoingMessageSize: 5 * 1024 * 1024,\n})\n\nif (import.meta.main) {\n\tregistry.start()\n}\n","import { openai } from \"@ai-sdk/openai\";\nimport { generateText, tool } from \"ai\";\nimport { actor, event } from \"rivetkit\";\nimport { z } from \"zod\";\nimport { getWeather } from \"./my-tools.ts\";\nimport type { Message } from \"./types.ts\";\n\nexport const aiAgent = actor({\n\t// Persistent state that survives restarts: https://rivet.dev/docs/actors/state\n\tstate: {\n\t\tmessages: [] as Message[],\n\t},\n\tevents: {\n\t\tmessageReceived: event(),\n\t},\n\n\tactions: {\n\t\t// Callable functions from clients: https://rivet.dev/docs/actors/actions\n\t\tgetMessages: (c) => c.state.messages,\n\n\t\tsendMessage: async (c, userMessage: string) => {\n\t\t\tconst userMsg: Message = {\n\t\t\t\trole: \"user\",\n\t\t\t\tcontent: userMessage,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t};\n\t\t\t// State changes are automatically persisted\n\t\t\tc.state.messages.push(userMsg);\n\n\t\t\tconst { text } = await generateText({\n\t\t\t\tmodel: openai(\"gpt-4o-mini\"),\n\t\t\t\tprompt: userMessage,\n\t\t\t\tmessages: c.state.messages,\n\t\t\t\ttools: {\n\t\t\t\t\tweather: tool({\n\t\t\t\t\t\tdescription: \"Get the weather in a location\",\n\t\t\t\t\t\tparameters: z.object({\n\t\t\t\t\t\t\tlocation: z\n\t\t\t\t\t\t\t\t.string()\n\t\t\t\t\t\t\t\t.describe(\n\t\t\t\t\t\t\t\t\t\"The location to get the weather for\",\n\t\t\t\t\t\t\t\t),\n\t\t\t\t\t\t}),\n\t\t\t\t\t\texecute: async ({ location }) => {\n\t\t\t\t\t\t\treturn await getWeather(location);\n\t\t\t\t\t\t},\n\t\t\t\t\t}),\n\t\t\t\t},\n\t\t\t});\n\n\t\t\tconst assistantMsg: Message = {\n\t\t\t\trole: \"assistant\",\n\t\t\t\tcontent: text,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t};\n\t\t\tc.state.messages.push(assistantMsg);\n\n\t\t\t// Send events to all connected clients: https://rivet.dev/docs/actors/events\n\t\t\tc.broadcast(\"messageReceived\", assistantMsg);\n\n\t\t\treturn assistantMsg;\n\t\t},\n\t},\n});\n","export async function getWeather(location: string) {\n\t// Mock weather API response\n\treturn {\n\t\tlocation,\n\t\ttemperature: Math.floor(Math.random() * 30) + 10,\n\t\tcondition: [\"sunny\", \"cloudy\", \"rainy\", \"snowy\"][\n\t\t\tMath.floor(Math.random() * 4)\n\t\t],\n\t\thumidity: Math.floor(Math.random() * 50) + 30,\n\t};\n}\n","import { registry } from \"./index.ts\";\nimport { resolveMode } from \"./mode.ts\";\nimport { serve } from \"@hono/node-server\";\nimport { Hono } from \"hono\";\nimport type { Server as HttpServer } from \"node:http\";\nimport * as v8 from \"node:v8\";\n\nconst app = new Hono();\nconst port = Number.parseInt(process.env.PORT ?? \"3000\", 10);\nconst mode = resolveMode();\n\nprocess.on(\"exit\", (code) => {\n\tconsole.log(JSON.stringify({ kind: \"process_exit\", code, pid: process.pid }));\n});\nif (process.env.SQLITE_MEMORY_SOAK_DIAGNOSTICS === \"1\") {\n\tfor (const signal of [\"SIGINT\", \"SIGTERM\"] as const) {\n\t\tprocess.on(signal, () => {\n\t\t\tconsole.log(\n\t\t\t\tJSON.stringify({\n\t\t\t\t\tkind: \"process_signal\",\n\t\t\t\t\tsignal,\n\t\t\t\t\tpid: process.pid,\n\t\t\t\t\tppid: process.ppid,\n\t\t\t\t\ttimestamp: new Date().toISOString(),\n\t\t\t\t}),\n\t\t\t);\n\t\t\tprocess.exit(signal === \"SIGINT\" ? 130 : 143);\n\t\t});\n\t}\n}\nprocess.on(\"beforeExit\", (code) => {\n\tconsole.log(JSON.stringify({ kind: \"process_before_exit\", code, pid: process.pid }));\n});\nprocess.on(\"uncaughtException\", (error) => {\n\tconsole.error(\n\t\tJSON.stringify({\n\t\t\tkind: \"uncaught_exception\",\n\t\t\terror: error.stack ?? error.message,\n\t\t}),\n\t);\n});\nprocess.on(\"unhandledRejection\", (reason) => {\n\tconsole.error(\n\t\tJSON.stringify({\n\t\t\tkind: \"unhandled_rejection\",\n\t\t\terror: reason instanceof Error ? reason.stack ?? reason.message : String(reason),\n\t\t}),\n\t);\n});\n\nasync function memoryBreakdown(forceGc: boolean) {\n\tconst gc = (globalThis as typeof globalThis & { gc?: () => void }).gc;\n\tif (forceGc && typeof gc === \"function\") gc();\n\n\tconst memory = process.memoryUsage();\n\tconst heap = v8.getHeapStatistics();\n\tconst spaces = v8.getHeapSpaceStatistics();\n\tconst nativeNonV8Estimate = Math.max(0, memory.rss - heap.total_heap_size);\n\n\treturn {\n\t\tpid: process.pid,\n\t\ttimestamp: new Date().toISOString(),\n\t\tuptimeSeconds: process.uptime(),\n\t\tgcRequested: forceGc,\n\t\tgcAvailable: typeof gc === \"function\",\n\t\tprocess: {\n\t\t\trssBytes: memory.rss,\n\t\t\theapTotalBytes: memory.heapTotal,\n\t\t\theapUsedBytes: memory.heapUsed,\n\t\t\texternalBytes: memory.external,\n\t\t\tarrayBuffersBytes: memory.arrayBuffers,\n\t\t},\n\t\tv8: {\n\t\t\ttotalHeapSizeBytes: heap.total_heap_size,\n\t\t\tusedHeapSizeBytes: heap.used_heap_size,\n\t\t\theapSizeLimitBytes: heap.heap_size_limit,\n\t\t\tmallocedMemoryBytes: heap.malloced_memory,\n\t\t\texternalMemoryBytes: heap.external_memory,\n\t\t\tpeakMallocedMemoryBytes: heap.peak_malloced_memory,\n\t\t\tspaces: spaces.map((space) => ({\n\t\t\t\tname: space.space_name,\n\t\t\t\tsizeBytes: space.space_size,\n\t\t\t\tusedBytes: space.space_used_size,\n\t\t\t\tavailableBytes: space.space_available_size,\n\t\t\t\tphysicalSizeBytes: space.physical_space_size,\n\t\t\t})),\n\t\t},\n\t\testimates: {\n\t\t\tjsHeapResidentBytes: memory.heapTotal,\n\t\t\tjsHeapUsedBytes: memory.heapUsed,\n\t\t\tv8ExternalBytes: memory.external,\n\t\t\tnativeNonV8ResidentEstimateBytes: nativeNonV8Estimate,\n\t\t},\n\t\tresourceUsage: process.resourceUsage(),\n\t};\n}\n\nfunction requestHeaders(headers: Headers) {\n\tconst entries: Array<[string, string]> = [];\n\theaders.forEach((value, key) => {\n\t\tentries.push([\n\t\t\tkey,\n\t\t\tkey === \"authorization\" || key === \"x-rivet-token\"\n\t\t\t\t? \"\"\n\t\t\t\t: value,\n\t\t]);\n\t});\n\treturn Object.fromEntries(entries);\n}\n\napp.get(\"/debug/memory\", async (c) => {\n\tconst forceGc = c.req.query(\"gc\") === \"1\";\n\treturn c.json(await memoryBreakdown(forceGc));\n});\n\napp.get(\"/health\", () => registry.routes.health());\n\napp.get(\"/metadata\", () => registry.routes.metadata());\n\napp.get(\"/metrics\", (c) => registry.routes.prometheusMetrics(c.req.raw));\n\napp.post(\"/debug/heap-snapshot\", (c) => {\n\tif (process.env.SQLITE_MEMORY_SOAK_DIAGNOSTICS !== \"1\") {\n\t\treturn c.json({ error: \"disabled\" }, 404);\n\t}\n\n\tconst path = c.req.query(\"path\");\n\tif (!path) {\n\t\treturn c.json({ error: \"missing path\" }, 400);\n\t}\n\n\tconst writtenPath = v8.writeHeapSnapshot(path);\n\treturn c.json({ path: writtenPath });\n});\n\napp.use(\"*\", async (c, next) => {\n\tconst startedAt = Date.now();\n\tawait next();\n\t// console.log(\n\t// \tJSON.stringify({\n\t// \t\tkind: \"request\",\n\t// \t\tmethod: c.req.method,\n\t// \t\tpath: new URL(c.req.url).pathname,\n\t// \t\theaders: requestHeaders(c.req.raw.headers),\n\t// \t\tstatus: c.res.status,\n\t// \t\tdurationMs: Date.now() - startedAt,\n\t// \t}),\n\t// );\n});\n\n// Only wire the serverless handler in serverless modes. In `serverful` mode\n// the runner connects via `registry.start()` and any stray hit on\n// `/api/rivet/start` would spin up a second envoy alongside the persistent\n// one.\nif (mode === \"serverful\") {\n\tregistry.start();\n} else {\n\tapp.all(\"/api/rivet/*\", (c) => registry.handler(c.req.raw));\n\tapp.all(\"/api/rivet\", (c) => registry.handler(c.req.raw));\n}\n\nconst server = serve({ fetch: app.fetch, port }, () => {\n\tif (mode === \"serverful\") {\n\t\tconsole.log(\n\t\t\t`kitchen sink (serverful) listening on http://127.0.0.1:${port}`,\n\t\t);\n\t} else {\n\t\tconsole.log(\n\t\t\t`kitchen sink (${mode}) listening on http://127.0.0.1:${port}/api/rivet`,\n\t\t);\n\t}\n});\nconst httpServer = server as unknown as HttpServer;\nhttpServer.requestTimeout = 0;\nhttpServer.headersTimeout = 0;\nhttpServer.keepAliveTimeout = 0;\nhttpServer.timeout = 0;\n"],"mappings":";;;;;;;AAAA,SAAS,SAAAA,cAAa;;;ACEf,SAAS,cAA+B;AAC9C,QAAM,WAAW,QAAQ,IAAI;AAC7B,MACC,aAAa,gBACb,aAAa,eACb,aAAa,oBACZ;AACD,WAAO;AAAA,EACR;AACA,MAAI,aAAa,UAAa,aAAa,IAAI;AAC9C,UAAM,IAAI;AAAA,MACT,iGAAiG,QAAQ;AAAA,IAC1G;AAAA,EACD;AAEA,MAAI,QAAQ,IAAI,qBAAqB,IAAK,QAAO;AACjD,MAAI,QAAQ,IAAI,yBAAyB,OAAW,QAAO;AAC3D,MAAI,QAAQ,IAAI,gCAAgC,QAAW;AAC1D,WAAO;AAAA,EACR;AAEA,SAAO;AACR;;;ACxBA,SAAS,OAAO,aAA8D;AAEvE,IAAM,UAAU,MAAM;AAAA,EAC5B,SAAS;AAAA,IACR,uBAAuB;AAAA,IACvB,kBAAkB;AAAA,EACnB;AAAA,EACA,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,QAAQ;AAAA,IACP,UAAU,MAAc;AAAA,EACzB;AAAA,EACA,YAAY,IAAI,WAA+B;AAI9C,cAAU,iBAAiB,WAAW,CAACC,YAA6B;AACnE,UAAI,UAAU,eAAe,EAAG;AAChC,gBAAU,KAAKA,QAAM,IAA4B;AAAA,IAClD,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,WAAW,CAAC,GAAG,MAAc;AAC5B,QAAE,MAAM,SAAS;AACjB,QAAE,UAAU,YAAY,EAAE,MAAM,KAAK;AACrC,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,UAAU,CAAC,GAAG,MAAc;AAC3B,QAAE,MAAM,QAAQ;AAChB,QAAE,UAAU,YAAY,CAAC;AACzB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,UAAU,CAAC,MAAM;AAChB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,MAAM,CAAC,OAAO;AACb,aAAO,EAAE,IAAI,KAAK;AAAA,IACnB;AAAA,IACA,WAAW,CAAC,MAAM;AACjB,QAAE,MAAM;AACR,aAAO,EAAE,IAAI,KAAK;AAAA,IACnB;AAAA,EACD;AACD,CAAC;;;AC1CD,SAAS,SAAAC,QAAO,SAAAC,cAAa;AAEtB,IAAM,cAAcD,OAAM;AAAA,EAChC,OAAO;AAAA,IACN,iBAAiB;AAAA,EAClB;AAAA,EACA,QAAQ;AAAA,IACP,UAAUC,OAAc;AAAA,EACzB;AAAA,EACA,WAAW,EAAE,OAAO,EAAE;AAAA,EACtB,WAAW,CAAC,GAAG,SAAS;AACvB,MAAE,MAAM,mBAAmB;AAAA,EAC5B;AAAA,EACA,cAAc,CAAC,GAAG,SAAS;AAI1B,MAAE,MAAM,mBAAmB;AAAA,EAC5B;AAAA,EACA,SAAS;AAAA,IACR,WAAW,CAAC,GAAG,MAAc;AAC5B,QAAE,KAAK,MAAM,SAAS;AACtB,QAAE,UAAU,YAAY,EAAE,KAAK,MAAM,KAAK;AAAA,IAC3C;AAAA,IACA,UAAU,CAAC,GAAG,MAAc;AAC3B,QAAE,KAAK,MAAM,QAAQ;AACrB,QAAE,UAAU,YAAY,CAAC;AAAA,IAC1B;AAAA,IACA,UAAU,CAAC,MAAM;AAChB,aAAO,EAAE,KAAK,MAAM;AAAA,IACrB;AAAA,IACA,oBAAoB,CAAC,MAAM;AAC1B,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,EACD;AACD,CAAC;;;ACnCD,SAAS,SAAAC,QAAO,SAAAC,cAAa;AAEtB,IAAM,oBAAoBD,OAAM;AAAA,EACtC,OAAO,EAAE,OAAO,GAAG,cAAc,CAAC,EAAc;AAAA,EAChD,QAAQ;AAAA,IACP,UAAUC,OAAqC;AAAA,EAChD;AAAA,EACA,iBAAiB,CAAC,GAAG,WAA8B;AAClD,WAAO;AAAA,MACN,MAAM,OAAO,QAAQ;AAAA,IACtB;AAAA,EACD;AAAA,EACA,WAAW,CAAC,GAAG,SAAS;AAEvB,MAAE,MAAM,aAAa,KAAK,KAAK,MAAM,IAAI;AAAA,EAC1C;AAAA,EACA,SAAS;AAAA,IACR,WAAW,CAAC,GAAG,MAAc;AAC5B,QAAE,MAAM,SAAS;AACjB,QAAE,UAAU,YAAY;AAAA,QACvB,OAAO,EAAE,MAAM;AAAA,QACf,IAAI,EAAE,KAAK,MAAM;AAAA,MAClB,CAAC;AACD,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,iBAAiB,CAAC,MAAM;AACvB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,EACD;AACD,CAAC;;;AC7BD,SAAS,SAAAC,cAAa;AAIf,IAAM,uBAAuBA,OAAM;AAAA,EACzC,OAAO;AAAA,IACN,OAAO;AAAA,IACP,QAAQ,CAAC;AAAA,EACV;AAAA,EACA,iBAAiB,CAAC,GAAG,YAAwB;AAAA,IAC5C,UAAU,KAAK,IAAI;AAAA,EACpB;AAAA,EACA,QAAQ,CAAC,MAAM;AACd,MAAE,MAAM,OAAO,KAAK,QAAQ;AAAA,EAC7B;AAAA,EACA,SAAS,OAAO,MAAM;AACrB,MAAE,MAAM,OAAO,KAAK,eAAe;AACnC,UAAM,IAAI,QAAc,CAAC,YAAY,WAAW,SAAS,GAAI,CAAC;AAC9D,MAAE,MAAM,OAAO,KAAK,aAAa;AAAA,EAClC;AAAA,EACA,iBAAiB,CAAC,GAAG,WAAuB;AAC3C,QAAI,QAAQ,eAAgB,GAAE,MAAM,OAAO,KAAK,iBAAiB;AAAA,EAClE;AAAA,EACA,WAAW,CAAC,GAAG,SAAS;AACvB,QAAI,KAAK,QAAQ,eAAgB,GAAE,MAAM,OAAO,KAAK,WAAW;AAAA,EACjE;AAAA,EACA,cAAc,CAAC,GAAG,SAAS;AAC1B,QAAI,KAAK,QAAQ,eAAgB,GAAE,MAAM,OAAO,KAAK,cAAc;AAAA,EACpE;AAAA,EACA,SAAS;AAAA,IACR,WAAW,CAAC,MAAM;AACjB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,WAAW,CAAC,GAAG,MAAc;AAC5B,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,EACD;AACD,CAAC;;;ACtCD,SAAS,SAAAC,cAAa;AAEf,IAAM,kBAAkBA,OAAM;AAAA,EACpC,OAAO;AAAA,IACN,WAAW;AAAA,EACZ;AAAA,EACA,YAAY,KAAK,WAAW;AAC3B,cAAU,iBAAiB,WAAW,CAACC,YAAe;AACrD,YAAM,OAAOA,QAAM;AACnB,UAAI,OAAO,SAAS,SAAU;AAE9B,UAAI;AACJ,UAAI;AACH,iBAAS,KAAK,MAAM,IAAI;AAAA,MACzB,QAAQ;AACP;AAAA,MACD;AAEA,UAAI,QAAQ,SAAS,QAAQ;AAC5B,YAAI,MAAM,YAAY,IAAI,MAAM,YAAY;AAC5C,kBAAU;AAAA,UACT,KAAK,UAAU;AAAA,YACd,MAAM;AAAA,YACN,WAAW,IAAI,MAAM;AAAA,YACrB,WAAW,KAAK,IAAI;AAAA,UACrB,CAAC;AAAA,QACF;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,aAAa,GAAG;AACf,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,eAAe,GAAG;AACjB,QAAE,MAAM,YAAY;AACpB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,EACD;AACD,CAAC;;;ACvCD,SAAS,SAAAC,cAAa;AAQf,IAAM,aAAaA,OAAM;AAAA,EAC/B,aAAa,CAAC,GAAG,UAAiB;AACjC,WAAO;AAAA,MACN,cAAc;AAAA,MACd,eAAe;AAAA,IAChB;AAAA,EACD;AAAA,EAEA,UAAU,CAAC,GAAG,UAAU;AACvB,MAAE,MAAM,gBAAgB;AAAA,EACzB;AAAA,EAEA,SAAS;AAAA,IACR,WAAW,CAAC,MAAM;AACjB,aAAO;AAAA,QACN,cAAc,EAAE,MAAM;AAAA,QACtB,eAAe,EAAE,MAAM;AAAA,MACxB;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;AC5BD,SAAS,SAAAC,QAAO,iBAAiB;AAG1B,IAAM,kBAAkBA,OAAM;AAAA,EACpC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,SAAS;AAAA;AAAA,IAER,WAAW,CAAC,GAAG,SAAS,MAAM;AAC7B,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAEA,SAAS,CAAC,MAAM;AACf,aAAO;AAAA,QACN,cAAc,EAAE,MAAM;AAAA,QACtB,WAAW,KAAK,IAAI;AAAA,MACrB;AAAA,IACD;AAAA;AAAA,IAEA,OAAO,CAAC,MAAM;AACb,QAAE,MAAM,QAAQ;AAAA,IACjB;AAAA,EACD;AACD,CAAC;AAGM,IAAM,mBAAmBA,OAAM;AAAA,EACrC,OAAO,EAAE,OAAO,GAAG,MAAM,KAAY;AAAA,EACrC,SAAS;AAAA;AAAA,IAER,kBAAkB,OAAO,GAAG,SAAS,MAAM;AAC1C,YAAM,QAAQ,QAAQ;AACtB,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAEA,WAAW,OAAO,GAAG,OAAe;AACnC,YAAM,QAAQ,QAAQ;AAGtB,YAAM,OAAO,EAAE,IAAI,WAAW,KAAK,IAAI,EAAE;AACzC,QAAE,MAAM,OAAO;AACf,aAAO;AAAA,IACR;AAAA;AAAA,IAEA,gBAAgB,OAAO,GAAG,gBAAyB;AAClD,YAAM,QAAQ,QAAQ;AAEtB,UAAI,aAAa;AAChB,cAAM,IAAI,UAAU,mBAAmB;AAAA,MACxC;AAEA,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;AAGM,IAAM,eAAeA,OAAM;AAAA,EACjC,OAAO,EAAE,SAAS,CAAC,EAAc;AAAA,EACjC,SAAS;AAAA;AAAA,IAER,iBAAiB,CAAC,MAAM;AACvB,aAAO,QAAQ,QAAQ,gBAAgB;AAAA,IACxC;AAAA;AAAA,IAEA,gBAAgB,CAAC,MAAuB;AACvC,aAAO,IAAI,QAAgB,CAAC,YAAY;AACvC,UAAE,MAAM,QAAQ,KAAK,SAAS;AAC9B,gBAAQ,eAAe;AAAA,MACxB,CAAC;AAAA,IACF;AAAA;AAAA,IAEA,iBAAiB,CAAC,MAAM;AACvB,aAAO,QAAQ,OAAO,IAAI,UAAU,oBAAoB,CAAC;AAAA,IAC1D;AAAA;AAAA,IAEA,YAAY,CAAC,MAAM;AAClB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,EACD;AACD,CAAC;;;ACjFD,SAAS,SAAAC,cAAa;AAGf,IAAM,oBAAoBA,OAAM;AAAA,EACtC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,SAAS;AAAA,IACR,eAAe;AAAA;AAAA,EAChB;AAAA,EACA,SAAS;AAAA,IACR,aAAa,OAAO,MAAM;AACzB,aAAO;AAAA,IACR;AAAA,IACA,YAAY,OAAO,MAAM;AAExB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,GAAG,CAAC;AACvD,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;AAGM,IAAM,mBAAmBA,OAAM;AAAA,EACrC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,SAAS;AAAA,IACR,eAAe;AAAA;AAAA,EAChB;AAAA,EACA,SAAS;AAAA,IACR,eAAe,OAAO,MAAM;AAE3B,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,GAAG,CAAC;AACvD,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;AAGM,IAAM,sBAAsBA,OAAM;AAAA,EACxC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,SAAS;AAAA,IACR,cAAc,OAAO,MAAM;AAC1B,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACtD,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;AAGM,IAAM,mBAAmBA,OAAM;AAAA,EACrC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,SAAS;AAAA,IACR,eAAe;AAAA;AAAA,EAChB;AAAA,EACA,SAAS;AAAA,IACR,YAAY,CAAC,MAAM;AAClB,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;;;ACzDD,SAAS,SAAAC,QAAO,aAAAC,kBAAiB;AAE1B,IAAM,qBAAqBD,OAAM;AAAA,EACvC,OAAO;AAAA,IACN,UAAU,CAAC;AAAA,EACZ;AAAA,EACA,SAAS;AAAA;AAAA,IAER,kBAAkB,MAAM;AACvB,YAAM,IAAIC,WAAU,sBAAsB;AAAA,IAC3C;AAAA;AAAA,IAGA,oBAAoB,MAAM;AACzB,YAAM,IAAIA,WAAU,0BAA0B;AAAA,QAC7C,MAAM;AAAA,QACN,UAAU;AAAA,UACT,QAAQ;AAAA,UACR,WAAW,KAAK,IAAI;AAAA,QACrB;AAAA,MACD,CAAC;AAAA,IACF;AAAA;AAAA,IAGA,oBAAoB,MAAM;AACzB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC5C;AAAA;AAAA,IAGA,kBAAkB,MAAM;AACvB,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,eAAe,OAAO,MAAM;AAE3B,aAAO,IAAI,QAAQ,CAAC,YAAY;AAC/B,mBAAW,MAAM;AAChB,kBAAQ,6CAA6C;AAAA,QACtD,GAAG,GAAK;AAAA,MACT,CAAC;AAAA,IACF;AAAA;AAAA,IAGA,eAAe,OAAO,GAAG,YAAoB;AAC5C,aAAO,IAAI,QAAQ,CAAC,YAAY;AAC/B,mBAAW,MAAM;AAChB,kBAAQ,mBAAmB,OAAO,IAAI;AAAA,QACvC,GAAG,OAAO;AAAA,MACX,CAAC;AAAA,IACF;AAAA;AAAA,IAGA,UAAU,CAAC,GAAG,UAAkB;AAC/B,QAAE,MAAM,SAAS,KAAK,KAAK;AAC3B,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,aAAa,CAAC,MAAM;AACnB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,eAAe,CAAC,MAAM;AACrB,QAAE,MAAM,WAAW,CAAC;AACpB,aAAO;AAAA,IACR;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,eAAe;AAAA;AAAA,EAChB;AACD,CAAC;AAGM,IAAM,qBAAqBD,OAAM;AAAA,EACvC,OAAO,CAAC;AAAA,EACR,SAAS;AAAA,IACR,aAAa,YAAY;AACxB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACtD,aAAO;AAAA,IACR;AAAA,IACA,YAAY,YAAY;AACvB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,GAAG,CAAC;AACvD,aAAO;AAAA,IACR;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,eAAe;AAAA;AAAA,EAChB;AACD,CAAC;;;AC1FD,SAAS,SAAAE,eAAa;AAEf,IAAM,qBAAqBA,QAAM;AAAA,EACvC,OAAO;AAAA,IACN,OAAO;AAAA,EACR;AAAA,EACA,MAAM;AAAA,IACL,aAAa;AAAA,EACd;AAAA,EACA,SAAS;AAAA;AAAA,IAER,UAAU,CAAC,GAAG,aAAqB;AAClC,QAAE,MAAM,QAAQ;AAChB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAEA,mBAAmB,CAAC,GAAG,UAAkB;AACxC,eAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC/B,UAAE,MAAM;AAAA,MACT;AACA,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAEA,UAAU,CAAC,MAAM;AAChB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAEA,YAAY,CAAC,MAAM;AAClB,YAAM,UAAU,EAAE,MAAM,QAAQ;AAChC,aAAO;AAAA,IACR;AAAA;AAAA,IAEA,gBAAgB,CAAC,MAAM;AACtB,aAAO,EAAE,KAAK;AAAA,IACf;AAAA;AAAA,IAEA,kBAAkB,CAAC,MAAM;AACxB,QAAE,KAAK,cAAc;AAAA,IACtB;AAAA,EACD;AAAA;AAAA,EAEA,eAAe,CAAC,MAAM;AACrB,MAAE,KAAK;AAAA,EACR;AACD,CAAC;;;AC5CD,SAAS,SAAAC,eAAa;AAIf,IAAM,gBAAgBA,QAAM;AAAA,EAClC,OAAO;AAAA,IACN,cAAc;AAAA,IACd,WAAW;AAAA;AAAA;AAAA,IAGX,YAAY,CAAC;AAAA,IACb,cAAc;AAAA,EACf;AAAA,EACA,QAAQ,CAAC,MAAM;AAEd,MAAE,MAAM,YAAY,EAAE;AAAA,EACvB;AAAA,EACA,SAAS;AAAA;AAAA,IAER,eAAe,CAAC,GAAG,SAAiC;AACnD,QAAE,MAAM,aAAa;AACrB,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,iBAAiB,CAAC,GAAG,WAAmB;AACvC,QAAE,MAAM,eAAe;AACvB,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,aAAa,CAAC,MAAM;AAEnB,YAAM,WAAW;AAAA,QAChB,MAAM,EAAE;AAAA,QACR,MAAM,EAAE,MAAM;AAAA,QACd,QAAQ,EAAE,MAAM;AAAA,MACjB;AAGA,QAAE,MAAM,eAAe;AACvB,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,cAAc,CAAC,MAAM;AACpB,aAAO,EAAE;AAAA,IACV;AAAA;AAAA,IAGA,QAAQ,CAAC,GAAG,QAAgB;AAC3B,aAAO,EAAE,MAAM,WAAW,GAAG,KAAK;AAAA,IACnC;AAAA;AAAA,IAGA,SAAS,CAAC,MAAM;AACf,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,WAAW,CAAC,MAAM;AACjB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,oBAAoB,CAAC,MAAM;AAC1B,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,iBAAiB,CAAC,MAAM;AACvB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,EACD;AACD,CAAC;;;AC1ED,SAAS,SAAAC,eAAa;AAGf,IAAM,iBAAiBA,QAAM;AAAA,EACnC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,WAAW,EAAE,OAAO,QAAQ;AAAA,EAC5B,MAAM,EAAE,SAAS,IAAI,MAAM,aAAa;AAAA,EACxC,SAAS;AAAA,IACR,SAAS,CAAC,MAAM;AACf,aAAO,EAAE;AAAA,IACV;AAAA,IACA,SAAS,CAAC,MAAM;AACf,aAAO,EAAE,KAAK;AAAA,IACf;AAAA,EACD;AACD,CAAC;AAGM,IAAM,iBAAiBA,QAAM;AAAA,EACnC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,WAAW,EAAE,OAAO,QAAQ;AAAA,EAC5B,MAAM;AAAA,IACL,SAAS;AAAA,IACT,QAAQ;AAAA,MACP,OAAO;AAAA,MACP,OAAO,CAAC,GAAG,GAAG,CAAC;AAAA,MACf,KAAK,EAAE,KAAK,QAAQ;AAAA,IACrB;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,SAAS,CAAC,MAAM;AACf,aAAO,EAAE;AAAA,IACV;AAAA,IACA,cAAc,CAAC,MAAM;AAEpB,QAAE,KAAK,OAAO,QAAQ;AACtB,QAAE,KAAK,OAAO,MAAM,KAAK,CAAC;AAC1B,QAAE,KAAK,OAAO,IAAI,MAAM;AACxB,aAAO,EAAE;AAAA,IACV;AAAA,EACD;AACD,CAAC;AAGM,IAAM,kBAAkBA,QAAM;AAAA,EACpC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,WAAW,EAAE,OAAO,QAAQ;AAAA,EAC5B,YAAY,MAAM;AACjB,WAAO;AAAA,MACN,QAAQ,KAAK,OAAO;AAAA,MACpB,UAAU,SAAS,KAAK,MAAM,KAAK,OAAO,IAAI,GAAI,CAAC;AAAA,IACpD;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,SAAS,CAAC,MAAW;AACpB,aAAO,EAAE;AAAA,IACV;AAAA,EACD;AACD,CAAC;AAGM,IAAM,iBAAiBA,QAAM;AAAA,EACnC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,WAAW,EAAE,OAAO,QAAQ;AAAA,EAC5B,YAAY,MAAM;AACjB,WAAO;AAAA,MACN,IAAI,KAAK,MAAM,KAAK,OAAO,IAAI,GAAO;AAAA,IACvC;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,SAAS,CAAC,MAAW;AACpB,aAAO,EAAE;AAAA,IACV;AAAA,EACD;AACD,CAAC;AAGM,IAAM,iBAAiBA,QAAM;AAAA,EACnC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,WAAW,EAAE,OAAO,QAAQ;AAAA,EAC5B,YAAY,CAAC,GAAG,cAAmB;AAClC,WAAO;AAAA,MACN,cAAc,QAAQ,WAAW,MAAM;AAAA,IACxC;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,SAAS,CAAC,MAAW;AACpB,aAAO,EAAE;AAAA,IACV;AAAA,EACD;AACD,CAAC;;;AC1FD,SAAS,SAAAC,eAAgC;AAElC,IAAM,UAAUA,QAAM;AAAA,EAC5B,SAAS;AAAA,IACR,SAAS,OACR,GACA,KACA,UACI;AACJ,YAAM,EAAE,GAAG,IAAI,KAAK,KAAK;AACzB,aAAO;AAAA,IACR;AAAA,IACA,SAAS,OACR,GACA,QACI;AACJ,aAAO,MAAM,EAAE,GAAG,IAAI,GAAG;AAAA,IAC1B;AAAA,IACA,UAAU,OACT,GACA,WACI;AACJ,YAAM,UAAU,MAAM,EAAE,GAAG,KAAK,QAAQ,EAAE,SAAS,OAAO,CAAC;AAC3D,aAAO,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,OAAO;AAAA,QACrC;AAAA,QACA;AAAA,MACD,EAAE;AAAA,IACH;AAAA,IACA,sBAAsB,OACrB,GACA,KACA,WACI;AACJ,YAAM,SAAS,IAAI,WAAW,MAAM,EAAE;AACtC,YAAM,EAAE,GAAG,IAAI,KAAK,QAAQ,EAAE,MAAM,cAAc,CAAC;AACnD,YAAM,SAAS,MAAM,EAAE,GAAG,IAAI,KAAK,EAAE,MAAM,cAAc,CAAC;AAC1D,UAAI,CAAC,QAAQ;AACZ,eAAO;AAAA,MACR;AACA,aAAO,MAAM,KAAK,IAAI,WAAW,MAAM,CAAC;AAAA,IACzC;AAAA,EACD;AACD,CAAC;;;AC1CD,SAAS,SAAAC,eAAa;AAKf,IAAM,oBAAoBA,QAAM;AAAA,EACtC,OAAO,CAAC;AAAA,EACR,SAAS;AAAA;AAAA;AAAA;AAAA,IAIR,qBAAqB,CAAC,GAAG,SAA8B;AACtD,aAAO;AAAA,QACN,WAAW,KAAK,MAAM;AAAA,QACtB,WAAW,KAAK,MAAM,CAAC;AAAA,QACvB,UAAU,KAAK,MAAM,KAAK,MAAM,SAAS,CAAC;AAAA,MAC3C;AAAA,IACD;AAAA;AAAA;AAAA;AAAA,IAKA,kBAAkB,CAAC,GAAG,cAAsB;AAC3C,YAAM,QAAkB,CAAC;AACzB,eAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AACnC,cAAM,KAAK,QAAQ,CAAC,6CAA6C;AAAA,MAClE;AACA,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA;AAAA;AAAA,IAKA,MAAM,CAAC,GAAG,SAAkB;AAC3B,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;AAKM,IAAM,wBAAwBA,QAAM;AAAA,EAC1C,OAAO,CAAC;AAAA,EACR,WAAW;AAAA,IACV,iBAAiB;AAAA,EAClB;AAAA,EACA,SAAS;AAAA;AAAA;AAAA;AAAA,IAIR,qBAAqB,CAAC,GAAG,SAA8B;AACtD,QAAE,KAAK,MAAM,kBAAkB,KAAK,MAAM;AAC1C,aAAO;AAAA,QACN,WAAW,KAAK,MAAM;AAAA,QACtB,WAAW,KAAK,MAAM,CAAC;AAAA,QACvB,UAAU,KAAK,MAAM,KAAK,MAAM,SAAS,CAAC;AAAA,MAC3C;AAAA,IACD;AAAA;AAAA;AAAA;AAAA,IAKA,kBAAkB,CAAC,GAAG,cAAsB;AAC3C,YAAM,QAAkB,CAAC;AACzB,eAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AACnC,cAAM,KAAK,QAAQ,CAAC,6CAA6C;AAAA,MAClE;AACA,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA;AAAA;AAAA,IAKA,MAAM,CAAC,GAAG,SAAkB;AAC3B,aAAO;AAAA,IACR;AAAA;AAAA;AAAA;AAAA,IAKA,oBAAoB,CAAC,MAAM;AAC1B,aAAO,EAAE,KAAK,MAAM;AAAA,IACrB;AAAA,EACD;AACD,CAAC;;;ACrFD,SAAS,SAAAC,eAAa;AACtB,SAAS,UAAU;AAEZ,IAAM,iBAAiBA,QAAM;AAAA,EACnC,IAAI,GAAG;AAAA,IACN,WAAW,OAAOC,SAAO;AACxB,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOhB;AAAA,IACF;AAAA,EACD,CAAC;AAAA,EACD,SAAS;AAAA,IACR,SAAS,OAAO,GAAG,UAAkB;AACpC,YAAM,YAAY,KAAK,IAAI;AAC3B,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,aAAO,EAAE,OAAO,UAAU;AAAA,IAC3B;AAAA,IACA,UAAU,OAAO,MAAM;AACtB,aAAO,MAAM,EAAE,GAAG,QAAQ,8CAA8C;AAAA,IACzE;AAAA,IACA,YAAY,OAAO,GAAG,OAAe;AACpC,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,MACD;AACA,YAAM,OAAO,MAAM,EAAE,GAAG,QAAQ,oCAAoC,EAAE;AACtE,aAAO,KAAK,CAAC;AAAA,IACd;AAAA,IACA,YAAY,OAAO,GAAG,OAAe;AACpC,YAAM,EAAE,GAAG,QAAQ,kCAAkC,EAAE;AACvD,aAAO,EAAE,SAAS,GAAG;AAAA,IACtB;AAAA,EACD;AACD,CAAC;;;AC1CD,SAAS,SAAAC,eAAa;AACtB,SAAS,MAAAC,WAAU;AACnB,SAAS,UAAU;;;ACFnB;AAAA;AAAA;AAAA;AAAA,SAAS,aAAa,MAAM,eAAe;AAEpC,IAAM,QAAQ,YAAY,SAAS;AAAA,EACzC,IAAI,QAAQ,IAAI,EAAE,WAAW,EAAE,eAAe,KAAK,CAAC;AAAA,EACpD,OAAO,KAAK,OAAO,EAAE,QAAQ;AAAA,EAC7B,WAAW,QAAQ,WAAW,EAAE,QAAQ,CAAC;AAAA,EACzC,WAAW,QAAQ,YAAY,EAAE,QAAQ;AAC1C,CAAC;;;ACPD;AAAA,EACE,SAAW;AAAA,EACX,SAAW;AAAA,EACX,SAAW;AAAA,IACT;AAAA,MACE,KAAO;AAAA,MACP,SAAW;AAAA,MACX,MAAQ;AAAA,MACR,KAAO;AAAA,MACP,aAAe;AAAA,IACjB;AAAA,EACF;AACF;;;ACZA;;;ACGE,IAAO,qBAAQ;AAAA,EACb;AAAA,EACA,YAAY;AAAA,IACV;AAAA,EACF;AACF;;;AJFF,IAAM,EAAE,OAAAC,OAAM,IAAI;AAEX,IAAM,qBAAqBC,QAAM;AAAA,EACvC,IAAIC,IAAG,EAAE,wBAAQ,+BAAW,CAAC;AAAA,EAC7B,SAAS;AAAA,IACR,SAAS,OAAO,GAAG,UAAkB;AACpC,YAAM,SAAS,MAAM,EAAE,GAAG,OAAOF,MAAK,EAAE,OAAO;AAAA,QAC9C;AAAA,QACA,WAAW,KAAK,IAAI;AAAA,MACrB,CAAC,EAAE,UAAU;AACb,aAAO,OAAO,CAAC;AAAA,IAChB;AAAA,IACA,UAAU,OAAO,MAAM;AACtB,aAAO,MAAM,EAAE,GAAG,OAAO,EAAE,KAAKA,MAAK,EAAE,QAAQA,OAAM,SAAS;AAAA,IAC/D;AAAA,IACA,YAAY,OAAO,GAAG,OAAe;AACpC,YAAM,WAAW,MAAM,EAAE,GAAG,OAAO,EAAE,KAAKA,MAAK,EAAE,MAAM,GAAGA,OAAM,IAAI,EAAE,CAAC;AACvE,UAAI,CAAC,SAAS,CAAC,EAAG,QAAO;AACzB,YAAM,eAAe,SAAS,CAAC,EAAE,YAAY,IAAI;AACjD,YAAM,SAAS,MAAM,EAAE,GAAG,OAAOA,MAAK,EACpC,IAAI,EAAE,WAAW,aAAa,CAAC,EAC/B,MAAM,GAAGA,OAAM,IAAI,EAAE,CAAC,EACtB,UAAU;AACZ,aAAO,OAAO,CAAC;AAAA,IAChB;AAAA,IACA,YAAY,OAAO,GAAG,OAAe;AACpC,YAAM,EAAE,GAAG,OAAOA,MAAK,EAAE,MAAM,GAAGA,OAAM,IAAI,EAAE,CAAC;AAC/C,aAAO,EAAE,SAAS,GAAG;AAAA,IACtB;AAAA,EACD;AACD,CAAC;;;AKpCD,SAAS,SAAAG,SAAO,SAAAC,cAAa;AAC7B,SAAS,MAAAC,WAAU;AAEZ,IAAM,kBAAkBF,QAAM;AAAA,EACpC,OAAO;AAAA,IACN,YAAY;AAAA,EACb;AAAA,EACA,IAAIE,IAAG;AAAA,IACN,WAAW,OAAOA,SAAO;AACxB,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKhB;AACD,YAAMA,KAAG,QAAQ;AAAA;AAAA,IAEhB;AAAA,IACF;AAAA,EACD,CAAC;AAAA,EACD,QAAQ;AAAA,IACP,mBAAmBD,OAAyB;AAAA,IAC5C,oBAAoBA,OAAyB;AAAA,EAC9C;AAAA,EACA,SAAS;AAAA,IACR,gBAAgB,CAAC,MAAM;AACtB,QAAE,MAAM,cAAc;AACtB,QAAE,UAAU,qBAAqB,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9D,aAAO,EAAE,OAAO,EAAE,MAAM,WAAW;AAAA,IACpC;AAAA,IACA,eAAe,CAAC,MAAM;AACrB,aAAO,EAAE,OAAO,EAAE,MAAM,WAAW;AAAA,IACpC;AAAA,IACA,iBAAiB,OAAO,MAAM;AAC7B,YAAM,EAAE,GAAG,QAAQ,mDAAmD;AACtE,YAAM,UAAU,MAAM,EAAE,GAAG;AAAA,QAC1B;AAAA,MACD;AACA,YAAM,QAAQ,QAAQ,CAAC,EAAE;AACzB,QAAE,UAAU,sBAAsB,EAAE,MAAM,CAAC;AAC3C,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,gBAAgB,OAAO,MAAM;AAC5B,YAAM,UAAU,MAAM,EAAE,GAAG;AAAA,QAC1B;AAAA,MACD;AACA,aAAO,EAAE,OAAO,QAAQ,CAAC,EAAE,MAAM;AAAA,IAClC;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;;;ACpDD,SAAS,SAAAE,SAAO,SAAAC,cAAa;AAUtB,IAAM,iBAAiBD,QAAM;AAAA,EACnC,OAAO;AAAA,IACN,eAAe;AAAA,IACf,oBAAoB;AAAA,EACrB;AAAA,EACA,QAAQ;AAAA,IACP,eAAeC,OAAsD;AAAA,IACrE,kBAAkBA,OAAsB;AAAA,IACxC,eAAeA,OAAyC;AAAA,EACzD;AAAA;AAAA,EAEA,iBAAiB,CAChB,GACA,WACe;AACf,WAAO;AAAA,MACN,UAAU,QAAQ,YAAY;AAAA,MAC9B,MAAM,QAAQ,QAAQ;AAAA,MACtB,SAAS;AAAA,MACT,WAAW,KAAK,IAAI;AAAA,MACpB,SAAS,QAAQ,WAAW;AAAA,IAC7B;AAAA,EACD;AAAA;AAAA,EAEA,WAAW,CAAC,GAAG,SAAS;AAEvB,MAAE,UAAU,iBAAiB;AAAA,MAC5B,IAAI,KAAK;AAAA,MACT,UAAU;AAAA,MACV,MAAM;AAAA,IACP,CAAC;AAAA,EACF;AAAA;AAAA,EAEA,cAAc,CAAC,GAAG,SAAS;AAC1B,QAAI,CAAC,KAAK,OAAO,SAAS;AACzB,QAAE,MAAM,sBAAsB;AAC9B,QAAE,UAAU,oBAAoB;AAAA,QAC/B,IAAI,KAAK;AAAA,MACV,CAAC;AAAA,IACF;AAAA,EACD;AAAA,EACA,SAAS;AAAA;AAAA,IAER,sBAAsB,CAAC,GAAG,SAAS,MAAM;AACxC,QAAE,KAAK,MAAM,WAAW;AAAA,IACzB;AAAA;AAAA,IAGA,wBAAwB,CAAC,GAAG,SAAS,MAAM;AAC1C,QAAE,MAAM,iBAAiB;AACzB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,oBAAoB,CAAC,MAAM;AAC1B,aAAO,EAAE,IAAI,EAAE,KAAK,IAAI,GAAG,EAAE,KAAK,MAAM;AAAA,IACzC;AAAA;AAAA,IAGA,kBAAkB,CAAC,MAAM;AACxB,aAAO,EAAE,MACP,QAAQ,EACR,OAAO,CAACC,OAAM,CAACA,GAAE,CAAC,EAAE,OAAO,OAAO,EAClC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,EACf,QAAQ;AAAA,IACX;AAAA;AAAA,IAGA,uBAAuB,CAAC,MAAM;AAC7B,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,wBAAwB,CAAC,MAAM;AAC9B,aAAO,EAAE,MACP,QAAQ,EACR,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,EAAE,IAAI,GAAG,KAAK,MAAM,EAAE,EAC3C,QAAQ;AAAA,IACX;AAAA;AAAA,IAGA,kBAAkB,CAAC,GAAG,UAAkB,YAAoB;AAC3D,UAAI,EAAE,MAAM,IAAI,QAAQ,GAAG;AAC1B,UAAE,MACA,IAAI,QAAQ,EACZ,KAAK,iBAAiB,EAAE,MAAM,EAAE,KAAK,IAAI,QAAQ,CAAC;AACpD,eAAO;AAAA,MACR,OAAO;AACN,eAAO;AAAA,MACR;AAAA,IACD;AAAA;AAAA,IAGA,kBAAkB,CACjB,GACA,YACI;AACJ,UAAI,QAAQ,SAAU,GAAE,KAAK,MAAM,WAAW,QAAQ;AACtD,UAAI,QAAQ,KAAM,GAAE,KAAK,MAAM,OAAO,QAAQ;AAC9C,aAAO,EAAE,KAAK;AAAA,IACf;AAAA,IACA,gBAAgB,CAAC,GAAG,WAAoB;AACvC,QAAE,KAAK,WAAW,UAAU,iBAAiB;AAC7C,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;;;ACpHD,SAAS,SAAAC,SAAO,aAAAC,kBAAiB;AAE1B,IAAM,wBAAwBD,QAAM;AAAA,EAC1C,iBAAiB,OAAO,IAAI,WAAiC;AAC5D,QAAI,QAAQ,QAAQ;AACnB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,GAAG,CAAC;AACvD,YAAM,IAAIC,WAAU,uBAAuB;AAAA,QAC1C,MAAM;AAAA,MACP,CAAC;AAAA,IACF;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,MAAM,MAAM;AAAA,EACb;AACD,CAAC;;;ACdD,SAAS,SAAAC,eAAqC;AAKvC,IAAM,qBAAqBA,QAAM;AAAA,EACvC,OAAO;AAAA;AAAA,IAEN,wBAAwB;AAAA,MACvB,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,eAAe;AAAA,MACf,gBAAgB,CAAC;AAAA,IAClB;AAAA,IACA,wBAAwB;AAAA,MACvB,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,eAAe;AAAA,MACf,gBAAgB,CAAC;AAAA,IAClB;AAAA,IACA,kBAAkB;AAAA,MACjB,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,eAAe;AAAA,MACf,gBAAgB,CAAC;AAAA,IAClB;AAAA,IACA,oBAAoB;AAAA,MACnB,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,eAAe;AAAA,MACf,gBAAgB,CAAC;AAAA,IAClB;AAAA,EACD;AAAA,EACA,iBAAiB,CAAC,GAAG,WAAuC;AAG3D,QAAI,cAKO;AAEX,QAAI,QAAQ,gBAAgB,EAAE,SAAS;AACtC,YAAM,UAAkC,CAAC;AACzC,QAAE,QAAQ,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACzC,gBAAQ,GAAG,IAAI;AAAA,MAChB,CAAC;AACD,oBAAc;AAAA,QACb,YAAY;AAAA,QACZ,YAAY,EAAE,QAAQ;AAAA,QACtB,eAAe,EAAE,QAAQ;AAAA,QACzB,gBAAgB;AAAA,MACjB;AAAA,IACD;AAEA,WAAO;AAAA,MACN,cAAc,QAAQ,gBAAgB;AAAA,MACtC;AAAA,IACD;AAAA,EACD;AAAA,EACA,WAAW,CAAC,GAAG,SAAS;AAEvB,QAAI,KAAK,MAAM,aAAa;AAC3B,QAAE,MAAM,yBAAyB,KAAK,MAAM;AAAA,IAC7C;AAAA,EACD;AAAA,EACA,iBAAiB,CAAC,GAAG,WAAW;AAC/B,QAAI,QAAQ,cAAc;AACzB,UAAI,EAAE,SAAS;AACd,UAAE,MAAM,uBAAuB,aAAa;AAC5C,UAAE,MAAM,uBAAuB,aAAa,EAAE,QAAQ;AACtD,UAAE,MAAM,uBAAuB,gBAAgB,EAAE,QAAQ;AAGzD,cAAM,UAAkC,CAAC;AACzC,UAAE,QAAQ,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACzC,kBAAQ,GAAG,IAAI;AAAA,QAChB,CAAC;AACD,UAAE,MAAM,uBAAuB,iBAAiB;AAAA,MACjD,OAAO;AAEN,UAAE,MAAM,uBAAuB,aAAa;AAAA,MAC7C;AAAA,IACD;AAAA,EACD;AAAA,EACA,WAAW,CAAC,GAAG,YAAY;AAE1B,MAAE,MAAM,iBAAiB,aAAa;AACtC,MAAE,MAAM,iBAAiB,aAAa,QAAQ;AAC9C,MAAE,MAAM,iBAAiB,gBAAgB,QAAQ;AAGjD,UAAM,UAAkC,CAAC;AACzC,YAAQ,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,cAAQ,GAAG,IAAI;AAAA,IAChB,CAAC;AACD,MAAE,MAAM,iBAAiB,iBAAiB;AAG1C,WAAO,IAAI;AAAA,MACV,KAAK,UAAU;AAAA,QACd,YAAY;AAAA,QACZ,YAAY,QAAQ;AAAA,QACpB,eAAe,QAAQ;AAAA,QACvB,gBAAgB;AAAA,MACjB,CAAC;AAAA,MACD;AAAA,QACC,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC/C;AAAA,IACD;AAAA,EACD;AAAA,EACA,aAAa,CAAC,GAAG,cAAc;AAC9B,QAAI,CAAC,EAAE,QAAS,OAAM;AAEtB,MAAE,MAAM,mBAAmB,aAAa;AACxC,MAAE,MAAM,mBAAmB,aAAa,EAAE,QAAQ;AAClD,MAAE,MAAM,mBAAmB,gBAAgB,EAAE,QAAQ;AAGrD,UAAM,UAAkC,CAAC;AACzC,MAAE,QAAQ,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACzC,cAAQ,GAAG,IAAI;AAAA,IAChB,CAAC;AACD,MAAE,MAAM,mBAAmB,iBAAiB;AAG5C,cAAU;AAAA,MACT,KAAK,UAAU;AAAA,QACd,YAAY;AAAA,QACZ,YAAY,EAAE,QAAQ;AAAA,QACtB,eAAe,EAAE,QAAQ;AAAA,QACzB,gBAAgB;AAAA,MACjB,CAAC;AAAA,IACF;AAGA,cAAU,iBAAiB,WAAW,CAACC,YAA6B;AACnE,gBAAU,KAAKA,QAAM,IAAI;AAAA,IAC1B,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,gBAAgB,CAAC,MAAM;AACtB,aAAO;AAAA,QACN,iBAAiB,EAAE,MAAM;AAAA,QACzB,iBAAiB,EAAE,MAAM;AAAA,QACzB,WAAW,EAAE,MAAM;AAAA,QACnB,aAAa,EAAE,MAAM;AAAA,MACtB;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;ACxJD,SAAS,YAAY;AACrB,SAAS,SAAAC,eAAkC;AAEpC,IAAM,eAAeA,QAAM;AAAA,EACjC,OAAO;AAAA,IACN,cAAc;AAAA,EACf;AAAA,EACA,UACC,KACA,SACC;AACD,UAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,UAAM,SAAS,QAAQ;AAGvB,QAAI,MAAM;AAGV,QAAI,IAAI,aAAa,cAAc;AAClC,aAAO,IAAI;AAAA,QACV,KAAK,UAAU,EAAE,SAAS,oBAAoB,CAAC;AAAA,QAC/C;AAAA,UACC,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC/C;AAAA,MACD;AAAA,IACD;AAEA,QAAI,IAAI,aAAa,eAAe,WAAW,QAAQ;AACtD,aAAO,IAAI,SAAS,QAAQ,MAAM;AAAA,QACjC,SAAS,QAAQ;AAAA,MAClB,CAAC;AAAA,IACF;AAEA,QAAI,IAAI,aAAa,cAAc;AAClC,aAAO,IAAI;AAAA,QACV,KAAK,UAAU;AAAA,UACd,cAAc,IAAI,MAAM;AAAA,QACzB,CAAC;AAAA,QACD;AAAA,UACC,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC/C;AAAA,MACD;AAAA,IACD;AAEA,QAAI,IAAI,aAAa,gBAAgB;AACpC,YAAM,UAAkC,CAAC;AACzC,cAAQ,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,gBAAQ,GAAG,IAAI;AAAA,MAChB,CAAC;AACD,aAAO,IAAI,SAAS,KAAK,UAAU,OAAO,GAAG;AAAA,QAC5C,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC/C,CAAC;AAAA,IACF;AAGA,WAAO,IAAI,SAAS,aAAa,EAAE,QAAQ,IAAI,CAAC;AAAA,EACjD;AAAA,EACA,SAAS,CAAC;AACX,CAAC;AAEM,IAAM,wBAAwBA,QAAM;AAAA,EAC1C,SAAS,CAAC;AACX,CAAC;AAEM,IAAM,yBAAyBA,QAAM;AAAA,EAC3C,UAAU,KAAK,SAAS;AAEvB,WAAO;AAAA,EACR;AAAA,EACA,SAAS,CAAC;AACX,CAAC;AAEM,IAAM,mBAAmBA,QAAM;AAAA,EACrC,aAAa;AACZ,UAAM,SAAS,IAAI,KAAK;AAGxB,WAAO;AAAA,MAAI;AAAA,MAAK,CAAC,MAChB,EAAE,KAAK,EAAE,SAAS,yBAAyB,CAAC;AAAA,IAC7C;AAEA,WAAO;AAAA,MAAI;AAAA,MAAU,CAAC,MACrB,EAAE,KAAK;AAAA,QACN,EAAE,IAAI,GAAG,MAAM,QAAQ;AAAA,QACvB,EAAE,IAAI,GAAG,MAAM,MAAM;AAAA,MACtB,CAAC;AAAA,IACF;AAEA,WAAO,IAAI,cAAc,CAAC,MAAW;AACpC,YAAM,KAAK,EAAE,IAAI,MAAM,IAAI;AAC3B,aAAO,EAAE,KAAK;AAAA,QACb,IAAI,SAAS,EAAE;AAAA,QACf,MAAM,OAAO,MAAM,UAAU;AAAA,MAC9B,CAAC;AAAA,IACF,CAAC;AAED,WAAO,KAAK,UAAU,OAAO,MAAW;AACvC,YAAM,OAAO,MAAM,EAAE,IAAI,KAAK;AAC9B,aAAO,EAAE,KAAK,EAAE,IAAI,GAAG,GAAG,KAAK,GAAG,GAAG;AAAA,IACtC,CAAC;AAED,WAAO,IAAI,cAAc,OAAO,MAAW;AAC1C,YAAM,KAAK,EAAE,IAAI,MAAM,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,IAAI,KAAK;AAC9B,aAAO,EAAE,KAAK,EAAE,IAAI,SAAS,EAAE,GAAG,GAAG,KAAK,CAAC;AAAA,IAC5C,CAAC;AAED,WAAO,OAAO,cAAc,CAAC,MAAW;AACvC,YAAM,KAAK,EAAE,IAAI,MAAM,IAAI;AAC3B,aAAO,EAAE,KAAK,EAAE,SAAS,QAAQ,EAAE,WAAW,CAAC;AAAA,IAChD,CAAC;AAGD,WAAO,EAAE,OAAO;AAAA,EACjB;AAAA,EACA,UACC,KACA,SACC;AAED,WAAO,IAAI,KAAK,OAAO,MAAM,OAAO;AAAA,EACrC;AAAA,EACA,SAAS,CAAC;AACX,CAAC;;;AC3HD,SAAS,SAAAC,eAAkC;AAEpC,IAAM,gCAAgCA,QAAM;AAAA,EAClD,SAAS,CAAC;AAAA,EACV,UACC,KACA,SACC;AAED,UAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,UAAM,SAAS,QAAQ;AAGvB,UAAM,UAAkC,CAAC;AACzC,YAAQ,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,cAAQ,GAAG,IAAI;AAAA,IAChB,CAAC;AAGD,UAAM,aAAa,YAAY;AAC9B,UAAI,CAAC,QAAQ,MAAM;AAClB,eAAO;AAAA,MACR;AAEA,YAAM,cAAc,QAAQ,QAAQ,IAAI,cAAc,KAAK;AAE3D,UAAI;AACH,YAAI,YAAY,SAAS,kBAAkB,GAAG;AAC7C,gBAAMC,QAAO,MAAM,QAAQ,KAAK;AAChC,iBAAOA,QAAO,KAAK,MAAMA,KAAI,IAAI;AAAA,QAClC,OAAO;AAEN,gBAAMA,QAAO,MAAM,QAAQ,KAAK;AAChC,iBAAOA,SAAQ;AAAA,QAChB;AAAA,MACD,SAAS,OAAO;AAEf,eAAO;AAAA,MACR;AAAA,IACD;AAGA,QAAI,WAAW,QAAQ;AACtB,aAAO,IAAI,SAAS,MAAM;AAAA,QACzB,QAAQ;AAAA,MACT,CAAC;AAAA,IACF;AAEA,QAAI,WAAW,WAAW;AACzB,aAAO,IAAI,SAAS,MAAM;AAAA,QACzB,QAAQ;AAAA,MACT,CAAC;AAAA,IACF;AAGA,WAAO,WAAW,EAAE,KAAK,CAAC,SAAS;AAClC,YAAM,eAAe;AAAA;AAAA,QAEpB,KAAK,QAAQ;AAAA,QACb,UAAU,IAAI;AAAA,QACd,QAAQ,IAAI;AAAA,QACZ,cAAc,OAAO,YAAY,IAAI,aAAa,QAAQ,CAAC;AAAA,QAC3D,MAAM,IAAI;AAAA;AAAA,QAGV,QAAQ,QAAQ;AAAA;AAAA,QAGhB;AAAA;AAAA,QAGA;AAAA,QACA,UACC,OAAO,SAAS,WACb,OACA,SAAS,QAAQ,QAAQ,SAAS,OACjC,KACA;AAAA;AAAA;AAAA;AAAA,QAKL,OAAO,QAAQ,SAAS;AAAA,QACxB,aAAa,QAAQ,eAAe;AAAA,QACpC,MAAM,QAAQ,QAAQ;AAAA,QACtB,UAAU,QAAQ,YAAY;AAAA,QAC9B,UAAU,QAAQ,YAAY;AAAA,MAC/B;AAEA,aAAO,IAAI,SAAS,KAAK,UAAU,YAAY,GAAG;AAAA,QACjD,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC/C,CAAC;AAAA,IACF,CAAC;AAAA,EACF;AACD,CAAC;;;AC9FD,SAA4B,SAAAC,eAAsC;AAE3D,IAAM,oBAAoBA,QAAM;AAAA,EACtC,OAAO;AAAA,IACN,iBAAiB;AAAA,IACjB,cAAc;AAAA,EACf;AAAA,EACA,YAAY,KAAK,WAAW;AAC3B,QAAI,MAAM,kBAAkB,IAAI,MAAM,kBAAkB;AACxD,YAAQ;AAAA,MACP,kCAAkC,IAAI,MAAM,eAAe;AAAA,IAC5D;AAGA,cAAU;AAAA,MACT,KAAK,UAAU;AAAA,QACd,MAAM;AAAA,QACN,iBAAiB,IAAI,MAAM;AAAA,MAC5B,CAAC;AAAA,IACF;AACA,YAAQ,IAAI,8BAA8B;AAG1C,cAAU,iBAAiB,WAAW,CAACC,YAAe;AACrD,UAAI,MAAM,eAAe,IAAI,MAAM,eAAe;AAClD,cAAQ;AAAA,QACP,0CAA0C,IAAI,MAAM,YAAY;AAAA,QAChEA,QAAM;AAAA,MACP;AAEA,YAAM,OAAOA,QAAM;AACnB,UAAI,OAAO,SAAS,UAAU;AAC7B,YAAI;AACH,gBAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,cAAI,OAAO,SAAS,QAAQ;AAC3B,sBAAU;AAAA,cACT,KAAK,UAAU;AAAA,gBACd,MAAM;AAAA,gBACN,WAAW,KAAK,IAAI;AAAA,cACrB,CAAC;AAAA,YACF;AAAA,UACD,WAAW,OAAO,SAAS,YAAY;AACtC,oBAAQ;AAAA,cACP,wCAAwC,IAAI,MAAM,eAAe,eAAe,IAAI,MAAM,YAAY;AAAA,YACvG;AACA,sBAAU;AAAA,cACT,KAAK,UAAU;AAAA,gBACd,MAAM;AAAA,gBACN,iBAAiB,IAAI,MAAM;AAAA,gBAC3B,cAAc,IAAI,MAAM;AAAA,cACzB,CAAC;AAAA,YACF;AAAA,UACD,WAAW,OAAO,SAAS,kBAAkB;AAE5C,kBAAM,MAAM,IAAI,SAAS,OAAO;AAChC,kBAAM,SAAS,IAAI,IAAI,GAAG;AAC1B,sBAAU;AAAA,cACT,KAAK,UAAU;AAAA,gBACd,MAAM;AAAA,gBACN;AAAA,gBACA,UAAU,OAAO;AAAA,gBACjB,QAAQ,OAAO;AAAA,cAChB,CAAC;AAAA,YACF;AAAA,UACD,OAAO;AAEN,sBAAU,KAAK,IAAI;AAAA,UACpB;AAAA,QACD,QAAQ;AAEP,oBAAU,KAAK,IAAI;AAAA,QACpB;AAAA,MACD,OAAO;AAEN,kBAAU,KAAK,IAAI;AAAA,MACpB;AAAA,IACD,CAAC;AAGD,cAAU,iBAAiB,SAAS,MAAM;AACzC,UAAI,MAAM,kBAAkB,IAAI,MAAM,kBAAkB;AACxD,cAAQ;AAAA,QACP,qCAAqC,IAAI,MAAM,eAAe;AAAA,MAC/D;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,SAAS,KAAU;AAClB,aAAO;AAAA,QACN,iBAAiB,IAAI,MAAM;AAAA,QAC3B,cAAc,IAAI,MAAM;AAAA,MACzB;AAAA,IACD;AAAA,EACD;AACD,CAAC;AAEM,IAAM,0BAA0BD,QAAM;AAAA,EAC5C,YAAY,KAAK,WAAW;AAE3B,cAAU,iBAAiB,WAAW,CAACC,YAAe;AACrD,YAAM,OAAOA,QAAM;AACnB,UAAI,gBAAgB,eAAe,gBAAgB,YAAY;AAE9D,cAAM,QAAQ,IAAI,WAAW,IAAI;AACjC,cAAM,WAAW,IAAI,WAAW,MAAM,MAAM;AAC5C,iBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACtC,mBAAS,CAAC,IAAI,MAAM,MAAM,SAAS,IAAI,CAAC;AAAA,QACzC;AACA,kBAAU,KAAK,QAAQ;AAAA,MACxB;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EACA,SAAS,CAAC;AACX,CAAC;;;ACjHD,SAAS,QAAAC,aAAY;AACrB,SAA8B,SAAAC,eAAa;AAEpC,IAAM,kBAAkBA,QAAM;AAAA,EACpC,OAAO;AAAA,IACN,OAAO;AAAA,EACR;AAAA,EACA,YAAY,MAAM;AAEjB,WAAO,EAAE,QAAQ,oBAAoB,EAAE;AAAA,EACxC;AAAA,EACA,WAAW,CAAC,GAAG,YAAY;AAC1B,WAAO,EAAE,KAAK,OAAO,MAAM,SAAS,EAAE,OAAO,EAAE,CAAC;AAAA,EACjD;AAAA,EACA,SAAS;AAAA;AAAA,EAET;AACD,CAAC;AAED,SAAS,sBAAiC;AACzC,QAAMC,OAAM,IAAIF,MAEb;AAEH,EAAAE,KAAI,IAAI,UAAU,CAAC,MAAM;AACxB,UAAM,EAAE,OAAAD,QAAM,IAAI,EAAE;AAEpB,WAAO,EAAE,KAAK;AAAA,MACb,OAAOA,QAAM,MAAM;AAAA,IACpB,CAAC;AAAA,EACF,CAAC;AAED,EAAAC,KAAI,KAAK,cAAc,CAAC,MAAM;AAC7B,UAAM,EAAE,OAAAD,QAAM,IAAI,EAAE;AAEpB,IAAAA,QAAM,MAAM;AACZ,WAAO,EAAE,KAAK;AAAA,MACb,OAAOA,QAAM,MAAM;AAAA,IACpB,CAAC;AAAA,EACF,CAAC;AAED,SAAOC;AACR;;;AC1CA,SAAS,SAAAC,eAAa;AAEf,IAAM,uBAAuBA,QAAM;AAAA,EACzC,OAAO;AAAA,IACN,UAAU,CAAC;AAAA,EAKZ;AAAA,EACA,YAAY,MAAM;AACjB,WAAO;AAAA,MACN,SAAS,oBAAI,IAAS;AAAA,IACvB;AAAA,EACD;AAAA,EACA,YAAY,KAAK,QAAQ;AAExB,QAAI,KAAK,QAAQ,IAAI,MAAM;AAG3B,WAAO;AAAA,MACN,KAAK,UAAU;AAAA,QACd,MAAM;AAAA,QACN,UAAU,IAAI,MAAM;AAAA,MACrB,CAAC;AAAA,IACF;AAGA,WAAO,iBAAiB,WAAW,CAACC,YAAe;AAClD,UAAI;AACH,cAAM,OAAO,KAAK,MAAMA,QAAM,IAAI;AAElC,YAAI,KAAK,SAAS,aAAa,KAAK,MAAM;AACzC,gBAAM,UAAU;AAAA,YACf,IAAI,OAAO,WAAW;AAAA,YACtB,MAAM,KAAK;AAAA,YACX,WAAW,KAAK,IAAI;AAAA,UACrB;AAGA,cAAI,MAAM,SAAS,KAAK,OAAO;AAC/B,cAAI,UAAU,CAAC,CAAC;AAGhB,cAAI,IAAI,MAAM,SAAS,SAAS,IAAI;AACnC,gBAAI,MAAM,SAAS,MAAM;AAAA,UAC1B;AAGA,gBAAM,YAAY,KAAK,UAAU;AAAA,YAChC,MAAM;AAAA,YACN,GAAG;AAAA,UACJ,CAAC;AAED,qBAAW,MAAM,IAAI,KAAK,SAAS;AAClC,gBAAI,GAAG,eAAe,GAAG;AAExB,iBAAG,KAAK,SAAS;AAAA,YAClB;AAAA,UACD;AAAA,QACD;AAAA,MACD,SAAS,GAAG;AACX,gBAAQ,MAAM,8BAA8B,CAAC;AAAA,MAC9C;AAAA,IACD,CAAC;AAGD,WAAO,iBAAiB,SAAS,MAAM;AACtC,UAAI,KAAK,QAAQ,OAAO,MAAM;AAAA,IAC/B,CAAC;AAAA,EACF;AAAA,EACA,SAAS,CAAC;AACX,CAAC;;;ACxED,SAAS,SAAAC,eAA8D;AAEvE,SAAS,MAAM,IAA2B;AACzC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACxD;AAEO,IAAM,8BAA8BA,QAAM;AAAA,EAChD,SAAS;AAAA,IACR,uBAAuB;AAAA,IACvB,kBAAkB;AAAA,EACnB;AAAA,EACA,OAAO;AAAA,IACN,iBAAiB;AAAA,IACjB,YAAY;AAAA,IACZ,gBAAgB;AAAA,IAChB,mBAAmB;AAAA,EACpB;AAAA,EACA,MAAM,QAAQ,GAAG;AAChB,UAAM,UAAU,KAAK,KAAK,MAAM,KAAK,OAAO,IAAI,IAAK;AACrD,MAAE,MAAM,cAAc;AACtB,MAAE,IAAI,KAAK;AAAA,MACV,KAAK;AAAA,MACL;AAAA,MACA,YAAY,EAAE,MAAM;AAAA,IACrB,CAAC;AACD,UAAM,MAAM,OAAO;AAAA,EACpB;AAAA,EACA,YAAY,GAAG,WAA+B;AAC7C,MAAE,MAAM,mBAAmB;AAC3B,UAAM,eAAe,OAAO,WAAW;AACvC,QAAI,QAAQ;AAEZ,UAAM,WAAW,MAAM;AACtB,UAAI,UAAU,eAAe,EAAG;AAEhC,YAAM,YAAY,KAAK,IAAI;AAC3B,YAAM,UAAU;AAAA,QACf,MAAM;AAAA,QACN;AAAA,QACA;AAAA,QACA;AAAA,QACA,KAAK,IAAI,KAAK,SAAS,EAAE,YAAY;AAAA,QACrC,gBAAgB,EAAE,MAAM;AAAA,MACzB;AAEA,QAAE,MAAM,kBAAkB;AAC1B,eAAS;AACT,gBAAU,KAAK,KAAK,UAAU,OAAO,CAAC;AAAA,IACvC;AAEA,MAAE,IAAI,KAAK;AAAA,MACV,KAAK;AAAA,MACL;AAAA,MACA,iBAAiB,EAAE,MAAM;AAAA,IAC1B,CAAC;AAED,aAAS;AACT,UAAM,WAAW,YAAY,UAAU,GAAK;AAE5C,cAAU,iBAAiB,WAAW,CAACC,YAA6B;AACnE,QAAE,MAAM,qBAAqB;AAC7B,QAAE,IAAI,KAAK;AAAA,QACV,KAAK;AAAA,QACL;AAAA,QACA,mBAAmB,EAAE,MAAM;AAAA,MAC5B,CAAC;AACD,gBAAU;AAAA,QACT,KAAK,UAAU;AAAA,UACd,MAAM;AAAA,UACN;AAAA,UACA;AAAA,UACA,WAAW,KAAK,IAAI;AAAA,UACpB,UAAUA,QAAM;AAAA,QACjB,CAAC;AAAA,MACF;AAAA,IACD,CAAC;AAED,cAAU,iBAAiB,SAAS,MAAM;AACzC,oBAAc,QAAQ;AACtB,QAAE,MAAM,mBAAmB;AAC3B,QAAE,IAAI,KAAK;AAAA,QACV,KAAK;AAAA,QACL;AAAA,QACA,iBAAiB,EAAE,MAAM;AAAA,MAC1B,CAAC;AAAA,IACF,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,SAAS,GAAG;AACX,aAAO;AAAA,QACN,iBAAiB,EAAE,MAAM;AAAA,QACzB,YAAY,EAAE,MAAM;AAAA,QACpB,gBAAgB,EAAE,MAAM;AAAA,QACxB,mBAAmB,EAAE,MAAM;AAAA,MAC5B;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;ACjGD,SAAS,SAAAC,eAA8D;AAEhE,IAAM,eAAeA,QAAM;AAAA,EACjC,SAAS;AAAA,IACR,uBAAuB;AAAA,IACvB,kBAAkB;AAAA,EACnB;AAAA,EACA,OAAO;AAAA,IACN,iBAAiB;AAAA,IACjB,cAAc;AAAA,IACd,gBAAgB;AAAA,EACjB;AAAA,EACA,YAAY,GAAG,WAA+B;AAC7C,MAAE,MAAM,mBAAmB;AAC3B,UAAM,eAAe,OAAO,WAAW;AAEvC,UAAM,gBAAgB,MAAM;AAC3B,UAAI,UAAU,eAAe,EAAG;AAEhC,QAAE,MAAM,kBAAkB;AAC1B,gBAAU;AAAA,QACT,KAAK,UAAU;AAAA,UACd,MAAM;AAAA,UACN;AAAA,UACA,gBAAgB,EAAE,MAAM;AAAA,UACxB,WAAW,KAAK,IAAI;AAAA,QACrB,CAAC;AAAA,MACF;AAAA,IACD;AAEA,UAAM,YAAY,YAAY,eAAe,GAAK;AAClD,kBAAc;AAEd,cAAU,iBAAiB,WAAW,OAAOC,YAA6B;AAIzE,UAAI,OAAOA,QAAM,SAAS,UAAU;AACnC,YAAI;AACJ,YAAI;AACH,mBAAS,KAAK,MAAMA,QAAM,IAAI;AAAA,QAC/B,QAAQ;AACP,mBAAS;AAAA,QACV;AACA,YACC,UACA,OAAO,WAAW,YACjB,OAA8B,SAAS,QACvC;AACD,gBAAM,KAAM,OAA4B;AACxC,cAAI,UAAU,eAAe,GAAG;AAC/B,sBAAU;AAAA,cACT,KAAK,UAAU;AAAA,gBACd,MAAM;AAAA,gBACN;AAAA,gBACA;AAAA,gBACA,WAAW,KAAK,IAAI;AAAA,cACrB,CAAC;AAAA,YACF;AAAA,UACD;AACA;AAAA,QACD;AAAA,MACD;AAEA,QAAE,MAAM,gBAAgB;AACxB,YAAM,EAAE,GAAG,IAAI,WAAW,OAAO,EAAE,MAAM,YAAY,CAAC;AACtD,gBAAU;AAAA,QACT,KAAK,UAAU;AAAA,UACd,MAAM;AAAA,UACN;AAAA,UACA,cAAc,EAAE,MAAM;AAAA,UACtB,WAAW,KAAK,IAAI;AAAA,UACpB,UAAUA,QAAM;AAAA,QACjB,CAAC;AAAA,MACF;AAAA,IACD,CAAC;AAED,cAAU,iBAAiB,SAAS,MAAM;AACzC,oBAAc,SAAS;AACvB,QAAE,MAAM,mBAAmB;AAAA,IAC5B,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,SAAS,GAAG;AACX,aAAO;AAAA,QACN,iBAAiB,EAAE,MAAM;AAAA,QACzB,cAAc,EAAE,MAAM;AAAA,QACtB,gBAAgB,EAAE,MAAM;AAAA,MACzB;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;AC3FD,SAAS,SAAAC,SAAc,aAAa;AAG7B,IAAM,oBAAoB;AAG1B,IAAM,eAAeA,QAAM;AAAA,EACjC,OAAO;AAAA,IACN,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ,WAAW;AAAA,EACZ;AAAA,EACA,KAAK,OAAO,MAAM;AACjB,MAAE,MAAM,aAAa;AACrB,MAAE,IAAI,KAAK,qBAAqB;AAEhC,WAAO,CAAC,EAAE,SAAS;AAClB,QAAE,MAAM,aAAa;AACrB,QAAE,MAAM,aAAa,KAAK,IAAI;AAC9B,QAAE,IAAI,KAAK,EAAE,KAAK,QAAQ,WAAW,EAAE,MAAM,UAAU,CAAC;AAGxD,YAAM,IAAI,QAAc,CAAC,YAAY;AACpC,cAAM,UAAU,WAAW,SAAS,EAAE;AACtC,UAAE,YAAY;AAAA,UACb;AAAA,UACA,MAAM;AACL,yBAAa,OAAO;AACpB,oBAAQ;AAAA,UACT;AAAA,UACA,EAAE,MAAM,KAAK;AAAA,QACd;AAAA,MACD,CAAC;AAAA,IACF;AAEA,MAAE,MAAM,YAAY;AACpB,MAAE,IAAI,KAAK,gCAAgC;AAAA,EAC5C;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,OAAO;AAAA,MACjB,WAAW,EAAE,MAAM;AAAA,MACnB,YAAY,EAAE,MAAM;AAAA,MACpB,YAAY,EAAE,MAAM;AAAA,MACpB,WAAW,EAAE,MAAM;AAAA,IACpB;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAGM,IAAM,uBAAuBA,QAAM;AAAA,EACzC,OAAO;AAAA,IACN,kBAAkB,CAAC;AAAA,IACnB,YAAY;AAAA,EACb;AAAA,EACA,QAAQ;AAAA,IACP,UAAU,MAAe;AAAA,EAC1B;AAAA,EACA,KAAK,OAAO,MAAM;AACjB,MAAE,MAAM,aAAa;AACrB,MAAE,IAAI,KAAK,2CAA2C;AAEtD,qBAAiB,WAAW,EAAE,MAAM,KAAK,GAAG;AAC3C,QAAE,IAAI,KAAK,EAAE,KAAK,oBAAoB,MAAM,QAAQ,KAAK,CAAC;AAC1D,QAAE,MAAM,iBAAiB,KAAK;AAAA,QAC7B,MAAM,QAAQ;AAAA,QACd,MAAM,QAAQ;AAAA,MACf,CAAC;AAAA,IACF;AAEA,MAAE,IAAI,KAAK,gCAAgC;AAAA,EAC5C;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,OAAO;AAAA,MACjB,kBAAkB,EAAE,MAAM;AAAA,MAC1B,YAAY,EAAE,MAAM;AAAA,IACrB;AAAA,IACA,aAAa,OAAO,GAAG,SAAkB;AACxC,YAAM,SAAS,EAAE,OAAwB;AACzC,YAAM,SAAS,OAAO,qBAAqB,SAAS,EAAE,OAAO;AAC7D,YAAM,OAAO,KAAK,YAAY,IAAI;AAClC,aAAO;AAAA,IACR;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAGM,IAAM,mBAAmBA,QAAM;AAAA,EACrC,OAAO;AAAA,IACN,YAAY;AAAA,IACZ,eAAe;AAAA,EAChB;AAAA,EACA,KAAK,OAAO,MAAM;AACjB,MAAE,MAAM,aAAa;AACrB,MAAE,IAAI,KAAK,4CAA4C;AAEvD,UAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,GAAG,CAAC;AACvD,MAAE,IAAI,KAAK,2BAA2B;AAAA,EAEvC;AAAA,EACA,WAAW,CAAC,MAAM;AACjB,MAAE,MAAM,gBAAgB;AAAA,EACzB;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,OAAO;AAAA,MACjB,YAAY,EAAE,MAAM;AAAA,MACpB,eAAe,EAAE,MAAM;AAAA,IACxB;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAGM,IAAM,eAAeA,QAAM;AAAA,EACjC,OAAO;AAAA,IACN,YAAY;AAAA,IACZ,eAAe;AAAA,EAChB;AAAA,EACA,KAAK,OAAO,MAAM;AACjB,MAAE,MAAM,aAAa;AACrB,MAAE,IAAI,KAAK,uCAAuC;AAClD,UAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACtD,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACnD;AAAA,EACA,WAAW,CAAC,MAAM;AACjB,MAAE,MAAM,gBAAgB;AAAA,EACzB;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,OAAO;AAAA,MACjB,YAAY,EAAE,MAAM;AAAA,MACpB,eAAe,EAAE,MAAM;AAAA,IACxB;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAGM,IAAM,oBAAoBA,QAAM;AAAA,EACtC,OAAO;AAAA,IACN,WAAW;AAAA,EACZ;AAAA,EACA,QAAQ,CAAC,MAAM;AACd,MAAE,MAAM,aAAa;AAAA,EACtB;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,OAAO;AAAA,MACjB,WAAW,EAAE,MAAM;AAAA,IACpB;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;;;AClKD,SAAS,SAAAC,SAAO,SAAAC,cAAsC;AACtD,SAAS,4BAA4B;AAE9B,IAAM,gBAAgB;AAEtB,IAAMC,SAAQF,QAAM;AAAA,EAC1B,OAAO,EAAE,YAAY,GAAG,YAAY,EAAE;AAAA,EACtC,QAAQ,CAAC,MAAM;AACd,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,SAAS,CAAC,MAAM;AACf,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,SAAS;AAAA,IACR,cAAc,CAAC,MAAM;AACpB,QAAE,MAAM;AAAA,IACT;AAAA,IACA,WAAW,CAAC,MAAM;AACjB,aAAO;AAAA,QACN,YAAY,EAAE,MAAM;AAAA,QACpB,YAAY,EAAE,MAAM;AAAA,MACrB;AAAA,IACD;AAAA,IACA,UAAU,OAAO,GAAG,aAAqB;AACxC,YAAM,EAAE,SAAS,MAAM,UAAU,SAAS;AAAA,IAC3C;AAAA,IACA,SAAS,CAAC,MAAM;AACf,QAAE,IAAI,KAAK,cAAc;AAAA,IAC1B;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAEM,IAAM,mBAAmBA,QAAM;AAAA,EACrC,OAAO,EAAE,YAAY,GAAG,YAAY,EAAE;AAAA,EACtC,YAAY,OACV,CAAC;AAAA,EACH,QAAQ;AAAA,IACP,SAASC,OAAU;AAAA,EACpB;AAAA,EACA,QAAQ,CAAC,MAAM;AACd,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,SAAS,CAAC,MAAM;AACf,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,SAAS;AAAA,IACR,WAAW,CAAC,MAAM;AACjB,aAAO;AAAA,QACN,YAAY,EAAE,MAAM;AAAA,QACpB,YAAY,EAAE,MAAM;AAAA,MACrB;AAAA,IACD;AAAA,IACA,gBAAgB,OAAO,MAAM;AAC5B,QAAE,IAAI,KAAK,2BAA2B;AACtC,QAAE,KAAK,qBAAqB,qBAAqB,MAAM;AAAA,MAAC,CAAC;AACzD,QAAE,UAAU,SAAS;AACrB,YAAM,EAAE,KAAK,mBAAmB;AAChC,QAAE,IAAI,KAAK,2BAA2B;AAAA,IACvC;AAAA,IACA,sBAAsB,CAAC,MAAM,EAAE,KAAK,oBAAoB,QAAQ;AAAA,EACjE;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAEM,IAAM,mBAAmBD,QAAM;AAAA,EACrC,OAAO,EAAE,YAAY,GAAG,YAAY,GAAG,cAAc,EAAE;AAAA,EACvD,QAAQ,CAAC,MAAM;AACd,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,SAAS,CAAC,MAAM;AACf,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,WAAW,OAAO,GAAG,YAAY;AAChC,MAAE,MAAM,gBAAgB;AACxB,UAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAE/B,QAAI,IAAI,aAAa,iBAAiB;AACrC,YAAM,WAAW;AAAA,QAChB,IAAI,aAAa,IAAI,UAAU,KAAK;AAAA,MACrC;AACA,QAAE,IAAI,KAAK,EAAE,KAAK,+BAA+B,SAAS,CAAC;AAC3D,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,QAAQ,CAAC;AAC5D,QAAE,IAAI,KAAK,6BAA6B;AACxC,aAAO,IAAI,SAAS,KAAK,UAAU,EAAE,WAAW,KAAK,CAAC,GAAG;AAAA,QACxD,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC/C,CAAC;AAAA,IACF;AAEA,WAAO,IAAI,SAAS,aAAa,EAAE,QAAQ,IAAI,CAAC;AAAA,EACjD;AAAA,EACA,SAAS;AAAA,IACR,WAAW,CAAC,MAAM;AACjB,aAAO;AAAA,QACN,YAAY,EAAE,MAAM;AAAA,QACpB,YAAY,EAAE,MAAM;AAAA,QACpB,cAAc,EAAE,MAAM;AAAA,MACvB;AAAA,IACD;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAEM,IAAM,wBAAwBA,QAAM;AAAA,EAC1C,OAAO,EAAE,YAAY,GAAG,YAAY,GAAG,iBAAiB,EAAE;AAAA,EAC1D,QAAQ,CAAC,MAAM;AACd,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,SAAS,CAAC,MAAM;AACf,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,aAAa,CAAC,GAAG,cAAkC;AAClD,MAAE,MAAM,mBAAmB;AAC3B,MAAE,IAAI,KAAK;AAAA,MACV,KAAK;AAAA,MACL,iBAAiB,EAAE,MAAM;AAAA,IAC1B,CAAC;AAED,cAAU;AAAA,MACT,KAAK,UAAU;AAAA,QACd,MAAM;AAAA,QACN,iBAAiB,EAAE,MAAM;AAAA,MAC1B,CAAC;AAAA,IACF;AAEA,cAAU,iBAAiB,WAAW,CAACC,YAAe;AACrD,YAAM,OAAOA,QAAM;AACnB,UAAI,OAAO,SAAS,UAAU;AAC7B,YAAI;AACH,gBAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,cAAI,OAAO,SAAS,aAAa;AAChC,sBAAU;AAAA,cACT,KAAK,UAAU;AAAA,gBACd,MAAM;AAAA,gBACN,YAAY,EAAE,MAAM;AAAA,gBACpB,YAAY,EAAE,MAAM;AAAA,gBACpB,iBAAiB,EAAE,MAAM;AAAA,cAC1B,CAAC;AAAA,YACF;AAAA,UACD,WAAW,OAAO,SAAS,aAAa;AAEvC,sBAAU,KAAK,KAAK,UAAU,EAAE,MAAM,MAAM,CAAC,CAAC;AAAA,UAC/C;AAAA,QACD,QAAQ;AAEP,oBAAU,KAAK,IAAI;AAAA,QACpB;AAAA,MACD;AAAA,IACD,CAAC;AAED,cAAU,iBAAiB,SAAS,MAAM;AACzC,QAAE,MAAM,mBAAmB;AAC3B,QAAE,IAAI,KAAK;AAAA,QACV,KAAK;AAAA,QACL,iBAAiB,EAAE,MAAM;AAAA,MAC1B,CAAC;AAAA,IACF,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,WAAW,CAAC,MAAM;AACjB,aAAO;AAAA,QACN,YAAY,EAAE,MAAM;AAAA,QACpB,YAAY,EAAE,MAAM;AAAA,QACpB,iBAAiB,EAAE,MAAM;AAAA,MAC1B;AAAA,IACD;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAEM,IAAM,yBAAyBD,QAAM;AAAA,EAC3C,OAAO,EAAE,YAAY,GAAG,YAAY,EAAE;AAAA,EACtC,QAAQ,CAAC,MAAM;AACd,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,SAAS,CAAC,MAAM;AACf,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,SAAS;AAAA,IACR,WAAW,CAAC,MAAM;AACjB,aAAO;AAAA,QACN,YAAY,EAAE,MAAM;AAAA,QACpB,YAAY,EAAE,MAAM;AAAA,MACrB;AAAA,IACD;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,IACd,SAAS;AAAA,EACV;AACD,CAAC;;;ACtMD,SAAS,SAAAG,SAAO,SAAAC,cAAa;AAEtB,IAAM,YAAYD,QAAM;AAAA,EAC9B,OAAO;AAAA,IACN,SAAS;AAAA,IACT,gBAAgB;AAAA,IAChB,aAAa,CAAC;AAAA,EACf;AAAA,EACA,QAAQ;AAAA,IACP,WAAWC,OAAuC;AAAA,IAClD,iBAAiBA,OAAuD;AAAA,EACzE;AAAA,EACA,SAAS;AAAA;AAAA,IAER,gBAAgB,CAAC,GAAG,cAAsB;AACzC,QAAE,SAAS,GAAG,WAAW,iBAAiB;AAC1C,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,mBAAmB,CAAC,GAAG,YAAoB;AAC1C,QAAE,SAAS,MAAM,SAAS,iBAAiB;AAC3C,aAAO,KAAK,IAAI,IAAI;AAAA,IACrB;AAAA;AAAA,IAGA,yBAAyB,CAAC,GAAG,QAAgB,YAAoB;AAChE,QAAE,SAAS,MAAM,SAAS,yBAAyB,MAAM;AACzD,aAAO,EAAE,QAAQ,cAAc,KAAK,IAAI,IAAI,QAAQ;AAAA,IACrD;AAAA;AAAA,IAGA,cAAc,CAAC,GAAG,YAAoB;AACrC,YAAM,YAAY,KAAK,IAAI,IAAI;AAC/B,QAAE,SAAS,GAAG,WAAW,iBAAiB;AAC1C,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,YAAY,CAAC,MAAM;AAClB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IAEA,mBAAmB,CAAC,MAAM;AACzB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IAEA,gBAAgB,CAAC,MAAM;AACtB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IAEA,cAAc,CAAC,MAAM;AACpB,QAAE,MAAM,cAAc,CAAC;AACvB,QAAE,MAAM,iBAAiB;AACzB,QAAE,MAAM,UAAU;AAClB,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,iBAAiB,CAAC,MAAM;AACvB,QAAE,MAAM,UAAU,KAAK,IAAI;AAC3B,QAAE,MAAM;AACR,QAAE,UAAU,aAAa;AAAA,QACxB,MAAM,EAAE,MAAM;AAAA,QACd,OAAO,EAAE,MAAM;AAAA,MAChB,CAAC;AAAA,IACF;AAAA,IAEA,uBAAuB,CAAC,GAAG,WAAmB;AAC7C,QAAE,MAAM,UAAU,KAAK,IAAI;AAC3B,QAAE,MAAM;AACR,QAAE,MAAM,YAAY,KAAK,MAAM;AAC/B,QAAE,UAAU,mBAAmB;AAAA,QAC9B;AAAA,QACA,MAAM,EAAE,MAAM;AAAA,QACd,OAAO,EAAE,MAAM;AAAA,MAChB,CAAC;AAAA,IACF;AAAA,EACD;AACD,CAAC;;;AC/ED,SAAS,SAAAC,eAAa;AAGf,IAAM,kBAAkBA,QAAM;AAAA,EACpC,OAAO,EAAE,iBAAiB,CAAC,EAAc;AAAA,EACzC,SAAS;AAAA,IACR,iBAAiB,CAAC,GAAG,aAAqB;AACzC,QAAE,MAAM,gBAAgB,KAAK,QAAQ;AAAA,IACtC;AAAA,IACA,cAAc,CAAC,GAAG,aAAqB;AACtC,aAAO,EAAE,MAAM,gBAAgB,SAAS,QAAQ;AAAA,IACjD;AAAA,IACA,OAAO,CAAC,MAAM;AACb,QAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5B;AAAA,EACD;AACD,CAAC;AAEM,IAAM,eAAeA,QAAM;AAAA,EACjC,OAAO,EAAE,OAAO,GAAG,KAAK,GAAG;AAAA,EAC3B,QAAQ,CAAC,MAAM;AAEd,MAAE,MAAM,MAAM,EAAE,IAAI,KAAK,GAAG;AAAA,EAC7B;AAAA,EACA,WAAW,OAAO,MAAM;AACvB,UAAM,SAAS,EAAE,OAAwB;AACzC,UAAM,WAAW,OAAO,gBAAgB,YAAY,CAAC,UAAU,CAAC;AAChE,UAAM,SAAS,gBAAgB,EAAE,MAAM,GAAG;AAAA,EAC3C;AAAA,EACA,SAAS;AAAA,IACR,UAAU,OAAO,GAAG,aAAqB;AACxC,QAAE,MAAM,QAAQ;AAChB,YAAM,EAAE,UAAU,EAAE,WAAW,KAAK,CAAC;AACrC,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,UAAU,CAAC,MAAM;AAChB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,SAAS,CAAC,MAAM;AACf,QAAE,QAAQ;AAAA,IACX;AAAA,EACD;AACD,CAAC;;;AC1CD,SAAS,SAAAC,eAAa;AAEf,IAAM,4BAA4B;AAQlC,IAAM,mBAAmBA,QAAM;AAAA,EACrC,OAAO;AAAA,IACN,YAAY;AAAA,IACZ,WAAW;AAAA,EACZ;AAAA,EACA,iBAAiB,CAAC,MAA4B;AAC7C,WAAO;AAAA,MACN,OAAO;AAAA,MACP,cAAc;AAAA,MACd,iBAAiB;AAAA,IAClB;AAAA,EACD;AAAA,EACA,QAAQ,CAAC,MAAM;AACd,MAAE,MAAM,aAAa;AAAA,EACtB;AAAA,EACA,SAAS,CAAC,MAAM;AACf,MAAE,MAAM,cAAc;AAAA,EACvB;AAAA,EACA,WAAW,CAAC,GAAG,SAAS;AACvB,SAAK,MAAM,gBAAgB;AAAA,EAC5B;AAAA,EACA,cAAc,CAAC,GAAG,SAAS;AAC1B,SAAK,MAAM,mBAAmB;AAAA,EAC/B;AAAA,EACA,SAAS;AAAA;AAAA,IAER,MAAM,CAAC,MAAM;AACZ,aAAO;AAAA,IACR;AAAA;AAAA,IAEA,eAAe,CAAC,MAAM;AACrB,QAAE,KAAK,MAAM,SAAS;AACtB,aAAO,EAAE,KAAK,MAAM;AAAA,IACrB;AAAA;AAAA,IAEA,cAAc,CAAC,MAAM;AACpB,aAAO,EAAE,KAAK,MAAM;AAAA,IACrB;AAAA;AAAA,IAEA,wBAAwB,CAAC,MAAM;AAC9B,aAAO;AAAA,QACN,cAAc,EAAE,KAAK,MAAM;AAAA,QAC3B,iBAAiB,EAAE,KAAK,MAAM;AAAA,MAC/B;AAAA,IACD;AAAA;AAAA,IAEA,kBAAkB,CAAC,MAAM;AACxB,aAAO,EAAE,MACP,QAAQ,EACR,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,EACf,QAAQ;AAAA,IACX;AAAA;AAAA,IAEA,gBAAgB,CAAC,MAAM;AACtB,aAAO;AAAA,QACN,YAAY,EAAE,MAAM;AAAA,QACpB,WAAW,EAAE,MAAM;AAAA,MACpB;AAAA,IACD;AAAA;AAAA,IAEA,cAAc,CAAC,MAAM;AACpB,QAAE,MAAM;AAAA,IACT;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;;;AC7ED,SAAS,SAAAC,SAAO,SAAAC,QAAO,SAAAC,cAAa;AAa7B,IAAM,SAASF,QAAM;AAAA,EAC3B,OAAO;AAAA,IACN,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,SAAS;AAAA,EACV;AAAA,EACA,QAAQ;AAAA,IACP,eAAeC,OAAyD;AAAA,IACxE,cAAcA,OAA6C;AAAA,EAC5D;AAAA,EACA,QAAQ;AAAA,IACP,MAAMC,OAAiB;AAAA,EACxB;AAAA,EACA,MAAM,IAAI,GAAG;AACZ,MAAE,MAAM,SAAS;AACjB,MAAE,UAAU,iBAAiB;AAAA,MAC5B,QAAQ,EAAE,MAAM;AAAA,MAChB,WAAW,EAAE,MAAM;AAAA,IACpB,CAAC;AAED,qBAAiB,OAAO,EAAE,MAAM,KAAK,GAAG;AACvC,QAAE,MAAM,aAAa;AACrB,QAAE,MAAM,UAAU,IAAI;AACtB,QAAE,UAAU,gBAAgB;AAAA,QAC3B,WAAW,EAAE,MAAM;AAAA,QACnB,KAAK,IAAI;AAAA,MACV,CAAC;AAAA,IACF;AAEA,MAAE,MAAM,SAAS;AAAA,EAClB;AAAA,EACA,SAAS;AAAA,IACR,SAAS,GAAgB;AACxB,aAAO;AAAA,QACN,QAAQ,EAAE,MAAM;AAAA,QAChB,WAAW,EAAE,MAAM;AAAA,QACnB,SAAS,EAAE,MAAM;AAAA,MAClB;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;ACrDD,SAAS,SAAAC,SAAO,SAAAC,SAAO,SAAAC,cAAa;AAEpC,IAAM,qBAAqB;AAgBpB,IAAM,gBAAgBF,QAAM;AAAA,EAClC,OAAO;AAAA,IACN,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,OAAO;AAAA,IACP,YAAY;AAAA,IACZ,SAAS;AAAA,IACT,WAAW;AAAA,EACZ;AAAA,EACA,QAAQ;AAAA,IACP,MAAMC,QAAqC;AAAA,IAC3C,cAAcA,QAAoD;AAAA,EACnE;AAAA,EACA,QAAQ;AAAA,IACP,MAAMC,OAAwB;AAAA,EAC/B;AAAA,EACA,KAAK,OAAO,MAAM;AACjB,MAAE,MAAM,SAAS;AAEjB,WAAO,CAAC,EAAE,SAAS;AAClB,YAAM,UAAU,MAAM,EAAE,MAAM,KAAK;AAAA,QAClC,OAAO,CAAC,MAAM;AAAA,QACd,SAAS,EAAE,MAAM;AAAA,MAClB,CAAC;AAED,UAAI,CAAC,SAAS;AACb,cAAM,KAAK,KAAK,IAAI;AACpB,UAAE,MAAM,SAAS;AACjB,UAAE,MAAM,aAAa;AACrB,UAAE,UAAU,QAAQ;AAAA,UACnB,OAAO,EAAE,MAAM;AAAA,UACf;AAAA,QACD,CAAC;AACD;AAAA,MACD;AAEA,QAAE,MAAM,aAAa;AACrB,QAAE,MAAM,UAAU,QAAQ;AAC1B,QAAE,UAAU,gBAAgB;AAAA,QAC3B,WAAW,EAAE,MAAM;AAAA,QACnB,KAAK,QAAQ;AAAA,MACd,CAAC;AAAA,IACF;AAEA,MAAE,MAAM,SAAS;AAAA,EAClB;AAAA,EACA,SAAS;AAAA,IACR,YAAY,OAAO,GAAGC,aAAoB;AACzC,YAAM,MAAM;AAAA,QACX,IAAI,OAAO,WAAW;AAAA,QACtB,SAAAA;AAAA,MACD;AACA,YAAM,EAAE,MAAM,KAAK,QAAQ,GAAG;AAC9B,aAAO;AAAA,IACR;AAAA,IACA,cAAc,CAAC,GAAG,cAAsB;AACvC,QAAE,MAAM,YAAY,KAAK,IAAI,KAAK,KAAK,MAAM,SAAS,CAAC;AACvD,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,UAAU,CAAC,MAA0B,EAAE;AAAA,EACxC;AACD,CAAC;;;AC/ED,SAAS,SAAAC,SAAO,SAAAC,SAAO,SAAAC,cAAa;AACpC,SAAS,MAAkC,gBAAgB;AAE3D,IAAM,wBAAwB;AAE9B,IAAM,sBAAsB;AAC5B,IAAM,8BAA8B;AAE7B,IAAM,uBAAuBF,QAAM;AAAA,EACzC,OAAO;AAAA,IACN,UAAU;AAAA,IACV,gBAAgB;AAAA,IAChB,SAAS,CAAC;AAAA,EACX;AAAA,EACA,KAAK,SAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,WAAW,OAAO,YAAY;AAC3C,UAAI;AAGH,gBAAQ;AAAA,MACT,QAAQ;AAAA,MAAC;AAET,YAAM,QAAQ,KAAK,aAAa,YAAY;AAC3C,iCAAyB,OAAO;AAAA,MACjC,CAAC;AAED,YAAM,QAAQ,MAAM,QAAQ,EAAE;AAC9B,aAAO,KAAK,SAAS,MAAS;AAAA,IAC/B,CAAC;AAAA,EACH,CAAC;AAAA,EACD,SAAS;AAAA,IACR,UAAU,OAAO,MAAM;AACtB,YAAM,YAAY,MAAM,EAAE,GAAG,IAAI,qBAAqB;AACtD,UAAI,cAAc,QAAQ;AACzB,UAAE,MAAM,iBAAiB;AAAA,MAC1B;AACA,aAAO,EAAE;AAAA,IACV;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAEM,IAAM,qBAAqBA,QAAM;AAAA,EACvC,OAAO;AAAA,IACN,UAAU,CAAC;AAAA,EACZ;AAAA,EACA,QAAQ;AAAA,IACP,CAAC,mBAAmB,GAAGE,OAAkC;AAAA,EAC1D;AAAA,EACA,KAAK,SAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,SAAS,OAAO,YAAY;AACzC,YAAM,UAAU,MAAM,QAAQ,MAAM,KAAK,cAAc;AAAA,QACtD,OAAO,CAAC,mBAAmB;AAAA,QAC3B,aAAa;AAAA,MACd,CAAC;AACD,UAAI,CAAC,QAAQ,UAAU;AACtB,eAAO,KAAK,SAAS,MAAS;AAAA,MAC/B;AACA,YAAM,WAAW,QAAQ;AACzB,YAAM,QAAQ,KAAK,iBAAiB,YAAY;AAC/C,cAAM,0BAA0B,SAAS,QAAQ,MAAM,QAAQ;AAAA,MAChE,CAAC;AACD,aAAO,KAAK,SAAS,MAAS;AAAA,IAC/B,CAAC;AAAA,EACH,CAAC;AAAA,EACD,SAAS;AAAA,IACR,aAAa,CAAC,MAAM,EAAE,MAAM;AAAA,EAC7B;AACD,CAAC;AAEM,IAAM,qBAAqBF,QAAM;AAAA,EACvC,OAAO;AAAA,IACN,OAAO;AAAA,EACR;AAAA,EACA,KAAK,SAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,SAAS,OAAO,YAAY;AACzC,YAAM,QAAQ,KAAK,QAAQ,YAAY;AACtC,mCAA2B,OAAO;AAAA,MACnC,CAAC;AACD,YAAM,QAAQ,MAAM,SAAS,EAAE;AAC/B,aAAO,KAAK,SAAS,MAAS;AAAA,IAC/B,CAAC;AAAA,EACH,CAAC;AAAA,EACD,SAAS;AAAA,IACR,UAAU,CAAC,MAAM,EAAE;AAAA,EACpB;AAAA,EACA,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;AAEM,IAAM,4BAA4BA,QAAM;AAAA,EAC9C,OAAO;AAAA,IACN,WAAW;AAAA,IACX,OAAO;AAAA,IACP,YAAY;AAAA,IACZ,SAAS;AAAA,IACT,WAAW;AAAA,EACZ;AAAA,EACA,QAAQ;AAAA,IACP,MAAMC,QAAqC;AAAA,IAC3C,cAAcA,QAAmE;AAAA,EAClF;AAAA,EACA,QAAQ;AAAA,IACP,CAAC,2BAA2B,GAAGC,OAAuC;AAAA,EACvE;AAAA,EACA,KAAK,SAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,sBAAsB,OAAO,YAAY;AACtD,YAAM,YAAY,MAAM,QAAQ,KAAK,gBAAgB,YAAY;AAChE,eAAO,sBAAsB,OAAO;AAAA,MACrC,CAAC;AAED,YAAM,CAAC,OAAO,IAAI,MAAM,QAAQ,MAAM,UAAU,uBAAuB;AAAA,QACtE,OAAO,CAAC,2BAA2B;AAAA,QACnC,SAAS;AAAA,MACV,CAAC;AAED,UAAI,CAAC,SAAS;AACb,cAAM,QAAQ,KAAK,QAAQ,YAAY;AACtC,oCAA0B,OAAO;AAAA,QAClC,CAAC;AACD,eAAO,KAAK,SAAS,MAAS;AAAA,MAC/B;AAEA,YAAM,QAAQ,KAAK,eAAe,YAAY;AAC7C,kCAA0B,SAAS,QAAQ,IAAI;AAAA,MAChD,CAAC;AACD,aAAO,KAAK,SAAS,MAAS;AAAA,IAC/B,CAAC;AAAA,EACH,CAAC;AAAA,EACD,SAAS;AAAA,IACR,YAAY,OAAO,GAAGC,aAAoB;AACzC,YAAM,MAAM,EAAE,IAAI,OAAO,WAAW,GAAG,SAAAA,SAAQ;AAC/C,YAAM,EAAE,MAAM,KAAK,6BAA6B,GAAG;AACnD,aAAO;AAAA,IACR;AAAA,IACA,cAAc,CAAC,GAAG,cAAsB;AACvC,QAAE,MAAM,YAAY,KAAK,IAAI,KAAK,KAAK,MAAM,SAAS,CAAC;AACvD,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,UAAU,CAAC,MAAM,EAAE;AAAA,EACpB;AACD,CAAC;AAED,SAAS,yBACR,KACO;AACP,MAAI,MAAM,YAAY;AACtB,MAAI,MAAM,QAAQ,KAAK,IAAI,MAAM,QAAQ;AAC1C;AAEA,eAAe,0BACd,KACA,MACA,UACgB;AAChB,MAAI,MAAM,SAAS,KAAK,IAAI;AAC5B,QAAM,SAAS,EAAE,MAAM,KAAK,CAAC;AAC9B;AAEA,SAAS,2BACR,KACO;AACP,MAAI,MAAM,SAAS;AACpB;AAEA,SAAS,sBACR,KACS;AACT,SAAO,IAAI,MAAM;AAClB;AAEA,SAAS,0BACR,KACO;AACP,QAAM,KAAK,KAAK,IAAI;AACpB,MAAI,MAAM,SAAS;AACnB,MAAI,MAAM,aAAa;AACvB,MAAI,UAAU,QAAQ;AAAA,IACrB,OAAO,IAAI,MAAM;AAAA,IACjB;AAAA,EACD,CAAC;AACF;AAEA,SAAS,0BACR,KACA,KACO;AACP,MAAI,MAAM,aAAa;AACvB,MAAI,MAAM,UAAU;AACpB,MAAI,UAAU,gBAAgB;AAAA,IAC7B,WAAW,IAAI,MAAM;AAAA,IACrB;AAAA,EACD,CAAC;AACF;;;AChMA,SAAS,SAAAC,SAAO,SAAAC,eAAa;AAC7B,SAAS,QAAAC,OAAM,YAAAC,iBAAgB;;;ACIxB,SAAS,SAAY,KAAmC;AAC9D,SAAO;AACR;;;ADYO,IAAM,QAAQC,QAAM;AAAA,EAC1B,aAAa,CAAC,GAAG,WAA+B;AAAA,IAC/C,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,MAAM,OAAO,QAAQ;AAAA,IACrB,YAAY,OAAO,cAAc;AAAA,IACjC,WAAW,KAAK,IAAI;AAAA,EACrB;AAAA,EACA,QAAQ;AAAA,IACP,cAAcC,QAAa;AAAA,IAC3B,gBAAgBA,QAAa;AAAA,EAC9B;AAAA,EAEA,SAAS;AAAA,IACR,UAAU,CAAC,MAAa,EAAE;AAAA,EAC3B;AAAA,EAEA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,cAAc,OAAO,YAAY;AAC9C,YAAM,IAAI,SAAgB,OAAO;AAGjC,YAAM,aAAa,MAAM,QAAQ,KAAK,eAAe,YAAY;AAChE,YAAI,IAAI,KAAK;AAAA,UACZ,KAAK;AAAA,UACL,SAAS,EAAE,MAAM;AAAA,UACjB,YAAY,EAAE,MAAM;AAAA,QACrB,CAAC;AACD,UAAE,UAAU,gBAAgB,EAAE,KAAK;AACnC,eAAO,EAAE,MAAM;AAAA,MAChB,CAAC;AAED,YAAM,QAAQ,MAAM,aAAa,UAAU;AAE3C,YAAM,QAAQ,KAAK,kBAAkB,YAAY;AAChD,UAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,UAAE,UAAU,kBAAkB,EAAE,KAAK;AACrC,YAAI,IAAI,KAAK,EAAE,KAAK,mBAAmB,SAAS,EAAE,MAAM,GAAG,CAAC;AAAA,MAC7D,CAAC;AAED,aAAOC,MAAK,MAAM,MAAS;AAAA,IAC5B,CAAC;AAAA,EACH,CAAC;AAAA,EAED,SAAS;AAAA,IACR,cAAc;AAAA,EACf;AACD,CAAC;;;AEjED,SAAS,SAAAC,SAAO,SAAAC,eAAa;AAC7B,SAAS,QAAAC,OAAM,YAAAC,iBAAgB;AAsB/B,eAAe,aAAa,MAAc,aAAa,KAAoB;AAC1E,QAAM,IAAI;AAAA,IAAQ,CAAC,YAClB,WAAW,SAAS,MAAM,KAAK,OAAO,IAAI,GAAI;AAAA,EAC/C;AACA,MAAI,KAAK,OAAO,IAAI,YAAY;AAC/B,UAAM,IAAI,MAAM,GAAG,IAAI,qBAAqB;AAAA,EAC7C;AACD;AAEO,IAAM,QAAQC,QAAM;AAAA,EAC1B,aAAa,CAAC,OAAc;AAAA,IAC3B,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,WAAW,KAAK,IAAI;AAAA,EACrB;AAAA,EACA,QAAQ;AAAA,IACP,cAAcC,QAAa;AAAA,EAC5B;AAAA,EAEA,SAAS;AAAA,IACR,UAAU,CAAC,MAAa,EAAE;AAAA,EAC3B;AAAA,EAEA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,iBAAiB,OAAO,YAAY;AACjD,YAAM,IAAI,SAAgB,OAAO;AAEjC,YAAM,QAAQ,KAAK,YAAY,YAAY;AAC1C,YAAI,IAAI,KAAK,EAAE,KAAK,oBAAoB,SAAS,EAAE,MAAM,GAAG,CAAC;AAC7D,UAAE,MAAM,SAAS;AACjB,UAAE,MAAM,OAAO;AACf,UAAE,UAAU,gBAAgB,EAAE,KAAK;AACnC,cAAM,aAAa,cAAc,IAAI;AAAA,MACtC,CAAC;AAED,YAAM,QAAQ,KAAK,UAAU,YAAY;AACxC,UAAE,MAAM,SAAS;AACjB,UAAE,MAAM,OAAO;AACf,UAAE,UAAU,gBAAgB,EAAE,KAAK;AACnC,cAAM,aAAa,WAAW,GAAG;AAAA,MAClC,CAAC;AAED,YAAM,QAAQ,KAAK,WAAW,YAAY;AACzC,UAAE,MAAM,SAAS;AACjB,UAAE,MAAM,OAAO;AACf,UAAE,UAAU,gBAAgB,EAAE,KAAK;AACnC,cAAM,aAAa,eAAe,IAAI;AAAA,MACvC,CAAC;AAED,YAAM,QAAQ,KAAK,YAAY,YAAY;AAC1C,UAAE,MAAM,SAAS;AACjB,UAAE,MAAM,OAAO;AACf,UAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,UAAE,UAAU,gBAAgB,EAAE,KAAK;AACnC,YAAI,IAAI,KAAK,EAAE,KAAK,mBAAmB,SAAS,EAAE,MAAM,GAAG,CAAC;AAAA,MAC7D,CAAC;AAED,aAAOC,MAAK,MAAM,MAAS;AAAA,IAC5B,CAAC;AAAA,EACH,CAAC;AACF,CAAC;;;ACpFD,SAAS,SAAAC,SAAO,SAAAC,eAAa;AAC7B,SAAS,QAAAC,OAAM,YAAAC,iBAAgB;AAuB/B,SAAS,WACR,QACA,WACA,YACwC;AACxC,QAAM,QAAQ,SAAS;AACvB,QAAM,MAAM,KAAK,IAAI,QAAQ,WAAW,UAAU;AAClD,QAAM,QAAQ,CAAC;AACf,WAAS,IAAI,OAAO,IAAI,KAAK,KAAK;AACjC,UAAM,KAAK,CAAC;AAAA,EACb;AACA,SAAO;AAAA,IACN;AAAA,IACA,SAAS,MAAM;AAAA,EAChB;AACD;AAOO,IAAM,QAAQC,QAAM;AAAA,EAC1B,aAAa,CAAC,GAAG,WAAqC;AAAA,IACrD,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,YAAY,OAAO,cAAc;AAAA,IACjC,WAAW,OAAO,aAAa;AAAA,IAC/B,QAAQ;AAAA,IACR,gBAAgB;AAAA,IAChB,cAAc;AAAA,IACd,SAAS,CAAC;AAAA,IACV,WAAW,KAAK,IAAI;AAAA,EACrB;AAAA,EACA,QAAQ;AAAA,IACP,gBAAgBC,QAAiB;AAAA,IACjC,cAAcA,QAAgB;AAAA,IAC9B,oBAAoBA,QAAoD;AAAA,EACzE;AAAA,EAEA,SAAS;AAAA,IACR,QAAQ,CAAC,MAAgB,EAAE;AAAA,EAC5B;AAAA,EAEA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK;AAAA,MACd,MAAM;AAAA,MACN,OAAO,EAAE,QAAQ,EAAE;AAAA,MACnB,KAAK,OAAO,UAAU,cAAkC;AACvD,cAAM,IAAI,SAAgB,QAAQ;AAElC,cAAMC,SAAQ,MAAM,SAAS,KAAK,eAAe,YAAY;AAC5D,cAAI,IAAI,KAAK;AAAA,YACZ,KAAK;AAAA,YACL,OAAO,EAAE,MAAM;AAAA,YACf,QAAQ,UAAU;AAAA,UACnB,CAAC;AACD,gBAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAM,KAAK,OAAO,IAAI,GAAG,CAAC;AACjE,iBAAO,WAAW,UAAU,QAAQ,EAAE,MAAM,WAAW,EAAE,MAAM,UAAU;AAAA,QAC1E,CAAC;AAED,cAAM,SAAS,KAAK,iBAAiB,YAAY;AAChD,gBAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAM,KAAK,OAAO,IAAI,GAAG,CAAC;AAEjE,gBAAM,YAAuB;AAAA,YAC5B,IAAI,UAAU;AAAA,YACd,OAAOA,OAAM,MAAM;AAAA,YACnB,aAAa,KAAK,IAAI;AAAA,UACvB;AAEA,YAAE,MAAM,eAAe,UAAU;AACjC,YAAE,MAAM,kBAAkBA,OAAM,MAAM;AACtC,YAAE,MAAM,QAAQ,KAAK,SAAS;AAE9B,YAAE,UAAU,kBAAkB,SAAS;AACvC,YAAE,UAAU,gBAAgB,EAAE,KAAK;AAEnC,cAAI,IAAI,KAAK;AAAA,YACZ,KAAK;AAAA,YACL,OAAO,EAAE,MAAM;AAAA,YACf,QAAQ,UAAU;AAAA,YAClB,OAAOA,OAAM,MAAM;AAAA,UACpB,CAAC;AAAA,QACF,CAAC;AAED,YAAI,CAACA,OAAM,SAAS;AACnB,gBAAM,SAAS,KAAK,iBAAiB,YAAY;AAChD,cAAE,MAAM,SAAS;AACjB,cAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,cAAE,UAAU,gBAAgB,EAAE,KAAK;AACnC,cAAE,UAAU,sBAAsB;AAAA,cACjC,cAAc,UAAU,SAAS;AAAA,cACjC,YAAY,EAAE,MAAM;AAAA,YACrB,CAAC;AAAA,UACF,CAAC;AACD,iBAAOC,MAAK,MAAM,UAAU,SAAS,CAAC;AAAA,QACvC;AAEA,eAAOA,MAAK,SAAS,EAAE,QAAQ,UAAU,SAAS,EAAE,CAAC;AAAA,MACtD;AAAA,IACD,CAAC;AAAA,EACF,CAAC;AACF,CAAC;;;AC7HD,SAAS,SAAAC,SAAO,SAAAC,SAAO,SAAAC,cAAa;AACpC,SAAS,QAAAC,OAAM,YAAAC,iBAAgB;AAkB/B,IAAM,iBAAiB;AAEvB,IAAM,sBAAsB;AAYrB,IAAM,WAAWC,QAAM;AAAA,EAC7B,aAAa,CAAC,GAAG,WAAmD;AAAA,IACnE,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,OAAO,OAAO,SAAS;AAAA,IACvB,aAAa,OAAO,eAAe;AAAA,IACnC,QAAQ;AAAA,IACR,WAAW,KAAK,IAAI;AAAA,EACrB;AAAA,EACA,QAAQ;AAAA,IACP,UAAUC,OAAwB;AAAA,EACnC;AAAA,EACA,QAAQ;AAAA,IACP,gBAAgBC,QAAuB;AAAA,IACvC,gBAAgBA,QAAuB;AAAA,EACxC;AAAA,EAEA,SAAS;AAAA,IACR,YAAY,CAAC,MAAuB,EAAE;AAAA,IAEtC,SAAS,OAAO,GAAG,aAAqB;AACvC,UAAI,EAAE,MAAM,WAAW,UAAW;AAClC,QAAE,MAAM,WAAW;AACnB,QAAE,UAAU,kBAAkB,EAAE,KAAK;AACrC,YAAM,EAAE,MAAM,KAAK,gBAAgB,EAAE,UAAU,MAAM,SAAS,CAAC;AAAA,IAChE;AAAA,IAEA,QAAQ,OAAO,GAAG,aAAqB;AACtC,UAAI,EAAE,MAAM,WAAW,UAAW;AAClC,QAAE,MAAM,WAAW;AACnB,QAAE,UAAU,kBAAkB,EAAE,KAAK;AACrC,YAAM,EAAE,MAAM,KAAK,gBAAgB,EAAE,UAAU,OAAO,SAAS,CAAC;AAAA,IACjE;AAAA,EACD;AAAA,EAEA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,iBAAiB,OAAO,YAAY;AACjD,YAAM,IAAI,SAAgB,OAAO;AAEjC,YAAM,QAAQ,KAAK,gBAAgB,YAAY;AAC9C,YAAI,IAAI,KAAK;AAAA,UACZ,KAAK;AAAA,UACL,WAAW,EAAE,MAAM;AAAA,UACnB,OAAO,EAAE,MAAM;AAAA,QAChB,CAAC;AACD,UAAE,UAAU,kBAAkB,EAAE,KAAK;AAAA,MACtC,CAAC;AAED,YAAM,CAAC,eAAe,IAAI,MAAM,QAAQ,MAAM;AAAA,QAC7C;AAAA,QACA;AAAA,UACC,OAAO,CAAC,cAAc;AAAA,UACtB,SAAS;AAAA,QACV;AAAA,MACD;AACA,YAAM,WAAW,iBAAiB,QAAQ;AAE1C,YAAM,QAAQ,KAAK,iBAAiB,YAAY;AAC/C,UAAE,MAAM,WAAW;AACnB,YAAI,aAAa,MAAM;AACtB,YAAE,MAAM,SAAS;AACjB,cAAI,IAAI,KAAK,EAAE,KAAK,qBAAqB,WAAW,EAAE,MAAM,GAAG,CAAC;AAAA,QACjE,WAAW,SAAS,UAAU;AAC7B,YAAE,MAAM,SAAS;AACjB,YAAE,MAAM,YAAY,SAAS;AAC7B,cAAI,IAAI,KAAK;AAAA,YACZ,KAAK;AAAA,YACL,WAAW,EAAE,MAAM;AAAA,YACnB,UAAU,SAAS;AAAA,UACpB,CAAC;AAAA,QACF,OAAO;AACN,YAAE,MAAM,SAAS;AACjB,YAAE,MAAM,YAAY,SAAS;AAC7B,cAAI,IAAI,KAAK;AAAA,YACZ,KAAK;AAAA,YACL,WAAW,EAAE,MAAM;AAAA,YACnB,UAAU,SAAS;AAAA,UACpB,CAAC;AAAA,QACF;AACA,UAAE,MAAM,YAAY,KAAK,IAAI;AAC7B,UAAE,UAAU,kBAAkB,EAAE,KAAK;AAAA,MACtC,CAAC;AAED,aAAOC,MAAK,MAAM,MAAS;AAAA,IAC5B,CAAC;AAAA,EACH,CAAC;AACF,CAAC;;;ACvHD,SAAS,SAAAC,SAAO,SAAAC,SAAO,SAAAC,cAAa;AACpC,SAAS,QAAAC,OAAM,YAAAC,iBAAgB;AA2C/B,IAAM,gBAAgB;AAGtB,eAAe,iBAAqC;AACnD,QAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC;AAClE,SAAO;AAAA,IACN,OAAO,KAAK,MAAM,MAAO,KAAK,OAAO,IAAI,GAAG;AAAA,IAC5C,aAAa,KAAK,MAAM,MAAM,KAAK,OAAO,IAAI,GAAG;AAAA,IACjD,aAAa,KAAK,MAAM,KAAK,KAAK,OAAO,IAAI,EAAE;AAAA,EAChD;AACD;AAEA,eAAe,kBAAuC;AACrD,QAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAM,KAAK,OAAO,IAAI,GAAI,CAAC;AAClE,QAAM,QAAQ,KAAK,MAAM,KAAK,KAAK,OAAO,IAAI,GAAG;AACjD,QAAM,UAAU,KAAK,MAAM,MAAO,KAAK,OAAO,IAAI,IAAK;AACvD,SAAO;AAAA,IACN;AAAA,IACA;AAAA,IACA,eAAe,KAAK,MAAM,UAAU,KAAK;AAAA,EAC1C;AACD;AAEA,eAAe,oBAA2C;AACzD,QAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAM,KAAK,OAAO,IAAI,GAAG,CAAC;AACjE,SAAO;AAAA,IACN,WAAW,KAAK,MAAM,MAAQ,KAAK,OAAO,IAAI,GAAK;AAAA,IACnD,UAAU,KAAK,MAAM,MAAO,KAAK,OAAO,IAAI,GAAI;AAAA,IAChD,YAAY,KAAK,MAAM,KAAK,KAAK,OAAO,IAAI,EAAE;AAAA,EAC/C;AACD;AAEO,IAAM,YAAYC,QAAM;AAAA,EAC9B,OAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,UAAU;AAAA,MACT,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,SAAS;AAAA,IACV;AAAA,IACA,aAAa;AAAA,EACd;AAAA,EACA,QAAQ;AAAA,IACP,CAAC,aAAa,GAAGC,OAAsB;AAAA,EACxC;AAAA,EACA,QAAQ;AAAA,IACP,cAAcC,QAAsB;AAAA,IACpC,iBAAiBA,QAAqB;AAAA,EACvC;AAAA,EAEA,SAAS;AAAA,IACR,SAAS,OAAO,MAAM;AACrB,UAAI,CAAC,EAAE,MAAM,SAAS;AACrB,UAAE,MAAM,UAAU;AAClB,UAAE,MAAM,WAAW;AAAA,UAClB,OAAO;AAAA,UACP,QAAQ;AAAA,UACR,SAAS;AAAA,QACV;AACA,UAAE,UAAU,gBAAgB,EAAE,KAAK;AACnC,cAAM,EAAE,MAAM,KAAK,eAAe,CAAC,CAAC;AAAA,MACrC;AAAA,IACD;AAAA,IAEA,UAAU,CAAC,MAAsB,EAAE;AAAA,EACpC;AAAA,EAEA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,gBAAgB,OAAO,YAAY;AAChD,YAAM,IAAI,SAAgB,OAAO;AAEjC,YAAM,QAAQ,MAAM,KAAK,gBAAgB;AAAA,QACxC,OAAO,CAAC,aAAa;AAAA,MACtB,CAAC;AAED,UAAI,IAAI,KAAK,EAAE,KAAK,6BAA6B,CAAC;AAElD,YAAM,UAAU,MAAM,QAAQ,KAAK,aAAa;AAAA,QAC/C,OAAO;AAAA,UACN,KAAK,OAAO,cAAc;AACzB,kBAAM,KAAK,SAAgB,SAAS;AAEpC,kBAAM,UAAU,KAAK,gBAAgB,YAAY;AAChD,iBAAG,MAAM,SAAS,QAAQ;AAC1B,iBAAG,UAAU,gBAAgB,GAAG,KAAK;AAAA,YACtC,CAAC;AAED,kBAAM,OAAO,MAAM,UAAU,KAAK,eAAe,YAAY;AAC5D,qBAAO,MAAM,eAAe;AAAA,YAC7B,CAAC;AAED,kBAAM,UAAU,KAAK,iBAAiB,YAAY;AACjD,iBAAG,MAAM,SAAS,QAAQ;AAC1B,iBAAG,UAAU,gBAAgB,GAAG,KAAK;AAAA,YACtC,CAAC;AAED,mBAAO;AAAA,UACR;AAAA,QACD;AAAA,QACA,QAAQ;AAAA,UACP,KAAK,OAAO,cAAc;AACzB,kBAAM,KAAK,SAAgB,SAAS;AAEpC,kBAAM,UAAU,KAAK,gBAAgB,YAAY;AAChD,iBAAG,MAAM,SAAS,SAAS;AAC3B,iBAAG,UAAU,gBAAgB,GAAG,KAAK;AAAA,YACtC,CAAC;AAED,kBAAM,OAAO,MAAM,UAAU,KAAK,gBAAgB,YAAY;AAC7D,qBAAO,MAAM,gBAAgB;AAAA,YAC9B,CAAC;AAED,kBAAM,UAAU,KAAK,iBAAiB,YAAY;AACjD,iBAAG,MAAM,SAAS,SAAS;AAC3B,iBAAG,UAAU,gBAAgB,GAAG,KAAK;AAAA,YACtC,CAAC;AAED,mBAAO;AAAA,UACR;AAAA,QACD;AAAA,QACA,SAAS;AAAA,UACR,KAAK,OAAO,cAAc;AACzB,kBAAM,KAAK,SAAgB,SAAS;AAEpC,kBAAM,UAAU,KAAK,gBAAgB,YAAY;AAChD,iBAAG,MAAM,SAAS,UAAU;AAC5B,iBAAG,UAAU,gBAAgB,GAAG,KAAK;AAAA,YACtC,CAAC;AAED,kBAAM,OAAO,MAAM,UAAU,KAAK,iBAAiB,YAAY;AAC9D,qBAAO,MAAM,kBAAkB;AAAA,YAChC,CAAC;AAED,kBAAM,UAAU,KAAK,iBAAiB,YAAY;AACjD,iBAAG,MAAM,SAAS,UAAU;AAC5B,iBAAG,UAAU,gBAAgB,GAAG,KAAK;AAAA,YACtC,CAAC;AAED,mBAAO;AAAA,UACR;AAAA,QACD;AAAA,MACD,CAAC;AAED,YAAM,QAAQ,KAAK,aAAa,YAAY;AAC3C,UAAE,MAAM,OAAO;AAAA,UACd,OAAO,QAAQ;AAAA,UACf,QAAQ,QAAQ;AAAA,UAChB,SAAS,QAAQ;AAAA,UACjB,WAAW,KAAK,IAAI;AAAA,QACrB;AACA,UAAE,MAAM,UAAU;AAClB,UAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,UAAE,UAAU,gBAAgB,EAAE,KAAK;AACnC,UAAE,UAAU,mBAAmB,EAAE,MAAM,IAAI;AAAA,MAC5C,CAAC;AAED,UAAI,IAAI,KAAK,EAAE,KAAK,6BAA6B,CAAC;AAElD,aAAOC,MAAK,SAAS,MAAS;AAAA,IAC/B,CAAC;AAAA,EACH,CAAC;AACF,CAAC;;;AC7MD,SAAS,SAAAC,SAAO,SAAAC,eAAa;AAC7B,SAAS,QAAAC,OAAM,YAAAC,iBAAgB;AAqBxB,IAAM,OAAOC,QAAM;AAAA,EACzB,aAAa,CAAC,GAAG,WAAqC;AAAA,IACrD,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,gBAAgB,OAAO,kBAAkB;AAAA,IACzC,WAAW,OAAO,aAAa;AAAA,IAC/B,QAAQ;AAAA,IACR,WAAW,KAAK,IAAI;AAAA,EACrB;AAAA,EACA,QAAQ;AAAA,IACP,aAAaC,QAAgB;AAAA,IAC7B,eAAeA,QAAgB;AAAA,EAChC;AAAA,EAEA,SAAS;AAAA,IACR,SAAS,CAAC,MAAgB,EAAE;AAAA,EAC7B;AAAA,EAEA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,aAAa,OAAO,YAAY;AAC7C,YAAM,IAAI,SAAgB,OAAO;AAGjC,YAAM,EAAE,gBAAgB,WAAW,OAAO,IAAI,MAAM,QAAQ;AAAA,QAC3D;AAAA,QACA,YAAY;AACX,cAAI,IAAI,KAAK;AAAA,YACZ,KAAK;AAAA,YACL,QAAQ,EAAE,MAAM;AAAA,YAChB,gBAAgB,EAAE,MAAM;AAAA,YACxB,WAAW,EAAE,MAAM;AAAA,UACpB,CAAC;AACD,YAAE,UAAU,eAAe,EAAE,KAAK;AAClC,iBAAO;AAAA,YACN,gBAAgB,EAAE,MAAM;AAAA,YACxB,WAAW,EAAE,MAAM;AAAA,YACnB,QAAQ,EAAE,MAAM;AAAA,UACjB;AAAA,QACD;AAAA,MACD;AAEA,YAAM,EAAE,QAAQ,MAAM,IAAI,MAAM,QAAQ,KAAK,mBAAmB;AAAA,QAC/D;AAAA,UACC,MAAM;AAAA,UACN,KAAK,OAAO,cAAc;AACzB,kBAAM,UAAU,MAAM,iBAAiB,cAAc;AACrD,mBAAO,MAAM,UAAU,KAAK,iBAAiB,YAAY;AACxD,qBAAO,mBAAmB,MAAM;AAAA,YACjC,CAAC;AAAA,UACF;AAAA,QACD;AAAA,QACA;AAAA,UACC,MAAM;AAAA,UACN,KAAK,OAAO,cAAc;AACzB,kBAAM,UAAU,MAAM,gBAAgB,SAAS;AAC/C,mBAAO;AAAA,UACR;AAAA,QACD;AAAA,MACD,CAAC;AAED,YAAM,QAAQ,KAAK,eAAe,YAAY;AAC7C,UAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,UAAE,MAAM,mBAAmB,EAAE,MAAM,cAAc,EAAE,MAAM;AAEzD,YAAI,WAAW,QAAQ;AACtB,YAAE,MAAM,SAAS;AACjB,YAAE,MAAM,SAAS;AACjB,cAAI,IAAI,KAAK;AAAA,YACZ,KAAK;AAAA,YACL,QAAQ,EAAE,MAAM;AAAA,YAChB,YAAY,EAAE,MAAM;AAAA,UACrB,CAAC;AAAA,QACF,OAAO;AACN,YAAE,MAAM,SAAS;AACjB,cAAI,IAAI,KAAK;AAAA,YACZ,KAAK;AAAA,YACL,QAAQ,EAAE,MAAM;AAAA,YAChB,YAAY,EAAE,MAAM;AAAA,UACrB,CAAC;AAAA,QACF;AAEA,UAAE,UAAU,iBAAiB,EAAE,KAAK;AAAA,MACrC,CAAC;AAED,aAAOC,MAAK,MAAM,MAAS;AAAA,IAC5B,CAAC;AAAA,EACH,CAAC;AACF,CAAC;;;AC5GD,SAAS,SAAAC,SAAO,SAAAC,eAAa;AAC7B,SAAS,QAAAC,OAAM,YAAAC,iBAAgB;AAmCxB,IAAM,UAAUC,QAAM;AAAA,EAC5B,aAAa,CAAC,GAAG,WAA2C;AAAA,IAC3D,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ,OAAO,UAAU;AAAA,IACzB,YAAY,OAAO,cAAc;AAAA,IACjC,QAAQ;AAAA,IACR,OAAO;AAAA,MACN,EAAE,MAAM,qBAAqB,QAAQ,UAAU;AAAA,MAC/C,EAAE,MAAM,eAAe,QAAQ,UAAU;AAAA,MACzC,EAAE,MAAM,kBAAkB,QAAQ,UAAU;AAAA,IAC7C;AAAA,IACA,WAAW,KAAK,IAAI;AAAA,EACrB;AAAA,EACA,QAAQ;AAAA,IACP,oBAAoBC,QAAmB;AAAA,IACvC,oBAAoBA,QAAmB;AAAA,IACvC,sBAAsBA,QAAmB;AAAA,EAC1C;AAAA,EAEA,SAAS;AAAA,IACR,gBAAgB,CAAC,MAAmB,EAAE;AAAA,EACvC;AAAA,EAEA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,gBAAgB,OAAO,YAAY;AAChD,YAAM,IAAI,SAAgB,OAAO;AAEjC,YAAM,QAAQ,KAAK,gBAAgB,YAAY;AAC9C,YAAI,IAAI,KAAK;AAAA,UACZ,KAAK;AAAA,UACL,MAAM,EAAE,MAAM;AAAA,UACd,QAAQ,EAAE,MAAM;AAAA,UAChB,YAAY,EAAE,MAAM;AAAA,QACrB,CAAC;AACD,UAAE,UAAU,sBAAsB,EAAE,KAAK;AAAA,MAC1C,CAAC;AAED,YAAM,QAAQ,mBAAmB,oBAAoB;AAGrD,YAAM,QAAQ,KAAK;AAAA,QAClB,MAAM;AAAA,QACN,KAAK,YAAY;AAChB,YAAE,MAAM,SAAS;AACjB,gBAAM,OAAO,EAAE,MAAM,MAAM;AAAA,YAC1B,CAAC,MAAM,EAAE,SAAS;AAAA,UACnB;AACA,cAAI,MAAM;AACT,iBAAK,SAAS;AACd,iBAAK,cAAc,KAAK,IAAI;AAAA,UAC7B;AACA,YAAE,UAAU,sBAAsB,EAAE,KAAK;AAEzC,gBAAM,IAAI;AAAA,YAAQ,CAAC,MAClB,WAAW,GAAG,MAAM,KAAK,OAAO,IAAI,GAAG;AAAA,UACxC;AACA,cAAI,IAAI,KAAK,EAAE,KAAK,sBAAsB,MAAM,EAAE,MAAM,GAAG,CAAC;AAC5D,iBAAO,EAAE,UAAU,KAAK;AAAA,QACzB;AAAA,QACA,UAAU,YAAY;AAErB,YAAE,MAAM,SAAS;AACjB,gBAAM,OAAO,EAAE,MAAM,MAAM;AAAA,YAC1B,CAAC,MAAM,EAAE,SAAS;AAAA,UACnB;AACA,cAAI,MAAM;AACT,iBAAK,SAAS;AACd,iBAAK,eAAe,KAAK,IAAI;AAAA,UAC9B;AACA,cAAI,IAAI,KAAK,EAAE,KAAK,sBAAsB,MAAM,EAAE,MAAM,GAAG,CAAC;AAC5D,YAAE,UAAU,sBAAsB,EAAE,KAAK;AAEzC,gBAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAG,CAAC;AAAA,QAC5C;AAAA,MACD,CAAC;AAGD,YAAM,QAAQ,KAAK;AAAA,QAClB,MAAM;AAAA,QACN,KAAK,YAAY;AAChB,YAAE,MAAM,SAAS;AACjB,gBAAM,OAAO,EAAE,MAAM,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,aAAa;AAC/D,cAAI,MAAM;AACT,iBAAK,SAAS;AACd,iBAAK,cAAc,KAAK,IAAI;AAAA,UAC7B;AACA,YAAE,UAAU,sBAAsB,EAAE,KAAK;AAEzC,gBAAM,IAAI;AAAA,YAAQ,CAAC,MAClB,WAAW,GAAG,MAAM,KAAK,OAAO,IAAI,GAAG;AAAA,UACxC;AAEA,cAAI,EAAE,MAAM,YAAY;AACvB,kBAAM,IAAI,MAAM,8BAA8B;AAAA,UAC/C;AAEA,cAAI,IAAI,KAAK,EAAE,KAAK,gBAAgB,MAAM,EAAE,MAAM,GAAG,CAAC;AACtD,iBAAO,EAAE,UAAU,MAAM,EAAE,MAAM,EAAE,GAAG;AAAA,QACvC;AAAA,QACA,UAAU,YAAY;AACrB,YAAE,MAAM,SAAS;AACjB,gBAAM,OAAO,EAAE,MAAM,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,aAAa;AAC/D,cAAI,MAAM;AACT,iBAAK,SAAS;AACd,iBAAK,eAAe,KAAK,IAAI;AAAA,UAC9B;AACA,cAAI,IAAI,KAAK,EAAE,KAAK,mBAAmB,MAAM,EAAE,MAAM,GAAG,CAAC;AACzD,YAAE,UAAU,sBAAsB,EAAE,KAAK;AAEzC,gBAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAG,CAAC;AAAA,QAC5C;AAAA,MACD,CAAC;AAGD,YAAM,QAAQ,KAAK,kBAAkB,YAAY;AAC/C,UAAE,MAAM,SAAS;AACjB,cAAM,OAAO,EAAE,MAAM,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,gBAAgB;AAClE,YAAI,KAAM,MAAK,SAAS;AACxB,UAAE,UAAU,sBAAsB,EAAE,KAAK;AAEzC,cAAM,IAAI;AAAA,UAAQ,CAAC,MAClB,WAAW,GAAG,MAAM,KAAK,OAAO,IAAI,GAAG;AAAA,QACxC;AAEA,UAAE,MAAM,SAAS;AACjB,UAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,YAAI,IAAI,KAAK,EAAE,KAAK,mBAAmB,MAAM,EAAE,MAAM,GAAG,CAAC;AACzD,UAAE,UAAU,wBAAwB,EAAE,KAAK;AAAA,MAC5C,CAAC;AAEF,aAAOC,MAAK,MAAM,MAAS;AAAA,IAC5B,CAAC;AAAA,EACH,CAAC;AACF,CAAC;;;AC7KD,SAAS,SAAAC,SAAc,SAAAC,cAAa;AACpC,SAAS,QAAAC,OAAM,YAAAC,iBAAgB;AAG/B,SAAS,MAAM,IAA2B;AACzC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACxD;AAWO,IAAM,wBAAwBC,QAAM;AAAA,EAC1C,aAAa,CAAC,OAAmC;AAAA,IAChD,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,EACT;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,MAAkC,EAAE;AAAA,EAChD;AAAA,EACA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,SAAS,YAAY;AACnC,YAAM,IAAI,SAAqC,GAAG;AAClD,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,WAAW;AACnB,QAAE,MAAM,YAAY,KAAK,IAAI;AAC7B,aAAO,EAAE,aAAa,KAAK;AAAA,IAC5B,CAAC;AAED,UAAM,MAAM,GAAG;AAEf,UAAM,IAAI,KAAK,WAAW,YAAY;AACrC,YAAM,IAAI,SAAqC,GAAG;AAClD,QAAE,MAAM,WAAW;AACnB,aAAO,EAAE,WAAW,MAAM,OAAO,EAAE;AAAA,IACpC,CAAC;AAED,UAAM,MAAM,IAAI;AAEhB,UAAM,IAAI,KAAK,YAAY,YAAY;AACtC,YAAM,IAAI,SAAqC,GAAG;AAClD,QAAE,MAAM,WAAW;AACnB,aAAO,EAAE,OAAO,KAAK;AAAA,IACtB,CAAC;AAED,UAAM,MAAM,GAAG;AAEf,UAAM,IAAI,KAAK,YAAY,YAAY;AACtC,YAAM,IAAI,SAAqC,GAAG;AAClD,QAAE,MAAM,WAAW;AACnB,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,QAAE,MAAM,SAAS,EAAE,SAAS,MAAM,gBAAgB,EAAE;AACpD,aAAO,EAAE,SAAS,KAAK;AAAA,IACxB,CAAC;AAAA,EACF,CAAC;AACF,CAAC;AAED,IAAM,aAAa,CAAC,KAAK,KAAK,GAAG;AAU1B,IAAM,sBAAsBD,QAAM;AAAA,EACxC,aAAa,CAAC,OAAiC;AAAA,IAC9C,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,SAAS,CAAC;AAAA,EACX;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,MAAgC,EAAE;AAAA,EAC9C;AAAA,EACA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,QAAQ,YAAY;AAClC,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,WAAW,WAAW,OAAO;AAAA,IACvC,CAAC;AAED,UAAM,IAAI,KAAK;AAAA,MACd,MAAM;AAAA,MACN,OAAO,EAAE,OAAO,EAAE;AAAA,MAClB,gBAAgB;AAAA,MAChB,cAAc;AAAA,MACd,aAAa,WAAW;AAAA,MACxB,KAAK,OAAO,SAAS,cAAiC;AACrD,cAAM,OAAO,WAAW,UAAU,KAAK;AAEvC,cAAM,QAAQ,KAAK,WAAW,UAAU,KAAK,IAAI,YAAY;AAC5D,gBAAM,IAAI,SAAmC,OAAO;AACpD,YAAE,MAAM,aAAa;AACrB,YAAE,MAAM,QAAQ,KAAK,EAAE,OAAO,UAAU,OAAO,KAAK,CAAC;AACrD,iBAAO,EAAE,MAAM,QAAQ,OAAO;AAAA,QAC/B,CAAC;AAED,YAAI,UAAU,SAAS,WAAW,SAAS,GAAG;AAC7C,iBAAOC,MAAK,MAAM,EAAE,WAAW,WAAW,OAAO,CAAC;AAAA,QACnD;AAEA,eAAOA,MAAK,SAAS,EAAE,OAAO,UAAU,QAAQ,EAAE,CAAC;AAAA,MACpD;AAAA,IACD,CAAC;AAED,UAAM,IAAI,KAAK,YAAY,YAAY;AACtC,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,aAAO,EAAE,cAAc,KAAK;AAAA,IAC7B,CAAC;AAAA,EACF,CAAC;AACF,CAAC;AAYM,IAAM,sBAAsBF,QAAM;AAAA,EACxC,aAAa,CAAC,OAAiC;AAAA,IAC9C,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,EACT;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,MAAgC,EAAE;AAAA,EAC9C;AAAA,EACA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,SAAS,YAAY;AACnC,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,OAAO,KAAK;AAAA,IACtB,CAAC;AAED,UAAM,UAAU,MAAM,IAAI,KAAK,kBAAkB;AAAA,MAChD,aAAa;AAAA,QACZ,KAAK,OAAO,cAAc;AACzB,gBAAM,UAAU,KAAK,UAAU,YAAY;AAC1C,kBAAM,MAAM,GAAG;AACf,mBAAO,EAAE,SAAS,KAAK;AAAA,UACxB,CAAC;AACD,iBAAO,EAAE,MAAM,eAAe;AAAA,QAC/B;AAAA,MACD;AAAA,MACA,YAAY;AAAA,QACX,KAAK,OAAO,cAAc;AACzB,gBAAM,UAAU,KAAK,UAAU,YAAY;AAC1C,kBAAM,MAAM,GAAG;AACf,mBAAO,EAAE,SAAS,KAAK;AAAA,UACxB,CAAC;AACD,iBAAO,EAAE,MAAM,GAAG;AAAA,QACnB;AAAA,MACD;AAAA,MACA,eAAe;AAAA,QACd,KAAK,OAAO,cAAc;AACzB,gBAAM,UAAU,KAAK,UAAU,YAAY;AAC1C,kBAAM,MAAM,EAAE;AACd,mBAAO,EAAE,SAAS,KAAK;AAAA,UACxB,CAAC;AACD,iBAAO,EAAE,KAAK,KAAK;AAAA,QACpB;AAAA,MACD;AAAA,IACD,CAAC;AAED,UAAM,IAAI,KAAK,iBAAiB,YAAY;AAC3C,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,SAAS;AAAA,QAChB,KAAK,QAAQ,WAAW,EAAE;AAAA,QAC1B,MAAM,QAAQ,UAAU,EAAE;AAAA,QAC1B,UAAU,QAAQ,aAAa,EAAE;AAAA,MAClC;AACA,aAAO,EAAE,QAAQ,KAAK;AAAA,IACvB,CAAC;AAAA,EACF,CAAC;AACF,CAAC;AASM,IAAM,sBAAsBD,QAAM;AAAA,EACxC,aAAa,CAAC,OAAiC;AAAA,IAC9C,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,EACT;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,MAAgC,EAAE;AAAA,EAC9C;AAAA,EACA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,SAAS,YAAY;AACnC,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,SAAS,KAAK;AAAA,IACxB,CAAC;AAED,UAAM,EAAE,QAAQ,MAAM,IAAI,MAAM,IAAI,KAGjC,kBAAkB;AAAA,MACpB;AAAA,QACC,MAAM;AAAA,QACN,KAAK,OAAO,cAAc;AACzB,gBAAM,UAAU,MAAM,sBAAsB,EAAE;AAC9C,iBAAO,EAAE,UAAU,YAAY,SAAS,GAAG;AAAA,QAC5C;AAAA,MACD;AAAA,MACA;AAAA,QACC,MAAM;AAAA,QACN,KAAK,OAAO,cAAc;AACzB,gBAAM,UAAU,MAAM,sBAAsB,GAAG;AAC/C,iBAAO,EAAE,UAAU,UAAU,SAAS,IAAI;AAAA,QAC3C;AAAA,MACD;AAAA,IACD,CAAC;AAED,UAAM,IAAI,KAAK,cAAc,YAAY;AACxC,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,SAAS,MAAM;AACvB,aAAO,EAAE,MAAM,MAAM,SAAS;AAAA,IAC/B,CAAC;AAAA,EACF,CAAC;AACF,CAAC;AAWD,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAC5B,IAAM,mBAAmB;AACzB,IAAM,uBAAuB;AAC7B,IAAM,oBAAoB;AAC1B,IAAM,uBAAuB;AAgB7B,IAAM,8BAA6C;AAAA,EAClD,EAAE,MAAM,qBAAqB,SAAS,EAAE,IAAI,UAAU,EAAE;AAAA,EACxD,EAAE,MAAM,qBAAqB,SAAS,EAAE,IAAI,WAAW,QAAQ,OAAO,EAAE;AAAA,EACxE,EAAE,MAAM,kBAAkB,SAAS,EAAE,KAAK,SAAS,KAAK,EAAE,EAAE;AAAA,EAC5D,EAAE,MAAM,kBAAkB,SAAS,EAAE,KAAK,SAAS,KAAK,EAAE,EAAE;AAAA,EAC5D,EAAE,MAAM,sBAAsB,SAAS,EAAE,YAAY,aAAa,EAAE;AAAA,EACpE,EAAE,MAAM,sBAAsB,SAAS,EAAE,YAAY,aAAa,EAAE;AAAA,EACpE,EAAE,MAAM,sBAAsB,SAAS,EAAE,YAAY,aAAa,EAAE;AAAA,EACpE,EAAE,MAAM,mBAAmB,SAAS,EAAE,OAAO,EAAE,EAAE;AAAA,EACjD,EAAE,MAAM,mBAAmB,SAAS,EAAE,OAAO,EAAE,EAAE;AAAA,EACjD,EAAE,MAAM,mBAAmB,SAAS,EAAE,OAAO,EAAE,EAAE;AAClD;AAEA,IAAM,sBAAsB;AAAA,EAC3B,EAAE,IAAI,UAAU,WAAW,KAAK,KAAK,EAAE;AAAA,EACvC,EAAE,IAAI,UAAU,WAAW,KAAK,KAAK,EAAE;AAAA,EACvC,EAAE,IAAI,UAAU,WAAW,KAAK,KAAK,GAAG;AAAA,EACxC,EAAE,IAAI,UAAU,WAAW,KAAK,KAAK,GAAG;AACzC;AAEO,IAAM,sBAAsBD,QAAM;AAAA,EACxC,aAAa,CAAC,OAAiC;AAAA,IAC9C,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,IACR,gBAAgB;AAAA,EACjB;AAAA,EACA,QAAQ;AAAA,IACP,CAAC,mBAAmB,GAAGG,OAA2B;AAAA,IAClD,CAAC,mBAAmB,GAAGA,OAA2B;AAAA,IAClD,CAAC,gBAAgB,GAAGA,OAAwB;AAAA,IAC5C,CAAC,oBAAoB,GAAGA,OAA4B;AAAA,IACpD,CAAC,iBAAiB,GAAGA,OAAyB;AAAA,IAC9C,CAAC,oBAAoB,GAAGA,OAA4B;AAAA,EACrD;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,MAAgC,EAAE;AAAA,IAC7C,cAAc,OAAO,MAAM;AAC1B,UAAI,EAAE,MAAM,eAAgB;AAC5B,iBAAW,QAAQ,6BAA6B;AAC/C,cAAM,EAAE,MAAM,KAAK,KAAK,MAAM,KAAK,OAAO;AAAA,MAC3C;AACA,QAAE,MAAM,iBAAiB;AAAA,IAC1B;AAAA,EACD;AAAA,EACA,KAAKF,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,aAAa,YAAY;AACvC,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,WAAW;AACnB,QAAE,MAAM,YAAY,KAAK,IAAI;AAC7B,aAAO;AAAA,QACN,WAAW,OAAO,EAAE,MAAM,EAAE;AAAA,QAC5B,WAAW,KAAK,IAAI;AAAA,MACrB;AAAA,IACD,CAAC;AAED,UAAM,IAAI,KAAK,kBAAkB,YAAY;AAC5C,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,WAAW;AACnB,aAAO;AAAA,IACR,CAAC;AAED,UAAM,IAAI,mBAAmB,6BAA6B;AAE1D,UAAM,IAAI,KAAK,qBAAqB,YAAY;AAC/C,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,WAAW;AACnB,aAAO;AAAA,QACN,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,OAAO,CAAC,kBAAkB,gBAAgB;AAAA,MAC3C;AAAA,IACD,CAAC;AAED,UAAM,IAAI,KAAK,oBAAoB,YAAY;AAC9C,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,WAAW;AACnB,aAAO,EAAE,SAAS,GAAG,QAAQ,gBAAgB;AAAA,IAC9C,CAAC;AAED,UAAM,IAAI,KAAK,yBAAyB,YAAY;AACnD,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,WAAW;AACnB,aAAO,EAAE,UAAU,OAAO,MAAM,WAAW;AAAA,IAC5C,CAAC;AAED,UAAM,IAAI,mBAAmB,2BAA2B;AAExD,UAAM,IAAI,KAAK;AAAA,MACd,MAAM;AAAA,MACN,OAAO,EAAE,OAAO,EAAE;AAAA,MAClB,gBAAgB;AAAA,MAChB,cAAc;AAAA,MACd,aAAa;AAAA,MACb,KAAK,OAAO,SAAS,cAAiC;AACrD,cAAM,OAAO,oBAAoB,UAAU,KAAK;AAChD,YAAI,CAAC,MAAM;AACV,iBAAOC,MAAK,MAAM,EAAE,OAAO,oBAAoB,OAAO,CAAC;AAAA,QACxD;AAEA,cAAM,QAAQ,KAAK,cAAc,UAAU,KAAK,IAAI,YAAY;AAC/D,iBAAO,EAAE,QAAQ,KAAK,IAAI,WAAW,KAAK,UAAU;AAAA,QACrD,CAAC;AAED,cAAM,QAAQ,KAAK,eAAe,UAAU,KAAK,IAAI,YAAY;AAChE,iBAAO,KAAK;AAAA,QACb,CAAC;AAED,cAAM,QAAQ;AAAA,UACb,qBAAqB,UAAU,KAAK;AAAA,UACpC,aAAa;AAAA,YACZ,eAAe,OAAO,UAAU,KAAK;AAAA,YACrC,QAAQ,KAAK;AAAA,UACd;AAAA,QACD;AAEA,YAAI,UAAU,SAAS,oBAAoB,SAAS,GAAG;AACtD,iBAAOA,MAAK,MAAM;AAAA,YACjB,OAAO,oBAAoB;AAAA,YAC3B,OAAO;AAAA,UACR,CAAC;AAAA,QACF;AAEA,eAAOA,MAAK,SAAS,EAAE,OAAO,UAAU,QAAQ,EAAE,CAAC;AAAA,MACpD;AAAA,IACD,CAAC;AAED,UAAM,IAAI,MAAM,kBAAkB,EAAE;AACpC,UAAM,IAAI,MAAM,kBAAkB,EAAE;AACpC,UAAM,IAAI,MAAM,uBAAuB,EAAE;AAEzC,UAAM,IAAI,KAAK,qBAAqB,YAAY;AAC/C,YAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAM,eAAe,KAAK,IAAI,IAAI;AAClC,aAAO,EAAE,SAAS,aAAa;AAAA,IAChC,CAAC;AAED,UAAM,IAAI,MAAM,KAAK,wBAAwB;AAAA,MAC5C,OAAO,CAAC,mBAAmB;AAAA,IAC5B,CAAC;AACD,UAAM,IAAI,MAAM,UAAU,gCAAgC;AAAA,MACzD,OAAO,CAAC,mBAAmB;AAAA,MAC3B,SAAS;AAAA,IACV,CAAC;AACD,UAAM,IAAI,MAAM,UAAU,oBAAoB;AAAA,MAC7C,OAAO,CAAC,gBAAgB;AAAA,MACxB,OAAO;AAAA,IACR,CAAC;AACD,UAAM,IAAI,MAAM,UAAU,4BAA4B;AAAA,MACrD,OAAO,CAAC,oBAAoB;AAAA,MAC5B,OAAO;AAAA,MACP,SAAS;AAAA,IACV,CAAC;AACD,UAAM,IAAI,MAAM,UAAU,mBAAmB;AAAA,MAC5C,OAAO,CAAC,oBAAoB;AAAA,MAC5B,SAAS;AAAA,IACV,CAAC;AACD,UAAM,IAAI,MAAM,UAAU,gBAAgB;AAAA,MACzC,OAAO,CAAC,iBAAiB;AAAA,MACzB,SAAS;AAAA,IACV,CAAC;AACD,UAAM,IAAI,MAAM,UAAU,sBAAsB;AAAA,MAC/C,OAAO,CAAC,iBAAiB;AAAA,MACzB,OAAO;AAAA,MACP,SAAS;AAAA,IACV,CAAC;AAED,UAAM,IAAI,KAAK,qBAAqB;AAAA,MACnC,WAAW;AAAA,QACV,KAAK,OAAO,cAAc;AACzB,gBAAM,WAAW,MAAM,UAAU;AAAA,YAChC;AAAA,YACA,YAAY;AAAA,UACb;AACA,gBAAM,UAAU,MAAM,wBAAwB,EAAE;AAChD,iBAAO;AAAA,YACN;AAAA,YACA,SAAS;AAAA,YACT,OAAO,CAAC,gBAAgB,SAAS;AAAA,UAClC;AAAA,QACD;AAAA,MACD;AAAA,MACA,SAAS;AAAA,QACR,KAAK,OAAO,cAAc;AACzB,gBAAM,SAAS,MAAM,UAAU;AAAA,YAC9B;AAAA,YACA,YAAY;AAAA,UACb;AACA,iBAAO;AAAA,YACN,UAAU;AAAA,YACV,UAAU;AAAA,YACV,OAAO;AAAA,YACP;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,MACA,UAAU;AAAA,QACT,KAAK,OAAO,cAAc;AACzB,gBAAM,OAAO,MAAM,UAAU;AAAA,YAC5B;AAAA,YACA,YAAY;AAAA,UACb;AACA,gBAAM,UAAU,MAAM,uBAAuB,EAAE;AAC/C,iBAAO,EAAE,QAAQ,UAAU,SAAS,GAAG,KAAK;AAAA,QAC7C;AAAA,MACD;AAAA,IACD,CAAC;AAED,UAAM,IAAI,KAAK,oBAAoB;AAAA,MAClC;AAAA,QACC,MAAM;AAAA,QACN,KAAK,OAAO,cAAc;AACzB,gBAAM,UAAU,MAAM,mBAAmB,EAAE;AAC3C,iBAAO,EAAE,QAAQ,WAAW,MAAM,IAAI,SAAS,EAAE;AAAA,QAClD;AAAA,MACD;AAAA,MACA;AAAA,QACC,MAAM;AAAA,QACN,KAAK,OAAO,cAAc;AACzB,gBAAM,UAAU,MAAM,mBAAmB,GAAG;AAC5C,iBAAO,EAAE,QAAQ,UAAU,MAAM,GAAG,SAAS,EAAE;AAAA,QAChD;AAAA,MACD;AAAA,IACD,CAAC;AAED,UAAM,IAAI,QAAQ,2BAA2B,MAAM;AAEnD,UAAM,IAAI,KAAK,YAAY,YAAY;AACtC,YAAM,IAAI,SAAmC,GAAG;AAChD,QAAE,MAAM,WAAW;AACnB,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,aAAO;AAAA,IACR,CAAC;AAAA,EACF,CAAC;AACF,CAAC;AAeM,IAAM,4BAA4BF,QAAM;AAAA,EAC9C,aAAa,CACZ,GACA,WACqC;AAAA,IACrC,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,IACR,sBAAsB,OAAO,wBAAwB;AAAA,IACrD,UAAU;AAAA,EACX;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,MAAsC,EAAE;AAAA,EACpD;AAAA,EACA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,QAAQ,YAAY;AAClC,YAAM,IAAI,SAAyC,GAAG;AACtD,QAAE,MAAM,YAAY,KAAK,IAAI;AAC7B,QAAE,MAAM,WAAW;AACnB,aAAO,EAAE,aAAa,KAAK;AAAA,IAC5B,CAAC;AAED,UAAM,IAAI,KAAK,cAAc,YAAY;AACxC,YAAM,IAAI,SAAyC,GAAG;AACtD,QAAE,MAAM,WAAW;AACnB,aAAO,EAAE,SAAS,MAAM,SAAS,IAAI;AAAA,IACtC,CAAC;AAED,UAAM,IAAI,KAAK,WAAW,YAAY;AACrC,YAAM,IAAI,SAAyC,GAAG;AACtD,QAAE,MAAM,WAAW;AACnB,YAAM,MAAM,EAAE,MAAM,oBAAoB;AACxC,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,cAAc,KAAK,IAAI;AAC/B,aAAO,EAAE,WAAW,KAAK;AAAA,IAC1B,CAAC;AAAA,EACF,CAAC;AACF,CAAC;AAUD,IAAM,oBAAoB;AAEnB,IAAM,0BAA0BD,QAAM;AAAA,EAC5C,aAAa,CAAC,OAAqC;AAAA,IAClD,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,IACR,UAAU;AAAA,IACV,cAAc;AAAA,EACf;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,MAAoC,EAAE;AAAA,IACjD,cAAc,CAAC,GAAG,iBAA0B;AAC3C,QAAE,MAAM,eAAe,gBAAgB,EAAE,MAAM,WAAW;AAAA,IAC3D;AAAA,EACD;AAAA,EACA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,SAAS,YAAY;AACnC,YAAM,IAAI,SAAuC,GAAG;AACpD,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,OAAO,KAAK;AAAA,IACtB,CAAC;AAED,UAAM,IAAI,KAAK;AAAA,MACd,MAAM;AAAA,MACN,YAAY;AAAA,MACZ,kBAAkB;AAAA,MAClB,iBAAiB;AAAA,MACjB,KAAK,YAAY;AAChB,cAAM,IAAI,SAAuC,GAAG;AACpD,UAAE,MAAM,YAAY;AACpB,YAAI,EAAE,MAAM,WAAW,EAAE,MAAM,cAAc;AAC5C,gBAAM,QAAQ,IAAI,MAAM,iCAAiC;AACzD,YAAE,MAAM,YAAY,MAAM;AAC1B,gBAAM;AAAA,QACP;AACA,UAAE,MAAM,SAAS;AACjB,UAAE,MAAM,YAAY;AACpB,eAAO,EAAE,SAAS,KAAK;AAAA,MACxB;AAAA,IACD,CAAC;AAAA,EACF,CAAC;AACF,CAAC;AASD,IAAM,qBAAqB;AAEpB,IAAM,wBAAwBD,QAAM;AAAA,EAC1C,aAAa,CAAC,OAAmC;AAAA,IAChD,IAAI,EAAE,IAAI,CAAC;AAAA,IACX,QAAQ;AAAA,IACR,UAAU;AAAA,EACX;AAAA,EACA,SAAS;AAAA,IACR,UAAU,CAAC,MAAkC,EAAE;AAAA,EAChD;AAAA,EACA,KAAKC,UAAS,OAAO,QAAQ;AAC5B,UAAM,IAAI,KAAK,QAAQ,YAAY;AAClC,YAAM,IAAI,SAAqC,GAAG;AAClD,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,aAAa,KAAK;AAAA,IAC5B,CAAC;AAED,UAAM,IAAI,KAAK,YAAY,YAAY;AACtC,aAAO,EAAE,OAAO,KAAK;AAAA,IACtB,CAAC;AAED,UAAM,IAAI,KAAK;AAAA,MACd,MAAM;AAAA,MACN,YAAY;AAAA,MACZ,kBAAkB;AAAA,MAClB,iBAAiB;AAAA,MACjB,KAAK,YAAY;AAChB,cAAM,IAAI,SAAqC,GAAG;AAClD,UAAE,MAAM,YAAY;AACpB,cAAM,QAAQ,IAAI;AAAA,UACjB;AAAA,QACD;AACA,UAAE,MAAM,YAAY,MAAM;AAC1B,cAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF,CAAC;AACF,CAAC;;;ACjpBD,SAAS,SAAAG,eAAa;AAoCf,IAAM,YAAYA,QAAM;AAAA;AAAA,EAE9B,aAAa,CAAC,IAAI,WAA4C;AAAA,IAC7D,UAAU,OAAO,YAAY;AAAA,IAC7B,OAAO,OAAO,gBAAgB;AAAA,IAC9B,cAAc,CAAC;AAAA,EAChB;AAAA,EAEA,SAAS;AAAA;AAAA,IAER,UAAU,CAAC,OAAO;AAAA,MACjB,UAAU,EAAE,MAAM;AAAA,MAClB,OAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,cAAc,CAAC,GAAG,YAAoB,aAAqB;AAC1D,UAAI,EAAE,MAAM,QAAQ,UAAU;AAC7B,eAAO;AAAA,UACN,SAAS;AAAA,UACT,SAAS,kCAAkC,EAAE,MAAM,KAAK,gBAAgB,QAAQ;AAAA,UAChF,gBAAgB,EAAE,MAAM;AAAA,QACzB;AAAA,MACD;AAGA,QAAE,MAAM,SAAS;AACjB,QAAE,MAAM,aAAa,KAAK,UAAU;AAEpC,aAAO;AAAA,QACN,SAAS;AAAA,QACT,SAAS,YAAY,QAAQ,uBAAuB,UAAU;AAAA,QAC9D,gBAAgB,EAAE,MAAM;AAAA,MACzB;AAAA,IACD;AAAA;AAAA,IAGA,cAAc,CAAC,GAAG,YAAoB,aAAqB;AAC1D,YAAM,QAAQ,EAAE,MAAM,aAAa,QAAQ,UAAU;AACrD,UAAI,QAAQ,IAAI;AACf,UAAE,MAAM,aAAa,OAAO,OAAO,CAAC;AACpC,UAAE,MAAM,SAAS;AAAA,MAClB;AACA,aAAO;AAAA,QACN,SAAS;AAAA,QACT,gBAAgB,EAAE,MAAM;AAAA,MACzB;AAAA,IACD;AAAA,EACD;AACD,CAAC;AAGM,IAAM,WAAWA,QAAM;AAAA,EAC7B,aAAa,CAAC,IAAI,WAA0C;AAAA,IAC3D,cAAc,OAAO,gBAAgB;AAAA,IACrC,OAAO,CAAC;AAAA,IACR,WAAW;AAAA,EACZ;AAAA,EAEA,SAAS;AAAA;AAAA,IAER,SAAS,OACR,GACA,QACA,aAC6B;AAG7B,YAAM,iBAAiB,EAAE,OAAO,EAAE,UAAU,YAAY,CAAC,MAAM,CAAC;AAGhE,YAAM,WAAW,MAAM,eAAe,SAAS;AAG/C,YAAM,cAAc,MAAM,eAAe;AAAA,QACxC,EAAE;AAAA;AAAA,QACF;AAAA,MACD;AAEA,UAAI,CAAC,YAAY,SAAS;AACzB,eAAO;AAAA,UACN,SAAS;AAAA,UACT,SAAS,YAAY;AAAA,QACtB;AAAA,MACD;AAGA,QAAE,MAAM,MAAM,KAAK;AAAA,QAClB;AAAA,QACA,UAAU,SAAS;AAAA,QACnB;AAAA,MACD,CAAC;AAED,aAAO;AAAA,QACN,SAAS;AAAA,QACT,SAAS,SAAS,QAAQ,IAAI,SAAS,QAAQ;AAAA,QAC/C,gBAAgB,YAAY;AAAA,MAC7B;AAAA,IACD;AAAA;AAAA,IAGA,YAAY,CAAC,OAAO;AAAA,MACnB,cAAc,EAAE,MAAM;AAAA,MACtB,OAAO,EAAE,MAAM;AAAA,MACf,WAAW,EAAE,MAAM;AAAA,MACnB,YAAY,EAAE,MAAM,MAAM;AAAA,QACzB,CAAC,KAAK,SAAS,MAAM,KAAK;AAAA,QAC1B;AAAA,MACD;AAAA,IACD;AAAA;AAAA,IAGA,kBAAkB,CAAC,MAAM;AACxB,QAAE,MAAM,YAAY;AACpB,aAAO;AAAA,QACN,SAAS;AAAA,QACT,SAAS;AAAA,MACV;AAAA,IACD;AAAA;AAAA,IAGA,gBAAgB,OAAO,MAAM;AAE5B,iBAAW,QAAQ,EAAE,MAAM,OAAO;AACjC,cAAM,iBAAiB,EACrB,OAAO,EACP,UAAU,YAAY,CAAC,KAAK,MAAM,CAAC;AACrC,cAAM,eAAe,aAAa,EAAE,SAAS,KAAK,QAAQ;AAAA,MAC3D;AAGA,QAAE,MAAM,QAAQ,CAAC;AAEjB,aAAO;AAAA,QACN,SAAS;AAAA,QACT,SAAS;AAAA,MACV;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;AC/KD,SAAS,SAAAC,eAAa;AAOtB,SAAS,0BAAmC;AAC3C,SAAO;AACR;AAEA,eAAe,sBAAsB,YAIlC;AACF,MAAI,WAAW,eAAe,aAAa;AAC1C;AAAA,EACD;AAEA,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,UAAM,kBAAkB,WAAW,OAAO,MAAM;AAC/C,sBAAgB;AAChB,uBAAiB;AACjB,cAAQ;AAAA,IACT,CAAC;AACD,UAAM,mBAAmB,WAAW,QAAQ,CAAC,UAAU;AACtD,sBAAgB;AAChB,uBAAiB;AACjB,aAAO,KAAK;AAAA,IACb,CAAC;AAAA,EACF,CAAC;AACF;AAEO,IAAM,oBAAoBA,QAAM;AAAA,EACtC,OAAO,EAAE,UAAU,CAAC,EAAc;AAAA,EAClC,SAAS;AAAA;AAAA,IAER,sBAAsB,OAAO,GAAG,WAAmB;AAClD,YAAM,SAAS,EAAE,OAAwB;AACzC,YAAM,SAAS,MAAM,OAAO,QAC1B,YAAY,CAAC,aAAa,CAAC,EAC3B,UAAU,MAAM;AAClB,QAAE,MAAM,SAAS;AAAA,QAChB,4BAA4B,MAAM,cAAc,MAAM;AAAA,MACvD;AACA,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,iBAAiB,OAAO,MAAM;AAC7B,YAAM,SAAS,EAAE,OAAwB;AACzC,YAAM,QAAQ,MAAM,OAAO,QACzB,YAAY,CAAC,aAAa,CAAC,EAC3B,SAAS;AACX,QAAE,MAAM,SAAS,KAAK,sBAAsB,KAAK,EAAE;AACnD,aAAO;AAAA,IACR;AAAA;AAAA,IAGA,8BAA8B,OAAO,GAAG,WAAmB;AAC1D,YAAM,SAAS,EAAE,OAAwB;AACzC,YAAM,SAAS,OAAO,QAAQ,YAAY,CAAC,sBAAsB,CAAC;AAElE,UAAI,wBAAwB,GAAG;AAC9B,cAAMC,UAAmB,CAAC;AAC1B,cAAMC,WAAU,MAAM,OAAO,UAAU,MAAM;AAC7C,QAAAD,QAAO,KAAKC,QAAO;AACnB,cAAMC,WAAU,MAAM,OAAO,UAAU,SAAS,CAAC;AACjD,QAAAF,QAAO,KAAKE,QAAO;AAEnB,UAAE,MAAM,SAAS;AAAA,UAChB,wCAAwC,MAAM,QAAQ,SAAS,CAAC,cAAcD,QAAO,KAAKC,QAAO,aAAa,KAAK,UAAUF,OAAM,CAAC;AAAA,QACrI;AAEA,eAAO,EAAE,SAAAC,UAAS,SAAAC,UAAS,QAAAF,QAAO;AAAA,MACnC;AAEA,YAAM,OAAO,SAAS;AACtB,YAAM,aAAa,OAAO,QAAQ;AAClC,YAAM,sBAAsB,UAAU;AAGtC,YAAM,SAAmB,CAAC;AAC1B,iBAAW,GAAG,YAAY,CAAC,UAAkB;AAC5C,eAAO,KAAK,KAAK;AAAA,MAClB,CAAC;AAGD,YAAM,UAAU,MAAM,WAAW,UAAU,MAAM;AACjD,YAAM,UAAU,MAAM,WAAW,UAAU,SAAS,CAAC;AAErD,YAAM,WAAW,QAAQ;AAEzB,QAAE,MAAM,SAAS;AAAA,QAChB,wCAAwC,MAAM,QAAQ,SAAS,CAAC,cAAc,OAAO,KAAK,OAAO,aAAa,KAAK,UAAU,MAAM,CAAC;AAAA,MACrI;AAEA,aAAO,EAAE,SAAS,SAAS,OAAO;AAAA,IACnC;AAAA;AAAA,IAGA,aAAa,CAAC,MAAM;AACnB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA;AAAA,IAGA,eAAe,CAAC,MAAM;AACrB,QAAE,MAAM,WAAW,CAAC;AAAA,IACrB;AAAA,EACD;AACD,CAAC;;;AC/GD,SAAS,SAAAG,eAAa;AAEf,IAAM,cAAcA,QAAM;AAAA,EAChC,OAAO,EAAE,OAAO,EAAE;AAAA,EAClB,SAAS;AAAA,IACR,WAAW,CAAC,GAAG,SAAiB,MAAM;AACrC,QAAE,MAAM,SAAS;AACjB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,UAAU,CAAC,MAAM;AAChB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,IACA,OAAO,CAAC,MAAM;AACb,QAAE,MAAM,QAAQ;AAChB,aAAO,EAAE,MAAM;AAAA,IAChB;AAAA,EACD;AACD,CAAC;;;ACjBD,SAAS,SAAAC,eAAa;AACtB,SAAS,MAAAC,WAAU;AAEZ,IAAM,oBAAoBD,QAAM;AAAA,EACtC,IAAIC,IAAG;AAAA,IACN,WAAW,OAAOA,SAAO;AACxB,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKhB;AACD,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AAAA,EACD,SAAS;AAAA,IACR,WAAW,OAAO,GAAG,SAAiB,MAAM;AAC3C,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,MACD;AACA,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAQ,KAAK,CAAC,EAAwB;AAAA,IACvC;AAAA,IACA,UAAU,OAAO,MAAM;AACtB,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAQ,KAAK,CAAC,EAAwB;AAAA,IACvC;AAAA,IACA,OAAO,OAAO,MAAM;AACnB,YAAM,EAAE,GAAG,QAAQ,2CAA2C;AAC9D,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;;;ACvCD,SAAS,SAAAC,eAAa;AACtB,SAAS,MAAAC,WAAU;AAEZ,IAAM,iBAAiBD,QAAM;AAAA,EACnC,IAAIC,IAAG;AAAA,IACN,WAAW,OAAOA,SAAO;AAExB,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKhB;AACD,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAYhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUhB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AAGD,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAGA,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKhB;AACD,YAAMA,KAAG;AAAA,QACR;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AAAA,EACD,SAAS;AAAA,IACR,WAAW,OAAO,GAAG,SAAiB,MAAM;AAC3C,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,MACD;AACA,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAQ,KAAK,CAAC,EAAwB;AAAA,IACvC;AAAA,IACA,UAAU,OAAO,MAAM;AACtB,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAQ,KAAK,CAAC,EAAwB;AAAA,IACvC;AAAA,IACA,OAAO,OAAO,MAAM;AACnB,YAAM,EAAE,GAAG,QAAQ,2CAA2C;AAC9D,aAAO;AAAA,IACR;AAAA,IACA,aAAa,OAAO,MAAM;AACzB,YAAM,MAAM,KAAK,IAAI;AACrB,YAAM,UAAoB,CAAC;AAG3B,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA,QAAQ,GAAG;AAAA,QACX;AAAA,MACD;AACA,cAAQ,KAAK,eAAe;AAG5B,YAAM,QAAQ,MAAM,EAAE,GAAG;AAAA,QACxB;AAAA,QACA,QAAQ,GAAG;AAAA,MACZ;AACA,cAAQ,KAAK,iBAAkB,MAAM,CAAC,EAAuB,IAAI,EAAE;AACnE,YAAM,SAAU,MAAM,CAAC,EAAqB;AAG5C,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,kBAAkB;AAG/B,YAAM,WAAW,MAAM,EAAE,GAAG,QAAQ,iCAAiC;AACrE,cAAQ,KAAK,WAAY,SAAuB,MAAM,WAAW;AACjE,YAAM,YAAa,SAAS,CAAC,EAAqB;AAGlD,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA,YAAY,GAAG;AAAA,QACf;AAAA,MACD;AACA,cAAQ,KAAK,mBAAmB;AAGhC,YAAM,aAAa,MAAM,EAAE,GAAG,QAAQ,0BAA0B;AAChE,cAAQ,KAAK,WAAY,WAAyB,MAAM,aAAa;AAGrE,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,gBAAgB;AAG7B,YAAM,SAAS,MAAM,EAAE,GAAG;AAAA,QACzB;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,WAAY,OAAqB,MAAM,kBAAkB;AACtE,YAAM,UAAW,OAAO,CAAC,EAAqB;AAG9C,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,qBAAqB;AAGlC,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,oBAAoB;AAGjC,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,iBAAiB;AAG9B,YAAM,UAAU,MAAM,EAAE,GAAG;AAAA,QAC1B;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,WAAY,QAAsB,MAAM,UAAU;AAG/D,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,uBAAuB;AAGpC,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,yBAAyB,MAAM;AAAA,QAC/B;AAAA,MACD;AACA,cAAQ,KAAK,oBAAoB;AAGjC,YAAM,aAAa,MAAM,EAAE,GAAG;AAAA,QAC7B;AAAA,MACD;AACA,cAAQ;AAAA,QACP,gBAAiB,WAAyB,MAAM;AAAA,MACjD;AAGA,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,kBAAkB;AAG/B,YAAM,eAAe,MAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOvC;AACD,cAAQ;AAAA,QACP,WAAY,aAA2B,MAAM;AAAA,MAC9C;AAGA,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,cAAQ,KAAK,sBAAsB;AAGnC,YAAM,UAAU,MAAM,EAAE,GAAG;AAAA,QAC1B;AAAA,MACD;AACA,cAAQ;AAAA,QACP,mBAAoB,QAAQ,CAAC,EAA0B,OAAO;AAAA,MAC/D;AAGA,YAAM,cAAc,MAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMtC;AACD,cAAQ;AAAA,QACP,iBAAkB,YAA0B,MAAM;AAAA,MACnD;AAEA,aAAO;AAAA,QACN,YAAY;AAAA,QACZ;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;AC5qBD,SAAS,SAAAC,eAAa;AACtB,SAAS,MAAAC,WAAU;AAEnB,IAAM,uBAAuB,IAAI;AACjC,IAAM,6BAA6B;AAEnC,SAAS,oBAAoB,KAAa,aAA6B;AACtE,QAAM,SAAS,WAAW,GAAG;AAC7B,SAAO,SAAS,IAAI,OAAO,KAAK,IAAI,GAAG,cAAc,OAAO,MAAM,CAAC;AACpE;AAEA,eAAe,YACd,UAGA,aACC;AACD,QAAM,WAAW,QAAQ,OAAO,WAAW,CAAC;AAC5C,QAAM,gBAAgB,KAAK,IAAI;AAC/B,MAAI,iBAAiB;AACrB,MAAI,OAAO;AAEX,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI;AACH,WAAO,iBAAiB,GAAG;AAC1B,YAAM,eAAyB,CAAC;AAChC,YAAM,OAAkB,CAAC;AAEzB,eACK,aAAa,GACjB,aAAa,8BAA8B,iBAAiB,GAC5D,cACC;AACD,cAAM,eAAe,KAAK,IAAI,sBAAsB,cAAc;AAClE,cAAM,MAAM;AACZ,cAAM,OAAO,MAAM,MAAM,IAAI,SAAS;AAEtC,qBAAa,KAAK,uBAAuB;AACzC,aAAK;AAAA,UACJ;AAAA,UACA;AAAA,UACA;AAAA,UACA,oBAAoB,KAAK,YAAY;AAAA,UACrC;AAAA,UACA,KAAK,KAAK,eAAe,CAAC;AAAA,UAC1B,gBAAgB;AAAA,QACjB;AAEA,0BAAkB;AAClB;AAAA,MACD;AAEA,YAAM,SAAS;AAAA,QACd,0GAA0G,aAAa,KAAK,IAAI,CAAC;AAAA,QACjI,GAAG;AAAA,MACJ;AAAA,IACD;AAEA,UAAM,SAAS,QAAQ,QAAQ;AAAA,EAChC,SAAS,KAAK;AACb,UAAM,SAAS,QAAQ,UAAU;AACjC,UAAM;AAAA,EACP;AAEA,SAAO,EAAE,UAAU,MAAM,YAAY,YAAY;AAClD;AAEO,IAAM,kBAAkBD,QAAM;AAAA,EACpC,SAAS;AAAA,IACR,eAAe;AAAA,EAChB;AAAA,EACA,IAAIC,IAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAOrB;AACF,YAAM,SAAS,QAAQ,wDAAwD;AAC/E,YAAM,SAAS,QAAQ,wDAAwD;AAE/E,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA,KAGrB;AAEF,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAMrB;AAEF,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KASrB;AACF,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AAAA,EACD,SAAS;AAAA,IACR,MAAM,CAAC,QAAQ,EAAE,IAAI,KAAK;AAAA,IAE1B,WAAW,CAAC,MAAM;AACjB,QAAE,MAAM;AACR,aAAO,EAAE,IAAI,KAAK;AAAA,IACnB;AAAA,IAEA,cAAc,OAAO,GAAG,MAAc;AACrC,YAAM,KAAK,YAAY,IAAI;AAC3B,eAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC3B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,KAAK,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QACjC;AAAA,MACD;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,IAEA,UAAU,OAAO,GAAG,MAAc;AACjC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC3B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,KAAK,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QACjC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,IAEA,aAAa,OAAO,GAAG,MAAc;AACpC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,eAAe,MAAM,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,EAAE,KAAK,IAAI;AAC9E,YAAM,OAAkB,CAAC;AACzB,eAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC3B,aAAK,KAAK,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,KAAK,IAAI,CAAC;AAAA,MAC5C;AACA,YAAM,EAAE,GAAG,QAAQ,0DAA0D,YAAY,IAAI,GAAG,IAAI;AACpG,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,IAEA,WAAW,OAAO,GAAG,MAAc;AAClC,YAAM,EAAE,GAAG,QAAQ,2EAA2E;AAC9F,YAAM,OAAO,MAAM,EAAE,GAAG,QAAQ,+CAA+C;AAC/E,YAAM,KAAM,KAAK,CAAC,EAAqB;AACvC,YAAM,KAAK,YAAY,IAAI;AAC3B,eAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC3B,cAAM,EAAE,GAAG,QAAQ,oCAAoC,EAAE;AAAA,MAC1D;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,IAEA,UAAU,OAAO,GAAG,aAAqB;AACxC,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,QAAQ,CAAC;AAAA,UAAI,OAAO,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QACtC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG,QAAQ,qBAAqB;AACrD,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,MAAO,KAAmB,OAAO;AAAA,IAC/E;AAAA,IAEA,kBAAkB,OAAO,MAAM;AAC9B,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,MAAM,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QAClC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG,QAAQ,mDAAmD;AACnF,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,MAAO,KAAmB,OAAO;AAAA,IAC/E;AAAA,IAEA,oBAAoB,OAAO,MAAM;AAChC,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,MAAM,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QAClC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG,QAAQ,6DAA6D;AAC7F,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,MAAO,KAAmB,OAAO;AAAA,IAC/E;AAAA,IAEA,YAAY,OAAO,MAAM;AACxB,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,MAAM,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QAClC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,4EAA4E;AAC/F,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,KAAK,IAAI;AAAA,IACvD;AAAA,IAEA,YAAY,OAAO,MAAM;AACxB,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,MAAM,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QAClC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,yCAAyC;AAC5D,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,KAAK,IAAI;AAAA,IACvD;AAAA,IAEA,eAAe,OAAO,GAAG,MAAc;AACtC,YAAM,EAAE,GAAG,QAAQ,2EAA2E;AAC9F,YAAM,OAAO,MAAM,EAAE,GAAG,QAAQ,gDAAgD;AAChF,YAAM,KAAM,KAAK,CAAC,EAAqB;AACvC,YAAM,KAAK,YAAY,IAAI;AAC3B,eAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC3B,cAAM,EAAE,GAAG,QAAQ,yCAAyC,GAAG,EAAE;AAAA,MAClE;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,IAEA,mBAAmB,OAAO,MAAM;AAC/B,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,OAAO,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QACnC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,EAAE,GAAG,QAAQ,0CAA0C;AAC7D,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,OAAO;AAAA,IAC7C;AAAA,IAEA,oBAAoB,OAAO,GAAG,MAAc;AAC3C,YAAM,OAAO,IAAI,OAAO,KAAK,IAAI;AACjC,YAAM,KAAK,YAAY,IAAI;AAC3B,eAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC3B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,MAAM,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG;AAAA,UAAM,KAAK,IAAI;AAAA,QACxC;AAAA,MACD;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,IAEA,WAAW,OAAO,MAAM;AACvB,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QAAQ;AAAA,QAAW;AAAA,QAAG,KAAK,IAAI;AAAA,MAChC;AACA,YAAM,OAAO,MAAM,EAAE,GAAG,QAAQ,gDAAgD;AAChF,YAAM,KAAM,KAAK,CAAC,EAAqB;AACvC,YAAM,EAAE,GAAG,QAAQ,4DAA4D,EAAE;AACjF,YAAM,EAAE,GAAG,QAAQ,oCAAoC,EAAE;AACzD,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,IAEA,oBAAoB,OAAO,MAAM;AAChC,YAAM,KAAK,YAAY,IAAI;AAC3B,eAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC5B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,KAAK,UAAU,EAAE,MAAM,QAAQ,CAAC,IAAI,MAAM,CAAC,KAAK,GAAG,GAAG,OAAO,KAAK,OAAO,IAAI,IAAI,CAAC;AAAA,QACnF;AAAA,MACD;AACA,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,IAAI,MAAO,KAAmB,OAAO;AAAA,IAChF;AAAA,IAEA,aAAa,OAAO,MAAM;AACzB,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA,KAAK,UAAU,EAAE,OAAO,MAAM,KAAK,EAAE,QAAQ,IAAI,GAAG,CAAC,GAAG,OAAO,EAAE,IAAI,GAAG,KAAK,IAAI,GAAG,EAAE,EAAE,CAAC;AAAA,MAC1F;AACA,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,OAAQ,KAAK,CAAC,EAAwB,MAAM;AAAA,IAClF;AAAA,IAEA,oBAAoB,OAAO,MAAM;AAChC,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,OAAO,IAAI,EAAE;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QACxC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,QAAS,KAAmB,OAAO;AAAA,IACjF;AAAA,IAEA,iBAAiB,OAAO,MAAM;AAC7B,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,MAAM,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QAClC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,MAAO,KAAmB,OAAO;AAAA,IAC/E;AAAA,IAEA,aAAa,OAAO,MAAM;AACzB,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,KAAK,CAAC;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QACjC;AACA,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,IAAI;AAAA,UAAG,SAAS,CAAC;AAAA,UAAI,KAAK,OAAO,IAAI;AAAA,QACtC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,MAAO,KAAmB,OAAO;AAAA,IAC/E;AAAA,IAEA,kBAAkB,OAAO,MAAM;AAC9B,YAAM,SAAS,YAAY,IAAI;AAC/B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,OAAO,IAAI,EAAE;AAAA,UAAI,KAAK,CAAC;AAAA,UAAI;AAAA,UAAG,KAAK,IAAI;AAAA,QACxC;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,SAAS,YAAY,IAAI,IAAI;AACnC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ/B;AACD,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,QAAQ,MAAO,KAAmB,OAAO;AAAA,IAC/E;AAAA,IAEA,iBAAiB,OAAO,GAAG,MAAc;AACxC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC3B,cAAM,EAAE,GAAG,QAAQ,kCAAkC,CAAC;AAAA;AAAA;AAAA,MAGpD;AAAA,MACH;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,IAEA,eAAe,OAAO,GAAG,eAAuB;AAC/C,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,SAAS,MAAM,YAAY,EAAE,IAAI,UAAU;AACjD,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,OAAO,MAAM,OAAO,OAAO,WAAW;AAAA,IACjF;AAAA,IAEA,oBAAoB,OAAO,GAAG,eAAuB;AACpD,YAAM,SAAS,MAAM,YAAY,EAAE,IAAI,UAAU;AACjD,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,MACD;AACA,aAAO;AAAA,QACN,IAAI,YAAY,IAAI,IAAI;AAAA,QACxB,KAAM,KAAmB;AAAA,QACzB,MAAO,KAAmB;AAAA,QAC1B,OAAO,OAAO;AAAA,MACf;AAAA,IACD;AAAA,IAEA,sBAAsB,OAAO,GAAG,eAAuB;AACtD,YAAM,SAAS,MAAM,YAAY,EAAE,IAAI,UAAU;AACjD,YAAM,aAAa,KAAK,IAAI,GAAG,OAAO,OAAO,GAAG;AAChD,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,QACA,OAAO;AAAA,QACP;AAAA,MACD;AACA,aAAO;AAAA,QACN,IAAI,YAAY,IAAI,IAAI;AAAA,QACxB,KAAM,KAAmB;AAAA,QACzB,MAAO,KAAmB;AAAA,QAC1B,OAAO,OAAO;AAAA,MACf;AAAA,IACD;AAAA,IAEA,cAAc,OAAO,GAAG,eAAuB;AAC9C,YAAM,SAAS,MAAM,YAAY,EAAE,IAAI,UAAU;AACjD,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,QACA,OAAO;AAAA,MACR;AACA,aAAO;AAAA,QACN,IAAI,YAAY,IAAI,IAAI;AAAA,QACxB,KAAK;AAAA,QACL,OAAQ,KAAK,CAAC,EAAwB;AAAA,QACtC,OAAO,OAAO;AAAA,MACf;AAAA,IACD;AAAA,IAEA,YAAY,OAAO,GAAG,eAAuB;AAC5C,YAAM,SAAS,MAAM,YAAY,EAAE,IAAI,UAAU;AACjD,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,OAAO,MAAM,EAAE,GAAG;AAAA,QACvB;AAAA,QACA,OAAO;AAAA,MACR;AACA,aAAO;AAAA,QACN,IAAI,YAAY,IAAI,IAAI;AAAA,QACxB,KAAK;AAAA,QACL,YAAa,KAAK,CAAC,EAAqC,eAAe;AAAA,QACvE,OAAO,OAAO;AAAA,MACf;AAAA,IACD;AAAA,IAEA,oBAAoB,OAAO,MAAM;AAChC,YAAM,cAAc,MAAM;AAC1B,YAAM,UAAU,IAAI;AACpB,YAAM,WAAW,KAAK,KAAK,cAAc,OAAO;AAChD,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,UAAU,OAAO,WAAW,QAAQ;AAAA,IAC/E;AAAA,IAEA,kBAAkB,OAAO,MAAM;AAC9B,YAAM,cAAc,OAAO;AAC3B,YAAM,UAAU,IAAI;AACpB,YAAM,WAAW,KAAK,KAAK,cAAc,OAAO;AAChD,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,UAAU,OAAO,WAAW,QAAQ;AAAA,IAC/E;AAAA;AAAA,IAGA,0BAA0B,OAAO,MAAM;AACtC,YAAM,cAAc,OAAO;AAC3B,YAAM,UAAU;AAChB,YAAM,WAAW,KAAK,KAAK,cAAc,OAAO;AAChD,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,UAAU,OAAO,WAAW,QAAQ;AAAA,IAC/E;AAAA;AAAA,IAGA,4BAA4B,OAAO,MAAM;AACxC,YAAM,cAAc,OAAO;AAC3B,YAAM,UAAU,IAAI;AACpB,YAAM,WAAW,KAAK,KAAK,cAAc,OAAO;AAChD,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,UAAU,OAAO,WAAW,QAAQ;AAAA,IAC/E;AAAA;AAAA,IAGA,wBAAwB,OAAO,MAAM;AACpC,YAAM,UAAU,OAAO;AACvB,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,GAAG,OAAO,QAAQ;AAAA,IAC7D;AAAA,IAEA,kBAAkB,OAAO,MAAM;AAC9B,YAAM,cAAc,IAAI,OAAO;AAC/B,YAAM,UAAU,IAAI;AACpB,YAAM,WAAW,KAAK,KAAK,cAAc,OAAO;AAChD,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,UAAU,OAAO,WAAW,QAAQ;AAAA,IAC/E;AAAA,IAEA,mBAAmB,OAAO,MAAM;AAC/B,YAAM,cAAc,KAAK,OAAO;AAChC,YAAM,UAAU,IAAI;AACpB,YAAM,WAAW,KAAK,KAAK,cAAc,OAAO;AAChD,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,UAAU,OAAO,WAAW,QAAQ;AAAA,IAC/E;AAAA,IAEA,mBAAmB,OAAO,MAAM;AAC/B,YAAM,cAAc,KAAK,OAAO;AAChC,YAAM,UAAU,IAAI;AACpB,YAAM,WAAW,KAAK,KAAK,cAAc,OAAO;AAChD,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,UAAU,OAAO,WAAW,QAAQ;AAAA,IAC/E;AAAA;AAAA;AAAA,IAIA,mBAAmB,OAAO,MAAM;AAC/B,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,SAAS;AACf,YAAM,WAAW;AACjB,eAAS,QAAQ,GAAG,QAAQ,QAAQ,SAAS;AAC5C,cAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,iBAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AAAA,QACD;AACA,cAAM,EAAE,GAAG,QAAQ,mBAAmB;AACtC,cAAM,EAAE,GAAG,QAAQ,QAAQ;AAAA,MAC5B;AACA,aAAO;AAAA,QACN,IAAI,YAAY,IAAI,IAAI;AAAA,QACxB,KAAK,SAAS;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA;AAAA;AAAA,IAIA,gBAAgB,OAAO,MAAM;AAC5B,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,KAIjB;AACF,YAAM,EAAE,GAAG,QAAQ,wBAAwB;AAC3C,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,UACA,IAAI;AAAA,QACL;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAE3B,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,MAAM;AAAA,UACN,IAAI;AAAA,QACL;AACA,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AACA,YAAI,IAAI,MAAM,GAAG;AAChB,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,YACA,IAAI,MAAM,IAAI,IAAI,KAAK;AAAA,UACxB;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI;AAAA,IACzD;AAAA;AAAA;AAAA,IAIA,oBAAoB,OAAO,MAAM;AAChC,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,EAAE,GAAG,QAAQ,sBAAsB;AACzC,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,UAAU;AAChB,YAAM,WAAW;AACjB,UAAI,UAAU;AACd,eAASC,SAAQ,GAAGA,SAAQ,SAASA,UAAS;AAC7C,cAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,iBAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AAClC,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,YACAA,SAAQ,WAAW;AAAA,UACpB;AAAA,QACD;AACA,cAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,cAAM,OAAQ,MAAM,EAAE,GAAG;AAAA,UACxB;AAAA,QACD;AACA,kBAAU,KAAK,CAAC,GAAG,KAAK;AAAA,MACzB;AACA,aAAO;AAAA,QACN,IAAI,YAAY,IAAI,IAAI;AAAA,QACxB,KAAK,UAAU;AAAA,QACf;AAAA,QACA;AAAA,MACD;AAAA,IACD;AAAA;AAAA,IAGA,2BAA2B,OAAO,MAAM;AACvC,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,KAIjB;AACF,YAAM,EAAE,GAAG,QAAQ,mCAAmC;AACtD,YAAM,EAAE,GAAG,QAAQ,sBAAsB;AACzC,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAO,KAAK;AAC/B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA,OAAO,IAAI,GAAI,IAAI,CAAC;AAAA,UACpB;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,4CAA4C;AAC/D,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,IAAM;AAAA,IACjD;AAAA;AAAA;AAAA,IAIA,oBAAoB,OAAO,MAAM;AAChC,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AACF,YAAM,EAAE,GAAG,QAAQ,yBAAyB;AAC5C,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAM,KAAK;AAC9B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAE3B,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAM,KAAK;AAC9B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,IAAK;AAAA,IAChD;AAAA;AAAA,IAGA,mBAAmB,OAAO,MAAM;AAC/B,YAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGjB;AAEF,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAE3B,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,oBAAoB;AACvC,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,IAAI;AAAA,IAC/C;AAAA;AAAA,IAGA,iBAAiB,OAAO,MAAM;AAC7B,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC5B,cAAM,EAAE,GAAG;AAAA,UACV,sCAAsC,CAAC;AAAA,QACxC;AACA,iBAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC5B,gBAAM,EAAE,GAAG;AAAA,YACV,uBAAuB,CAAC;AAAA,YACxB;AAAA,YACA,IAAI;AAAA,UACL;AAAA,QACD;AAAA,MACD;AACA,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO,EAAE,IAAI,YAAY,IAAI,IAAI,IAAI,KAAK,KAAK,IAAI,QAAQ,GAAG;AAAA,IAC/D;AAAA,EACD;AACD,CAAC;;;ACj1BD,SAAS,mBAAmB;AAC5B,SAAS,SAAAC,eAAa;AACtB,SAAS,MAAAC,WAAU;AAEnB,IAAM,uBAAuB,KAAK,OAAO;AACzC,IAAM,oBAAoB,KAAK;AAC/B,IAAM,qBAAqB;AAC3B,IAAM,4BAA4B,KAAK;AACvC,IAAM,kBAAkB;AACxB,IAAM,qBAAqB,KAAK;AAChC,IAAM,gBAAgB;AACtB,IAAM,sBAAsB;AAsB5B,SAAS,gBAAgB,OAA2B,UAAkB,MAAc;AACnF,QAAM,WAAW,SAAS;AAC1B,MAAI,CAAC,OAAO,UAAU,QAAQ,KAAK,WAAW,GAAG;AAChD,UAAM,IAAI,MAAM,GAAG,IAAI,6BAA6B;AAAA,EACrD;AACA,SAAO;AACR;AAEA,SAAS,kBAAkB,OAAuB;AACjD,SAAO,YAAY,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,SAAS,KAAK,EAAE,MAAM,GAAG,KAAK;AACxE;AAEA,eAAe,aACd,UAGA,YAAoC,WACnC;AACD,QAAM,KAAK,YAAY,IAAI;AAC3B,QAAM,CAAC,MAAM,IAAK,MAAM,SAAS;AAAA,IAChC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,UAOQ,aAAa;AAAA;AAAA,EAEtB;AAEA,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,6BAA6B;AAE1D,MAAI,OAAO;AACX,MAAI,QAAQ;AACZ,MAAI,gBAAgB;AACpB,MAAI,SAAS;AACb,QAAM,QAAQ,OAAO,UAAU;AAC/B,QAAM,QAAQ,OAAO,UAAU;AAE/B,MAAI,cAAc,YAAY;AAC7B,UAAM,CAAC,WAAW,IAAK,MAAM,SAAS;AAAA,MACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAOQ,mBAAmB;AAAA;AAAA,IAE5B;AACA,QAAI,CAAC,YAAa,OAAM,IAAI,MAAM,sCAAsC;AACxE,UAAM,aAAa,YAAY,UAAU;AACzC,UAAM,aAAa,YAAY,UAAU;AAEzC,aACK,UAAU,YACd,WAAW,cAAc,UAAU,GACnC,WAAW,iBACV;AACD,YAAM,UAAU,KAAK,IAAI,YAAY,UAAU,kBAAkB,CAAC;AAClE,YAAM,YAAa,MAAM,SAAS;AAAA,QACjC;AAAA;AAAA;AAAA;AAAA,YAIQ,mBAAmB;AAAA;AAAA;AAAA;AAAA,QAI3B;AAAA,QACA;AAAA,MACD;AAEA,iBAAW,OAAO,WAAW;AAC5B,gBAAQ;AACR,iBAAS,IAAI;AACb,yBAAiB,IAAI;AAAA,MACtB;AACA,gBAAU;AAAA,IACX;AAEA,WAAO;AAAA,MACN,IAAI,YAAY,IAAI,IAAI;AAAA,MACxB,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,MACf;AAAA,IACD;AAAA,EACD;AAEA,WACK,UAAU,OACd,WAAW,OACX,WAAW,iBACV;AACD,UAAM,UAAU,UAAU,kBAAkB;AAC5C,UAAM,CAAC,KAAK,IAAK,MAAM,SAAS;AAAA,MAC/B;AAAA;AAAA;AAAA;AAAA;AAAA,WAKQ,aAAa;AAAA;AAAA;AAAA,MAGrB;AAAA,MACA;AAAA,IACD;AACA,QAAI,CAAC,MAAO,OAAM,IAAI,MAAM,qCAAqC;AAEjE,YAAQ,MAAM;AACd,aAAS,MAAM;AACf,qBAAiB,MAAM;AACvB,cAAU;AAAA,EACX;AAEA,SAAO;AAAA,IACN,IAAI,YAAY,IAAI,IAAI;AAAA,IACxB,KAAK;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,eAAe;AAAA,EAChB;AACD;AAEO,IAAM,uBAAuBD,QAAM;AAAA,EACzC,SAAS;AAAA,IACR,eAAe;AAAA,EAChB;AAAA,EACA,IAAIC,IAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AAAA,IACF;AAAA,EACD,CAAC;AAAA,EACD,SAAS;AAAA,IACR,OAAO,OAAO,MAAM;AACnB,YAAM,EAAE,GAAG,QAAQ,eAAe,aAAa,EAAE;AACjD,YAAM,EAAE,GAAG,QAAQ,eAAe,mBAAmB,EAAE;AACvD,aAAO,EAAE,IAAI,KAAK;AAAA,IACnB;AAAA,IAEA,oBAAoB,OAAO,GAAG,QAAoB,CAAC,MAAM;AACxD,YAAM,cAAc;AAAA,QACnB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACD;AACA,YAAM,WAAW,gBAAgB,MAAM,UAAU,mBAAmB,UAAU;AAC9E,YAAM,YAAY;AAAA,QACjB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACD;AACA,YAAM,mBAAmB;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACD;AACA,YAAM,YAAY,KAAK,IAAI;AAC3B,UAAI,iBAAiB;AACrB,UAAI,OAAO;AACX,UAAI,eAAe;AACnB,UAAI,iBAAiB;AACrB,UAAI,iBAAiB;AACrB,UAAI,WAAW;AACf,UAAI,gBAAgB;AAEpB,YAAM,SAAS,YAAY,IAAI;AAC/B,UAAI;AACH,eAAO,iBAAiB,GAAG;AAC1B,cAAI,4BAA4B,KAAK;AAAA,YACpC;AAAA,YACA;AAAA,UACD;AACA,gBAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,0BAAgB;AAChB,0BAAgB;AAEhB,iBAAO,4BAA4B,GAAG;AACrC,kBAAM,eAAyB,CAAC;AAChC,kBAAM,OAAkB,CAAC;AACzB,kBAAM,aAAa,YAAY,IAAI;AAEnC,qBACK,aAAa,GACjB,aAAa,aACb,4BAA4B,KAC5B,iBAAiB,GACjB,cAAc,GACb;AACD,oBAAM,eAAe,KAAK;AAAA,gBACzB;AAAA,gBACA;AAAA,gBACA;AAAA,cACD;AACA,2BAAa,KAAK,WAAW;AAC7B,mBAAK;AAAA,gBACJ,kBAAkB,YAAY;AAAA,gBAC9B;AAAA,gBACA,YAAY;AAAA,cACb;AACA,2CAA6B;AAC7B,gCAAkB;AAClB,sBAAQ;AAAA,YACT;AAEA,8BAAkB,YAAY,IAAI,IAAI;AACtC,kBAAM,WAAW,YAAY,IAAI;AACjC,kBAAM,EAAE,GAAG;AAAA,cACV,eAAe,aAAa,gDAAgD,aAAa,KAAK,IAAI,CAAC;AAAA,cACnG,GAAG;AAAA,YACJ;AACA,8BAAkB,YAAY,IAAI,IAAI;AAAA,UACvC;AAEA,gBAAM,WAAW,YAAY,IAAI;AACjC,gBAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,sBAAY,YAAY,IAAI,IAAI;AAChC,0BAAgB;AAAA,QACjB;AAEA,cAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,wBAAgB;AAChB,iBACK,UAAU,GACd,WAAW,oBACX,WAAW,KACV;AACD,gBAAM,UAAU,KAAK,IAAI,oBAAoB,UAAU,GAAG;AAC1D,gBAAM,eAAyB,CAAC;AAChC,gBAAM,OAAkB,CAAC;AACzB,mBAAS,KAAK,SAAS,MAAM,SAAS,MAAM,GAAG;AAC9C,yBAAa,KAAK,QAAQ;AAC1B,iBAAK,KAAK,IAAI,CAAC;AAAA,UAChB;AACA,gBAAM,WAAW,YAAY,IAAI;AACjC,gBAAM,EAAE,GAAG;AAAA,YACV,eAAe,mBAAmB,wBAAwB,aAAa,KAAK,IAAI,CAAC;AAAA,YACjF,GAAG;AAAA,UACJ;AACA,4BAAkB,YAAY,IAAI,IAAI;AAAA,QACvC;AACA,cAAM,kBAAkB,YAAY,IAAI;AACxC,cAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,oBAAY,YAAY,IAAI,IAAI;AAChC,wBAAgB;AAEhB,eAAO;AAAA,UACN,IAAI,iBAAiB;AAAA,UACrB,aAAa,YAAY,IAAI,IAAI;AAAA,UACjC;AAAA,UACA;AAAA,UACA;AAAA,UACA,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA;AAAA,UACA;AAAA,UACA,kBAAkB;AAAA,QACnB;AAAA,MACD,SAAS,KAAK;AACb,YAAI,eAAe;AAClB,gBAAM,EAAE,GAAG,QAAQ,UAAU;AAAA,QAC9B;AACA,cAAM;AAAA,MACP;AAAA,IACD;AAAA,IAEA,SAAS,OAAO,MAAM;AACrB,aAAO,aAAa,EAAE,EAAE;AAAA,IACzB;AAAA,IAEA,gBAAgB,OAAO,MAAM;AAC5B,aAAO,aAAa,EAAE,IAAI,UAAU;AAAA,IACrC;AAAA,IAEA,YAAY,OAAO,MAAM;AACxB,YAAM,KAAK,YAAY,IAAI;AAC3B,YAAM,CAAC,GAAG,IAAK,MAAM,EAAE,GAAG;AAAA,QACzB,gCAAgC,aAAa;AAAA,MAC9C;AACA,aAAO;AAAA,QACN,IAAI,YAAY,IAAI,IAAI;AAAA,QACxB,MAAM,KAAK,QAAQ;AAAA,MACpB;AAAA,IACD;AAAA,IAEA,WAAW,CAAC,MAAM;AACjB,QAAE,MAAM;AACR,aAAO,EAAE,IAAI,KAAK;AAAA,IACnB;AAAA,EACD;AACD,CAAC;;;AC3VD,SAAS,SAAAC,eAAa;AACtB,SAAS,MAAAC,WAAU;AAEnB,IAAMC,qBAAoB,IAAI;AAC9B,IAAM,mBAAmB;AACzB,IAAM,iBAAiB;AACvB,IAAM,oBAAoB;AAC1B,IAAM,mBAAmB;AACzB,IAAM,mBAAmB;AACzB,IAAM,yBAAyB;AAC/B,IAAM,iBAAiB;AACvB,IAAMC,wBAAuB,IAAI;AACjC,IAAMC,8BAA6B;AACnC,IAAM,iBAAiB;AACvB,IAAM,sBAAsB;AAC5B,IAAM,2BAA2B;AACjC,IAAM,wBAAwB;AAC9B,IAAM,oBAAoB;AAC1B,IAAM,uBAAuB;AAC7B,IAAM,sBAAsB;AAK5B,IAAM,YAAY;AAAA,EACjB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAyCA,SAASC,iBAAgB,OAA2B,UAAkB,MAAc;AACnF,QAAM,WAAW,SAAS;AAC1B,MAAI,CAAC,OAAO,UAAU,QAAQ,KAAK,WAAW,GAAG;AAChD,UAAM,IAAI,MAAM,GAAG,IAAI,6BAA6B;AAAA,EACrD;AACA,SAAO;AACR;AAEA,SAAS,eAAe,UAAoD;AAC3E,MAAI,CAAE,UAAgC,SAAS,QAAQ,GAAG;AACzD,UAAM,IAAI,MAAM,sCAAsC,QAAQ,EAAE;AAAA,EACjE;AACD;AAEA,SAAS,aAAa,OAAe;AACpC,SAAO,KAAK,KAAK,QAAQ,YAAY,UAAU,MAAM;AACtD;AAEA,SAAS,UAAU,OAAe;AACjC,SAAO,aAAa,KAAK,EAAE,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AACxD;AAEA,SAAS,QAAQ,QAAgB,OAAe;AAC/C,SAAO,SAAS,IAAI,OAAO,KAAK,IAAI,GAAG,QAAQ,OAAO,MAAM,CAAC;AAC9D;AAEA,SAAS,UAAa,MAAsB;AAC3C,SAAO;AACR;AAEA,eAAe,eAAe,UAE3B;AACF,QAAM,CAAC,GAAG,IAAI,UAAwB,MAAM,SAAS,QAAQ,mBAAmB,CAAC;AACjF,SAAO,KAAK,cAAc;AAC3B;AAEA,eAAe,cAAc,UAE1B;AACF,QAAM,SAAS,QAAQ,4BAA4B;AACnD,QAAM,SAAS,QAAQ,uBAAuB;AAC9C,QAAM,SAAS,QAAQ,0BAA0B;AACjD,QAAM,SAAS,QAAQ,uBAAuB;AAC/C;AAEA,eAAe,UAAU,UAEtB;AACF,QAAM,SAAS,QAAQ,qBAAqB;AAC7C;AAEA,eAAe,YAAY,UAExB;AACF,QAAM,SAAS,QAAQ,uBAAuB;AAC/C;AAEA,eAAe,aAAa,UAEzB;AACF,QAAM,SAAS,QAAQ,yBAAyB;AACjD;AAEA,eAAe,aAAa,UAEzB;AACF,QAAM,SAAS,QAAQ,uBAAuB;AAC9C,QAAM,SAAS,QAAQ,kBAAkB;AACzC,QAAM,SAAS,QAAQ,oBAAoB;AAC3C,QAAM,SAAS,QAAQ,gBAAgB;AACvC,QAAM,SAAS,QAAQ,mBAAmB;AAC1C,QAAM,SAAS,QAAQ,kBAAkB;AAC1C;AAEA,eAAe,eAAe,UAE3B;AACF,QAAM,SAAS,QAAQ,sDAAsD;AAC7E,QAAM,SAAS,QAAQ,sDAAsD;AAC7E,QAAM,SAAS,QAAQ,2DAA2D;AAClF,QAAM,SAAS,QAAQ,2DAA2D;AAClF,QAAM,SAAS,QAAQ,0DAA0D;AACjF,QAAM,SAAS,QAAQ,kDAAkD;AACzE,QAAM,SAAS,QAAQ,0CAA0C;AACjE,QAAM,SAAS,QAAQ,yCAAyC;AAChE,QAAM,SAAS,QAAQ,yCAAyC;AACjE;AAEA,eAAe,gBACd,UAGA,IACC;AACD,MAAI,gBAAgB;AACpB,QAAM,SAAS,QAAQ,OAAO;AAC9B,kBAAgB;AAChB,MAAI;AACH,UAAM,GAAG;AACT,UAAM,SAAS,QAAQ,QAAQ;AAC/B,oBAAgB;AAAA,EACjB,SAAS,KAAK;AACb,QAAI,eAAe;AAClB,YAAM,SAAS,QAAQ,UAAU,EAAE,MAAM,MAAM,MAAS;AAAA,IACzD;AACA,UAAM;AAAA,EACP;AACD;AAEA,eAAe,aACd,UAGA,aACA,UACC;AACD,QAAM,cAAc,QAAQ;AAC5B,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,KAAK,cAAc,QAAQ,CAAC;AAC1D,QAAM,gBAAgB,KAAK,IAAI,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;AACvD,QAAM,YAAY,YAAY,IAAI;AAElC,QAAM,gBAAgB,UAAU,YAAY;AAC3C,aAAS,SAAS,GAAG,SAAS,eAAe,UAAU,kBAAkB;AACxE,YAAM,eAAyB,CAAC;AAChC,YAAM,OAAkB,CAAC;AACzB,YAAM,WAAW,KAAK,IAAI,eAAe,SAAS,gBAAgB;AAClE,eAAS,IAAI,QAAQ,IAAI,UAAU,KAAK,GAAG;AAC1C,qBAAa,KAAK,iBAAiB;AACnC,aAAK;AAAA,UACJ,IAAI;AAAA,UACJ,QAAQ,IAAI,EAAE;AAAA,UACd,QAAQ,UAAU,CAAC,CAAC;AAAA,UACpB,CAAC,QAAQ,OAAO,QAAQ,YAAY,EAAE,IAAI,CAAC;AAAA,UAC3C,CAAC,OAAO,OAAO,OAAO,KAAK,EAAE,IAAI,CAAC;AAAA,QACnC;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd,yEAAyE,aAAa,KAAK,IAAI,CAAC;AAAA,QAChG,GAAG;AAAA,MACJ;AAAA,IACD;AAAA,EACD,CAAC;AAED,WAAS,UAAU,GAAG,UAAU,MAAM,WAAW,wBAAwB;AACxE,UAAM,QAAQ,KAAK,IAAI,MAAM,UAAU,sBAAsB;AAC7D,UAAM,gBAAgB,UAAU,YAAY;AAC3C,eAAS,SAAS,SAAS,SAAS,OAAO,UAAU,kBAAkB;AACtE,cAAM,oBAA8B,CAAC;AACrC,cAAM,YAAuB,CAAC;AAC9B,cAAM,mBAA6B,CAAC;AACpC,cAAM,WAAsB,CAAC;AAC7B,cAAM,oBAA8B,CAAC;AACrC,cAAM,YAAuB,CAAC;AAC9B,cAAM,WAAW,KAAK,IAAI,OAAO,SAAS,gBAAgB;AAE1D,iBAAS,IAAI,QAAQ,IAAI,UAAU,KAAK,GAAG;AAC1C,gBAAM,KAAK,IAAI;AACf,gBAAM,aAAc,aAAa,CAAC,IAAI,gBAAiB;AACvD,gBAAM,YAAY,QAAoB,IAAI;AAC1C,gBAAM,SAAS,CAAC,WAAW,QAAQ,WAAW,UAAU,EAAE,IAAI,CAAC;AAC/D,gBAAM,aAAa,MAAO,aAAa,IAAI,EAAE,IAAI;AACjD,gBAAM,OAAO,QAAQ,SAAS,EAAE,IAAI,MAAM,KAAK,QAAQ;AAEvD,4BAAkB,KAAK,uBAAuB;AAC9C,oBAAU;AAAA,YACT;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,IAAI;AAAA,YACJ;AAAA,UACD;AAEA,mBAAS,OAAO,GAAG,OAAO,GAAG,QAAQ,GAAG;AACvC,6BAAiB,KAAK,iBAAiB;AACvC,qBAAS;AAAA,cACR;AAAA,cACA,OAAO,UAAU,IAAI,IAAI,EAAE,MAAM,GAAG,CAAC,CAAC;AAAA,cACtC,KAAM,IAAI,QAAQ;AAAA,cAClB,MAAO,aAAa,IAAI,OAAO,EAAE,IAAI;AAAA,cACrC;AAAA,YACD;AAAA,UACD;AAEA,4BAAkB,KAAK,iBAAiB;AACxC,oBAAU;AAAA,YACT,QAAQ,aAAa,EAAE;AAAA,YACvB,CAAC,SAAS,YAAY,UAAU,UAAU,EAAE,IAAI,CAAC;AAAA,YACjD;AAAA,YACA,SAAS,EAAE;AAAA,YACX,QAAQ,SAAS,EAAE,KAAK,KAAK,IAAI,UAAU,GAAG,CAAC;AAAA,UAChD;AAAA,QACD;AAEA,cAAM,SAAS;AAAA,UACd,gGAAgG,kBAAkB,KAAK,IAAI,CAAC;AAAA,UAC5H,GAAG;AAAA,QACJ;AACA,cAAM,SAAS;AAAA,UACd,qFAAqF,iBAAiB,KAAK,IAAI,CAAC;AAAA,UAChH,GAAG;AAAA,QACJ;AACA,cAAM,SAAS;AAAA,UACd,6FAA6F,kBAAkB,KAAK,IAAI,CAAC;AAAA,UACzH,GAAG;AAAA,QACJ;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AAEA,SAAO;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,YAAY,IAAI,IAAI;AAAA,IAC7B,WAAW,MAAM,eAAe,QAAQ;AAAA,EACzC;AACD;AAEA,eAAe,SACd,UAGA,aACA,UACC;AACD,QAAM,UAAU,QAAQ;AACxB,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,KAAK,cAAc,QAAQ,CAAC;AAC1D,QAAM,YAAY,YAAY,IAAI;AAElC,WAAS,UAAU,GAAG,UAAU,MAAM,WAAW,wBAAwB;AACxE,UAAM,QAAQ,KAAK,IAAI,MAAM,UAAU,sBAAsB;AAC7D,UAAM,gBAAgB,UAAU,YAAY;AAC3C,eAAS,SAAS,SAAS,SAAS,OAAO,UAAU,gBAAgB;AACpE,cAAM,eAAyB,CAAC;AAChC,cAAM,OAAkB,CAAC;AACzB,cAAM,WAAW,KAAK,IAAI,OAAO,SAAS,cAAc;AACxD,iBAAS,IAAI,QAAQ,IAAI,UAAU,KAAK,GAAG;AAC1C,gBAAM,OAAO,aAAa,CAAC;AAC3B,gBAAM,OAAO,QAAQ,OAAO,CAAC,IAAI,IAAI,KAAK,QAAQ;AAClD,uBAAa,KAAK,iBAAiB;AACnC,eAAK;AAAA,YACJ,OAAO,UAAU,CAAC,CAAC;AAAA,YACnB;AAAA,YACA,UAAU,OAAO,GAAG;AAAA,YACpB;AAAA,YACA;AAAA,UACD;AAAA,QACD;AACA,cAAM,SAAS;AAAA,UACd,oFAAoF,aAAa,KAAK,IAAI,CAAC;AAAA,UAC3G,GAAG;AAAA,QACJ;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AAEA,SAAO;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,YAAY,IAAI,IAAI;AAAA,IAC7B,WAAW,MAAM,eAAe,QAAQ;AAAA,EACzC;AACD;AAEA,eAAe,WACd,UAGA,aACA,UACC;AACD,QAAM,YAAY,QAAQ;AAC1B,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,KAAK,cAAc,QAAQ,CAAC;AAC1D,QAAM,YAAY,YAAY,IAAI;AAElC,WAAS,UAAU,GAAG,UAAU,MAAM,WAAW,wBAAwB;AACxE,UAAM,QAAQ,KAAK,IAAI,MAAM,UAAU,sBAAsB;AAC7D,UAAM,gBAAgB,UAAU,YAAY;AAC3C,eAAS,SAAS,SAAS,SAAS,OAAO,UAAU,mBAAmB;AACvE,cAAM,eAAyB,CAAC;AAChC,cAAM,OAAkB,CAAC;AACzB,cAAM,WAAW,KAAK,IAAI,OAAO,SAAS,iBAAiB;AAC3D,iBAAS,IAAI,QAAQ,IAAI,UAAU,KAAK,GAAG;AAC1C,gBAAM,YAAY,QAAQ,OAAO,IAAI,GAAG,EAAE,SAAS,GAAG,GAAG,CAAC;AAC1D,gBAAM,UAAU,KAAK,MAAM,IAAI,GAAG,IAAI;AACtC,uBAAa,KAAK,iBAAiB;AACnC,eAAK;AAAA,YACJ;AAAA,YACA;AAAA,aACC,IAAI,MAAM,IAAI,IAAI,OAAO,MAAO,IAAI;AAAA,YACrC,QAAoB,IAAI;AAAA,YACxB,QAAQ,UAAU,SAAS,IAAI,OAAO,KAAK,KAAK,IAAI,UAAU,GAAG,CAAC;AAAA,UACnE;AAAA,QACD;AACA,cAAM,SAAS;AAAA,UACd,uFAAuF,aAAa,KAAK,IAAI,CAAC;AAAA,UAC9G,GAAG;AAAA,QACJ;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AAEA,SAAO;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,YAAY,IAAI,IAAI;AAAA,IAC7B,WAAW,MAAM,eAAe,QAAQ;AAAA,EACzC;AACD;AAEA,SAASC,qBAAoB,KAAa,aAA6B;AACtE,QAAM,SAAS,WAAW,GAAG;AAC7B,SAAO,SAAS,IAAI,OAAO,KAAK,IAAI,GAAG,cAAc,OAAO,MAAM,CAAC;AACpE;AAEA,eAAeC,aACd,UAGA,aACC;AACD,QAAM,aAAa,QAAQ;AAC3B,QAAM,gBAAgB;AACtB,MAAI,iBAAiB;AACrB,MAAI,OAAO;AACX,QAAM,YAAY,YAAY,IAAI;AAElC,QAAM,gBAAgB,UAAU,YAAY;AAC3C,WAAO,iBAAiB,GAAG;AAC1B,YAAM,eAAyB,CAAC;AAChC,YAAM,OAAkB,CAAC;AAEzB,eACK,aAAa,GACjB,aAAaH,+BAA8B,iBAAiB,GAC5D,cAAc,GACb;AACD,cAAM,eAAe,KAAK,IAAID,uBAAsB,cAAc;AAClE,cAAM,MAAM;AACZ,cAAM,OAAO,MAAM,MAAM,IAAI,SAAS;AAEtC,qBAAa,KAAK,uBAAuB;AACzC,aAAK;AAAA,UACJ;AAAA,UACA;AAAA,UACA;AAAA,UACAG,qBAAoB,KAAK,YAAY;AAAA,UACrC;AAAA,UACA,KAAK,KAAK,eAAe,CAAC;AAAA,UAC1B,gBAAgB;AAAA,QACjB;AAEA,0BAAkB;AAClB,gBAAQ;AAAA,MACT;AAEA,YAAM,SAAS;AAAA,QACd,6GAA6G,aAAa,KAAK,IAAI,CAAC;AAAA,QACpI,GAAG;AAAA,MACJ;AAAA,IACD;AAAA,EACD,CAAC;AAED,SAAO;AAAA,IACN;AAAA,IACA;AAAA,IACA,UAAUH;AAAA,IACV,SAAS,YAAY,IAAI,IAAI;AAAA,IAC7B,WAAW,MAAM,eAAe,QAAQ;AAAA,EACzC;AACD;AAEA,eAAe,YACd,UAGA,KACA,MACA,WACC;AACD,MAAI,KAAK,WAAW,EAAG;AACvB,QAAM,aAAa,KAAK,CAAC,GAAG,UAAU;AACtC,MAAI,eAAe,EAAG;AACtB,QAAM,cAAc,IAAI,KAAK,OAAO,UAAU,EAAE,MAAM,GAAG,EAAE,CAAC;AAC5D,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,WAAW;AAChD,UAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,SAAS;AACzC,UAAM,SAAS,IAAI,MAAM,MAAM,MAAM,EAAE,KAAK,WAAW,EAAE,KAAK,GAAG;AACjE,UAAM,OAAkB,CAAC;AACzB,eAAW,OAAO,MAAO,MAAK,KAAK,GAAG,GAAG;AACzC,UAAM,SAAS,QAAQ,GAAG,GAAG,WAAW,MAAM,IAAI,GAAG,IAAI;AAAA,EAC1D;AACD;AAEA,eAAe,YACd,UAGA,aACC;AACD,QAAM,aAAa,QAAQ;AAC3B,QAAM,MAAM;AACZ,QAAM,YAAY,YAAY,IAAI;AAElC,QAAM,gBAAgB,UAAU,YAAY;AAC3C,UAAM,WAAwB,CAAC;AAC/B,aAAS,IAAI,GAAG,IAAI,qBAAqB,KAAK,GAAG;AAChD,eAAS,KAAK;AAAA,QACb,MAAM,IAAI,OAAO;AAAA,QACjB,IAAI,MAAM,IAAI,SAAS;AAAA,QACvB,QAAQ,QAAQ,GAAG;AAAA,QACnB;AAAA,QACA,OAAO,sBAAsB,KAAK;AAAA,MACnC,CAAC;AAAA,IACF;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAEA,UAAM,eAA4B,CAAC;AACnC,aAAS,IAAI,GAAG,IAAI,0BAA0B,KAAK,GAAG;AACrD,mBAAa,KAAK;AAAA,QACjB,IAAI;AAAA,QACJ,QAAQ,IAAI,EAAE;AAAA,QACd,QAAQ,CAAC;AAAA,QACT,IAAI,MAAM,IAAI,YAAY;AAAA,MAC3B,CAAC;AAAA,IACF;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAEA,UAAM,aAA0B,CAAC;AACjC,aAAS,IAAI,GAAG,IAAI,uBAAuB,KAAK,GAAG;AAClD,iBAAW,KAAK;AAAA,QACf,IAAI;AAAA,QACJ,SAAS,IAAI,CAAC;AAAA,QACd,QAAQ,UAAU,GAAG;AAAA,QACrB,OAAO,wBAAwB,KAAK;AAAA,MACrC,CAAC;AAAA,IACF;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAEA,UAAM,SAAsB,CAAC;AAC7B,aAAS,IAAI,GAAG,IAAI,mBAAmB,KAAK,GAAG;AAC9C,aAAO,KAAK,CAAC,MAAM,CAAC,IAAI,QAAQ,OAAO,GAAG,GAAG,GAAG,CAAC;AAAA,IAClD;AACA,UAAM,YAAY,UAAU,2CAA2C,QAAQ,EAAE;AAEjF,UAAM,YAAyB,CAAC;AAChC,aAAS,IAAI,GAAG,IAAI,sBAAsB,KAAK,GAAG;AACjD,gBAAU,KAAK,CAAC,UAAU,QAAQ,CAAC,IAAI,QAAQ,SAAS,IAAI,GAAG,GAAG,CAAC;AAAA,IACpE;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAEA,UAAM,WAAwB,CAAC;AAC/B,aAAS,IAAI,GAAG,IAAI,qBAAqB,KAAK,GAAG;AAChD,eAAS,KAAK,CAAC,OAAO,CAAC,IAAI,QAAQ,SAAS,EAAE,CAAC,CAAC;AAAA,IACjD;AACA,UAAM,YAAY,UAAU,iCAAiC,UAAU,EAAE;AAAA,EAC1E,CAAC;AAED,SAAO;AAAA,IACN,MACC,sBACA,2BACA,wBACA,oBACA,uBACA;AAAA,IACD;AAAA,IACA,UAAU;AAAA,IACV,SAAS,YAAY,IAAI,IAAI;AAAA,IAC7B,WAAW,MAAM,eAAe,QAAQ;AAAA,EACzC;AACD;AAEA,eAAe,oBACd,UAGA,aACA,UACA,SAAS,OACR;AACD,QAAM,eAAe,QAAQ;AAC7B,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOrB;AAEF,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,KAAK,cAAc,QAAQ,CAAC;AAC1D,QAAM,YAAY,YAAY,IAAI;AAElC,WAAS,UAAU,GAAG,UAAU,MAAM,WAAW,wBAAwB;AACxE,UAAM,QAAQ,KAAK,IAAI,MAAM,UAAU,sBAAsB;AAC7D,UAAM,gBAAgB,UAAU,YAAY;AAC3C,eAAS,SAAS,SAAS,SAAS,OAAO,UAAU,kBAAkB;AACtE,cAAM,eAAyB,CAAC;AAChC,cAAM,OAAkB,CAAC;AACzB,cAAM,WAAW,KAAK,IAAI,OAAO,SAAS,gBAAgB;AAC1D,iBAAS,IAAI,QAAQ,IAAI,UAAU,KAAK,GAAG;AAC1C,gBAAM,YAAY,SACf,QAAQ,IAAI,OAAO,IAAI,IAAI,MAAM,IAAI,CAAC,KACtC,QAAQ,aAAa,CAAC,IAAI,GAAG;AAChC,gBAAM,SAAS,SACZ,IAAI,OAAO,IACV,WACA,SACD,CAAC,QAAQ,UAAU,UAAU,SAAS,EAAE,IAAI,CAAC;AAChD,uBAAa,KAAK,oBAAoB;AACtC,eAAK;AAAA,YACJ,IAAI;AAAA,YACJ;AAAA,YACA;AAAA,YACA,QAAoB,IAAI;AAAA,YACxB,MAAO,aAAa,IAAI,EAAE,IAAI;AAAA,YAC9B,QAAQ,aAAa,CAAC,KAAK,QAAQ;AAAA,UACpC;AAAA,QACD;AACA,cAAM,SAAS;AAAA,UACd,kGAAkG,aAAa,KAAK,IAAI,CAAC;AAAA,UACzH,GAAG;AAAA,QACJ;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AAEA,SAAO;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,YAAY,IAAI,IAAI;AAAA,IAC7B,WAAW,MAAM,eAAe,QAAQ;AAAA,EACzC;AACD;AAEA,eAAe,eACd,UAGA,WACC;AACD,QAAM,CAAC,KAAK,IAAI;AAAA,IACf,MAAM,SAAS,QAAQ,wCAAwC;AAAA,EAChE;AACA,QAAM,OAAO,OAAO,QAAQ;AAC5B,MAAI,QAAQ;AACZ,MAAI,cAAc;AAElB,MAAI,cAAc,YAAY;AAC7B,aAAS,QAAQ,MAAM,QAAQ,GAAG,SAAS,kBAAkB;AAC5D,YAAM,QAAQ,KAAK,IAAI,GAAG,QAAQ,mBAAmB,CAAC;AACtD,YAAM,QAAQ;AAAA,QACb,MAAM,SAAS;AAAA,UACd;AAAA,UACA;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,iBAAW,OAAO,OAAO;AACxB,iBAAS,IAAI;AACb,uBAAe;AAAA,MAChB;AAAA,IACD;AACA,WAAO,EAAE,MAAM,aAAa,MAAM;AAAA,EACnC;AAEA,WAAS,QAAQ,GAAG,SAAS,MAAM,SAAS,kBAAkB;AAC7D,UAAM,QAAQ,QAAQ,mBAAmB;AACzC,UAAM,CAAC,KAAK,IAAI;AAAA,MACf,MAAM,SAAS;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACD;AAAA,IACD;AACA,aAAS,OAAO,SAAS;AACzB,mBAAe,OAAO,QAAQ;AAAA,EAC/B;AAEA,SAAO,EAAE,MAAM,aAAa,MAAM;AACnC;AAEO,IAAM,uBAAuBH,QAAM;AAAA,EACzC,SAAS;AAAA,IACR,eAAe;AAAA,IACf,kBAAkB;AAAA,EACnB;AAAA,EACA,IAAIC,IAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAMrB;AACF,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAQrB;AACF,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAOrB;AACF,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAOrB;AACF,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAOrB;AACF,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBAOP;AAChB,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KASrB;AACF,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AAAA,EACD,SAAS;AAAA,IACR,oBAAoB,OAAO,MAAM;AAChC,YAAM,CAAC,SAAS,IAAI;AAAA,QACnB,MAAM,EAAE,GAAG,QAAQ,mBAAmB;AAAA,MACvC;AACA,YAAM,CAAC,QAAQ,IAAI;AAAA,QAClB,MAAM,EAAE,GAAG,QAAQ,kBAAkB;AAAA,MACtC;AACA,aAAO;AAAA,QACN,uBAAuB,WAAW,cAAc;AAAA,QAChD,gBAAgB,UAAU,aAAa;AAAA,QACvC,WAAW,MAAM,eAAe,EAAE,EAAE;AAAA,MACrC;AAAA,IACD;AAAA,IAEA,eAAe,OAAO,GAAG,UAAsB;AAC9C,qBAAe,MAAM,QAAQ;AAC7B,YAAM,WAAWI,iBAAgB,MAAM,UAAUH,oBAAmB,UAAU;AAC9E,UAAI,MAAM,aAAa,uBAAuB;AAC7C,cAAM,eAAe,EAAE,EAAE;AACzB,eAAO;AAAA,UACN,MAAM;AAAA,UACN,aAAa;AAAA,UACb;AAAA,UACA,SAAS;AAAA,UACT,WAAW,MAAM,eAAe,EAAE,EAAE;AAAA,QACrC;AAAA,MACD;AACA,YAAM,cAAcG;AAAA,QACnB,MAAM;AAAA,QACN,IAAI,OAAO;AAAA,QACX;AAAA,MACD;AAEA,cAAQ,MAAM,UAAU;AAAA,QACvB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACJ,iBAAO,aAAa,EAAE,IAAI,aAAa,QAAQ;AAAA,QAChD,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACJ,iBAAO,SAAS,EAAE,IAAI,aAAa,QAAQ;AAAA,QAC5C,KAAK;AACJ,iBAAO,WAAW,EAAE,IAAI,aAAa,QAAQ;AAAA,QAC9C,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACJ,iBAAOE,aAAY,EAAE,IAAI,WAAW;AAAA,QACrC,KAAK;AACJ,iBAAO,YAAY,EAAE,IAAI,WAAW;AAAA,QACrC,KAAK;AACJ,iBAAO,oBAAoB,EAAE,IAAI,aAAa,QAAQ;AAAA,QACvD,KAAK;AACJ,iBAAO,oBAAoB,EAAE,IAAI,aAAa,UAAU,IAAI;AAAA,QAC7D,KAAK;AAAA,QACL,KAAK;AACJ,iBAAO,oBAAoB,EAAE,IAAI,aAAa,QAAQ;AAAA,MACxD;AAAA,IACD;AAAA,IAEA,aAAa,OAAO,GAAG,UAAoB;AAC1C,qBAAe,MAAM,QAAQ;AAC7B,YAAM,KAAK,YAAY,IAAI;AAC3B,UAAI;AAEJ,cAAQ,MAAM,UAAU;AAAA,QACvB,KAAK,qBAAqB;AACzB,cAAI,QAAQ;AACZ,mBAAS,IAAI,GAAG,IAAI,IAAI,KAAK,GAAG;AAC/B,kBAAM,KAAM,IAAI,KAAM;AACtB,kBAAM,CAAC,GAAG,IAAI;AAAA,cACb,MAAM,EAAE,GAAG;AAAA,gBACV;AAAA,gBACA;AAAA,cACD;AAAA,YACD;AACA,qBAAS,KAAK,SAAS;AAAA,UACxB;AACA,oBAAU,EAAE,KAAK,IAAI,MAAM;AAC3B;AAAA,QACD;AAAA,QACA,KAAK,qBAAqB;AACzB,gBAAM,SAAS,MAAM,EAAE,GAAG;AAAA,YACzB;AAAA,UACD;AACA,gBAAM,UAAU,MAAM,EAAE,GAAG,QAAQ,8BAA8B;AACjE,gBAAM,CAAC,KAAK,IAAI;AAAA,YACf,MAAM,EAAE,GAAG,QAAQ,wCAAwC;AAAA,UAC5D;AACA,oBAAU;AAAA,YACT,SAAS,OAAO;AAAA,YAChB,SAAS,QAAQ;AAAA,YACjB,MAAM,OAAO,QAAQ;AAAA,UACtB;AACA;AAAA,QACD;AAAA,QACA,KAAK;AAAA,QACL,KAAK,uBAAuB;AAC3B,oBAAU,MAAM,eAAe,EAAE,IAAI,SAAS;AAC9C;AAAA,QACD;AAAA,QACA,KAAK,wBAAwB;AAC5B,oBAAU,MAAM,eAAe,EAAE,IAAI,UAAU;AAC/C;AAAA,QACD;AAAA,QACA,KAAK,kCAAkC;AACtC,gBAAM,OAAO;AAAA,YACZ,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA,YAGD;AAAA,UACD;AACA,cAAIC,YAAW;AACf,qBAAW,OAAO,KAAM,CAAAA,YAAYA,YAAW,IAAI,aAAc;AACjE,oBAAU,EAAE,MAAM,KAAK,QAAQ,UAAAA,UAAS;AACxC;AAAA,QACD;AAAA,QACA,KAAK,mCAAmC;AACvC,gBAAM,OAAO;AAAA,YACZ,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA,YAGD;AAAA,UACD;AACA,cAAI,QAAQ;AACZ,qBAAW,OAAO,KAAM,UAAS,IAAI;AACrC,oBAAU,EAAE,MAAM,KAAK,QAAQ,MAAM;AACrC;AAAA,QACD;AAAA,QACA,KAAK,oBAAoB;AACxB,gBAAM,OAAO;AAAA,YACZ,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA;AAAA,YAID;AAAA,UACD;AACA,oBAAU;AAAA,YACT,QAAQ,KAAK;AAAA,YACb,MAAM,KAAK,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,MAAM,CAAC;AAAA,YACjD,OAAO,KAAK,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,OAAO,CAAC;AAAA,UACpD;AACA;AAAA,QACD;AAAA,QACA,KAAK,yBAAyB;AAC7B,gBAAM,OAAO;AAAA,YACZ,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA;AAAA,YAID;AAAA,UACD;AACA,oBAAU;AAAA,YACT,SAAS,KAAK;AAAA,YACd,MAAM,KAAK,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,MAAM,CAAC;AAAA,YACjD,OAAO,KAAK,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,OAAO,CAAC;AAAA,UACpD;AACA;AAAA,QACD;AAAA,QACA,KAAK,+BAA+B;AACnC,gBAAM,OAAO;AAAA,YACZ,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,cAMA;AAAA,cACA;AAAA,cACA,QAAoB;AAAA,YACrB;AAAA,UACD;AACA,oBAAU;AAAA,YACT,QAAQ,KAAK;AAAA,YACb,MAAM,KAAK,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,MAAM,CAAC;AAAA,YACjD,OAAO,KAAK,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,OAAO,CAAC;AAAA,UACpD;AACA;AAAA,QACD;AAAA,QACA,KAAK,4BAA4B;AAChC,gBAAM;AAAA,YACL;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACD,IAAI,MAAM,QAAQ,IAAI;AAAA,YACrB,EAAE,GAAG;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,YAID;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,YAID;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,cAMA;AAAA,cACA;AAAA,cACA,QAAoB;AAAA,YACrB;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA;AAAA,YAKD;AAAA,UACD,CAAC;AACD,gBAAM,aAAa;AAAA,YAClB,GAAG,UAAwB,UAAU;AAAA,YACrC,GAAG,UAAwB,UAAU;AAAA,YACrC,GAAG,UAAwB,UAAU;AAAA,YACrC,GAAG,UAAwB,QAAQ;AAAA,UACpC;AACA,oBAAU;AAAA,YACT,KAAK;AAAA,YACL,QAAQ,WAAW;AAAA,YACnB,MAAM,WAAW,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,MAAM,CAAC;AAAA,YACvD,OAAO,WAAW,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,OAAO,CAAC;AAAA,UAC1D;AACA;AAAA,QACD;AAAA,QACA,KAAK,kCAAkC;AACtC,gBAAM,aAAa,EAAE,GAAG;AAAA,YACvB;AAAA;AAAA;AAAA;AAAA,UAID;AACA,gBAAM,WAAW,EAAE,GAAG;AAAA,YACrB;AAAA;AAAA;AAAA;AAAA;AAAA,UAKD;AACA,gBAAM,gBAAgB,EAAE,GAAG;AAAA,YAC1B;AAAA,UACD;AACA,gBAAM,iBAAiB,EAAE,GAAG;AAAA,YAC3B;AAAA,UACD;AACA,gBAAM,CAAC,YAAY,UAAU,EAAE,SAAS,IAAI,MAAM,QAAQ,IAAI;AAAA,YAC7D;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACD,CAAC;AACD,gBAAM,aAAa;AAAA,YAClB,GAAG,UAAwB,UAAU;AAAA,YACrC,GAAG,UAAwB,QAAQ;AAAA,UACpC;AACA,gBAAM,CAAC,UAAU,IAAI,UAAoB,SAAS;AAClD,oBAAU;AAAA,YACT,KAAK;AAAA,YACL,SAAS;AAAA,YACT,UAAU;AAAA,YACV,QAAQ,WAAW;AAAA,YACnB,MACC,WAAW,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,MAAM,CAAC,KAChD,YAAY,QAAQ;AAAA,YACtB,OAAO,WAAW,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,OAAO,CAAC;AAAA,UAC1D;AACA;AAAA,QACD;AAAA,QACA,KAAK,uBAAuB;AAC3B,gBAAM,OAAO,MAAM,EAAE,GAAG;AAAA,YACvB;AAAA;AAAA;AAAA;AAAA;AAAA,YAKA;AAAA,UACD;AACA,oBAAU,EAAE,MAAM,KAAK,OAAO;AAC9B;AAAA,QACD;AAAA,QACA,KAAK,4BAA4B;AAChC,gBAAM,YAAY;AAAA,YACjB,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA;AAAA;AAAA,cAKA;AAAA,cACA;AAAA,YACD;AAAA,UACD;AACA,gBAAM,SAAS,UAAU,GAAG,EAAE,GAAG,cAAc;AAC/C,gBAAM,aAAa,MAAM,EAAE,GAAG;AAAA,YAC7B;AAAA;AAAA;AAAA;AAAA;AAAA,YAKA;AAAA,YACA;AAAA,UACD;AACA,oBAAU,EAAE,eAAe,UAAU,QAAQ,MAAM,WAAW,OAAO;AACrE;AAAA,QACD;AAAA,QACA,KAAK,oBAAoB;AACxB,gBAAM,OAAO;AAAA,YACZ,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA;AAAA;AAAA,YAKD;AAAA,UACD;AACA,oBAAU;AAAA,YACT,QAAQ,KAAK;AAAA,YACb,MAAM,KAAK,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,MAAM,CAAC;AAAA,YACjD,OAAO,KAAK,OAAO,CAAC,KAAK,QAAQ,MAAM,IAAI,OAAO,CAAC;AAAA,UACpD;AACA;AAAA,QACD;AAAA,QACA,KAAK,wBAAwB;AAC5B,gBAAM,CAAC,KAAK,IAAI;AAAA,YACf,MAAM,EAAE,GAAG,QAAQ,wCAAwC;AAAA,UAC5D;AACA,gBAAM,OAAO,KAAK,IAAI,GAAG,OAAO,QAAQ,CAAC;AACzC,cAAI,QAAQ;AACZ,mBAAS,IAAI,GAAG,IAAI,kBAAkB,KAAK,GAAG;AAC7C,kBAAM,KAAM,aAAa,CAAC,IAAI,OAAQ;AACtC,kBAAM,CAAC,GAAG,IAAI;AAAA,cACb,MAAM,EAAE,GAAG;AAAA,gBACV;AAAA,gBACA;AAAA,cACD;AAAA,YACD;AACA,qBAAS,KAAK,SAAS;AAAA,UACxB;AACA,oBAAU,EAAE,KAAK,kBAAkB,MAAM;AACzC;AAAA,QACD;AAAA,QACA,KAAK,wBAAwB;AAC5B,gBAAM,YAAY;AAAA,YACjB,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA;AAAA;AAAA,cAKA;AAAA,YACD;AAAA,UACD;AACA,cAAI,QAAQ;AACZ,qBAAW,OAAO,WAAW;AAC5B,kBAAM,CAAC,GAAG,IAAI;AAAA,cACb,MAAM,EAAE,GAAG;AAAA,gBACV;AAAA,gBACA,IAAI;AAAA,cACL;AAAA,YACD;AACA,qBAAS,KAAK,SAAS;AAAA,UACxB;AACA,oBAAU,EAAE,MAAM,UAAU,QAAQ,MAAM;AAC1C;AAAA,QACD;AAAA,QACA,KAAK,8BAA8B;AAClC,gBAAM,OAAO;AAAA,YACZ,MAAM,EAAE,GAAG;AAAA,cACV;AAAA;AAAA;AAAA;AAAA,YAID;AAAA,UACD;AACA,cAAI,QAAQ;AACZ,qBAAW,OAAO,KAAM,UAAS,IAAI;AACrC,oBAAU,EAAE,MAAM,KAAK,QAAQ,MAAM;AACrC;AAAA,QACD;AAAA,QACA,KAAK,yBAAyB;AAC7B,gBAAM,OAAO,MAAM,EAAE,GAAG;AAAA,YACvB;AAAA,UACD;AACA,oBAAU,EAAE,MAAM,KAAK,OAAO;AAC9B;AAAA,QACD;AAAA,QACA,KAAK,2BAA2B;AAC/B,gBAAM,eAAe,KAAK;AAAA,YACzB;AAAA,YACA,KAAK;AAAA,cACJH,iBAAgB,MAAM,aAAaF,uBAAsB,aAAa,IACrEA;AAAA,YACF;AAAA,UACD;AACA,gBAAM,aAAa,KAAK,IAAI,GAAG,eAAe,GAAG;AACjD,gBAAM,OAAO,MAAM,EAAE,GAAG;AAAA,YACvB;AAAA,YACA;AAAA,YACA;AAAA,UACD;AACA,oBAAU,EAAE,MAAM,KAAK,OAAO;AAC9B;AAAA,QACD;AAAA,QACA,KAAK,kBAAkB;AACtB,gBAAM,CAAC,GAAG,IAAI;AAAA,YACb,MAAM,EAAE,GAAG;AAAA,cACV;AAAA,cACA;AAAA,YACD;AAAA,UACD;AACA,oBAAU,EAAE,KAAK,GAAG,MAAM,KAAK,SAAS,EAAE;AAC1C;AAAA,QACD;AAAA,QACA,KAAK,gBAAgB;AACpB,gBAAM,CAAC,GAAG,IAAI;AAAA,YACb,MAAM,EAAE,GAAG;AAAA,cACV;AAAA,cACA;AAAA,YACD;AAAA,UACD;AACA,oBAAU,EAAE,KAAK,GAAG,OAAO,KAAK,eAAe,EAAE;AACjD;AAAA,QACD;AAAA,QACA,KAAK,yBAAyB;AAC7B,gBAAM,eAAe,KAAK;AAAA,YACzB;AAAA,YACA,KAAK;AAAA,cACJE,iBAAgB,MAAM,aAAaF,uBAAsB,aAAa,IACrEA;AAAA,YACF;AAAA,UACD;AACA,gBAAM,aAAa,KAAK,IAAI,GAAG,eAAe,GAAG;AACjD,gBAAM,CAAC,WAAW,aAAa,WAAW,OAAO,IAAI,MAAM,QAAQ,IAAI;AAAA,YACtE,EAAE,GAAG;AAAA,cACJ;AAAA,YACD;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA,cACA;AAAA,cACA;AAAA,YACD;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA,cACA;AAAA,YACD;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA,cACA;AAAA,YACD;AAAA,UACD,CAAC;AACD,gBAAM,CAAC,QAAQ,IAAI,UAA6B,SAAS;AACzD,gBAAM,CAAC,MAAM,IAAI,UAA0C,OAAO;AAClE,oBAAU;AAAA,YACT,KAAK;AAAA,YACL,WAAW,UAAU;AAAA,YACrB,aAAa,YAAY;AAAA,YACzB,MAAM,UAAU,SAAS;AAAA,YACzB,OAAO,QAAQ,eAAe;AAAA,UAC/B;AACA;AAAA,QACD;AAAA,QACA,KAAK,oBAAoB;AACxB,gBAAM;AAAA,YACL;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACD,IAAI,MAAM,QAAQ,IAAI;AAAA,YACrB,EAAE,GAAG;AAAA,cACJ;AAAA,cACA;AAAA,YACD;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA,cACA;AAAA,YACD;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA,cACA;AAAA,YACD;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA,YACD;AAAA,YACA,EAAE,GAAG;AAAA,cACJ;AAAA,cACA;AAAA,YACD;AAAA,YACA,EAAE,GAAG,QAAQ,8CAA8C;AAAA,YAC3D,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAMF;AAAA,UACZ,CAAC;AACD,oBAAU;AAAA,YACT,KAAK;AAAA,YACL,UAAU,SAAS;AAAA,YACnB,cAAc,aAAa;AAAA,YAC3B,YAAY,WAAW;AAAA,YACvB,QAAQ,OAAO;AAAA,YACf,WAAW,UAAU;AAAA,YACrB,UAAU,SAAS;AAAA,YACnB,gBAAgB,eAAe;AAAA,UAChC;AACA;AAAA,QACD;AAAA,QACA,KAAK,0BAA0B;AAC9B,gBAAM,CAAC,KAAK,IAAI;AAAA,YACf,MAAM,EAAE,GAAG,QAAQ,wCAAwC;AAAA,UAC5D;AACA,gBAAM,WAAW,OAAO,QAAQ,KAAK;AACrC,gBAAM,EAAE,GAAG,QAAQ,OAAO;AAC1B,mBAAS,SAAS,GAAG,SAAS,KAAM,UAAU,kBAAkB;AAC/D,kBAAM,eAAyB,CAAC;AAChC,kBAAM,OAAkB,CAAC;AACzB,qBAAS,IAAI,QAAQ,IAAI,SAAS,kBAAkB,KAAK,GAAG;AAC3D,oBAAM,KAAK,UAAU;AACrB,2BAAa,KAAK,uBAAuB;AACzC,mBAAK;AAAA,gBACJ;AAAA,gBACC,IAAI,MAAO;AAAA,gBACZ,QAAoB;AAAA,gBACpB;AAAA,gBACA,MAAO;AAAA,gBACP,IAAI;AAAA,gBACJ,QAAQ,eAAe,EAAE,KAAKD,kBAAiB;AAAA,cAChD;AAAA,YACD;AACA,kBAAM,EAAE,GAAG;AAAA,cACV,gGAAgG,aAAa,KAAK,IAAI,CAAC;AAAA,cACvH,GAAG;AAAA,YACJ;AAAA,UACD;AACA,gBAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,oBAAU,EAAE,MAAM,IAAK;AACvB;AAAA,QACD;AAAA,QACA,KAAK,wBAAwB;AAC5B,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,gBAAM,CAAC,KAAK,IAAI;AAAA,YACf,MAAM,EAAE,GAAG;AAAA,cACV;AAAA,YACD;AAAA,UACD;AACA,oBAAU,EAAE,MAAM,OAAO,QAAQ,EAAE;AACnC;AAAA,QACD;AAAA,QACA,KAAK,2BAA2B;AAC/B,gBAAM,EAAE,GAAG,QAAQ,oDAAoD;AACvE,gBAAM,SAAS,MAAM,eAAe,EAAE,IAAI,SAAS;AACnD,oBAAU;AAAA,YACT,GAAG;AAAA,YACH,mBAAmB;AAAA,UACpB;AACA;AAAA,QACD;AAAA,QACA,KAAK,kCAAkC;AACtC,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,oBAAU,EAAE,SAAS,EAAE;AACvB;AAAA,QACD;AAAA,QACA,KAAK,yCAAyC;AAC7C,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,oBAAU,EAAE,SAAS,GAAG,QAAQ,KAAK;AACrC;AAAA,QACD;AAAA,QACA,KAAK,iCAAiC;AACrC,gBAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQjB;AACF,gBAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,8BAIM;AACzB,gBAAM,EAAE,GAAG,QAAQ,gCAAgC;AACnD,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,oBAAU,EAAE,SAAS,KAAK;AAC1B;AAAA,QACD;AAAA,QACA,KAAK,8BAA8B;AAClC,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,oBAAU,EAAE,QAAQ,GAAG,cAAc,MAAM;AAC3C;AAAA,QACD;AAAA,QACA,KAAK,uBAAuB;AAC3B,gBAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,OAIjB;AACF,gBAAM,EAAE,GAAG,QAAQ,uDAAuD;AAC1E,gBAAM,EAAE,GAAG;AAAA,YACV;AAAA,UACD;AACA,gBAAM,EAAE,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,OAIjB;AACF,oBAAU,EAAE,QAAQ,GAAG,SAAS,GAAG,QAAQ,EAAE;AAC7C;AAAA,QACD;AAAA,MACD;AAEA,YAAM,KAAK,YAAY,IAAI,IAAI;AAC/B,aAAO;AAAA,QACN;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,GAAG;AAAA,QACH,WAAW,MAAM,eAAe,EAAE,EAAE;AAAA,MACrC;AAAA,IACD;AAAA,IAEA,WAAW,CAAC,MAAM;AACjB,QAAE,MAAM;AACR,aAAO,EAAE,IAAI,KAAK;AAAA,IACnB;AAAA,EACD;AACD,CAAC;;;AC19CD,SAAS,SAAAO,eAAa;AACtB,SAAS,MAAAC,WAAU;AA4HnB,IAAM,gBAAgB;AACtB,IAAM,0BAA0B;AAChC,IAAM,oBAAoB;AAC1B,IAAM,4BAA4B,IAAI;AACtC,IAAM,8BAA8B,OAAO;AAC3C,IAAM,0BAA0B,KAAK;AACrC,IAAM,sBAAsB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAEA,SAAS,SAAS,OAAuB;AACxC,MAAI,OAAO;AACX,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;AACzC,YAAQ,MAAM,WAAW,CAAC;AAC1B,WAAO,KAAK,KAAK,MAAM,QAAQ;AAAA,EAChC;AACA,SAAO,SAAS;AACjB;AAEA,SAAS,QAAQ,MAA4B;AAC5C,MAAI,QAAQ,SAAS,IAAI,KAAK;AAC9B,SAAO,MAAM;AACZ,YAAS,QAAQ,eAAgB;AACjC,QAAI,IAAI;AACR,QAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;AACnC,SAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;AACxC,aAAS,IAAK,MAAM,QAAS,KAAK;AAAA,EACnC;AACD;AAEA,SAAS,WAAW,KAAmB,KAAa,KAAqB;AACxE,SAAO,MAAM,KAAK,MAAM,IAAI,KAAK,MAAM,MAAM,EAAE;AAChD;AAEA,SAAS,SAAS,OAAuB;AACxC,MAAI,OAAO;AACX,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;AACzC,YAAQ,MAAM,WAAW,CAAC;AAC1B,WAAO,KAAK,KAAK,MAAM,QAAQ;AAAA,EAChC;AACA,SAAO,SAAS;AACjB;AAEA,SAAS,WACR,MACA,OACA,OACA,OACS;AACT,QAAM,SAAS,GAAG,IAAI,IAAI,KAAK,IAAI,KAAK;AACxC,MAAI,SAAS,OAAO,OAAQ,QAAO,OAAO,MAAM,GAAG,KAAK;AACxD,SAAO,SAAS,IAAI,OAAO,QAAQ,OAAO,MAAM;AACjD;AAEA,eAAe,SACd,UACA,QACG,MACsB;AACzB,QAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,GAAG,IAAI;AAChD,SAAO,KAAK,CAAC;AACd;AAEA,eAAe,YACd,UACA,IACa;AACb,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI;AACH,UAAM,SAAS,MAAM,GAAG;AACxB,UAAM,SAAS,QAAQ,QAAQ;AAC/B,WAAO;AAAA,EACR,SAAS,KAAK;AACb,UAAM,SAAS,QAAQ,UAAU,EAAE,MAAM,MAAM,MAAS;AACxD,UAAM;AAAA,EACP;AACD;AAEA,eAAe,YACd,UACA,OACA,UACA,MACA,UACA,QACA,UACgB;AAChB,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO,QAAQ;AAAA,IACf,OAAO,MAAM;AAAA,IACb,WAAW,IAAI;AAAA,IACf,KAAK,IAAI;AAAA,EACV;AACD;AAEA,SAAS,YAAY,KAAuB;AAC3C,MAAI,CAAC,OAAO,OAAO,QAAQ,SAAU,QAAO;AAC5C,QAAM,SAAS,OAAO,OAAO,GAA8B;AAC3D,SAAO,OAAO,CAAC;AAChB;AAEA,eAAe,eACd,UACgB;AAChB,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI;AACH,aAAS,IAAI,GAAG,IAAI,eAAe,KAAK,GAAG;AAC1C,YAAM,SAAS;AAAA,QACd;AAAA,QACA,QAAQ,CAAC;AAAA,QACT;AAAA,MACD;AAAA,IACD;AACA,UAAM,SAAS,QAAQ,QAAQ;AAAA,EAChC,SAAS,KAAK;AACb,UAAM,SAAS,QAAQ,UAAU,EAAE,MAAM,MAAM,MAAS;AACxD,UAAM;AAAA,EACP;AACD;AAEA,eAAe,gBACd,UACA,OACA,YACA,MACA,SACA,SACA,OACA,SACA,aACAC,UACA,SACgB;AAChB,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA;AAAA,IAIA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,IAAI;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAASA,QAAO;AAAA,IAChBA,SAAQ;AAAA,IACR,UAAU,IAAI;AAAA,IACd,KAAK,IAAI;AAAA,EACV;AACD;AAEA,eAAe,eACd,UACA,KACAA,UACgB;AAChB,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAYA,IAAI;AAAA,IACJ,IAAI;AAAA,IACJ,IAAI;AAAA,IACJ,IAAI;AAAA,IACJA;AAAA,IACA,IAAI;AAAA,IACJ,IAAI;AAAA,IACJ,KAAK,IAAI;AAAA,EACV;AACD;AAEA,eAAe,mBACd,UACA,MAQgB;AAChB,MAAI;AACJ,MAAI;AACH,cAAU,MAAM;AAAA,MACf;AAAA,MACA;AAAA,MACA,KAAK;AAAA,IACN;AAAA,EACD,SAAS,OAAO;AACf,UAAM,IAAI;AAAA,MACT,yCAAyC,KAAK,IAAI,QAAQ,KAAK,UAAU,KAAK,OAAO,CAAC,iBAAiB,KAAK,YAAY;AAAA,MACxH,EAAE,OAAO,MAAM;AAAA,IAChB;AAAA,EACD;AACA,QAAMA,WAAU;AAAA,IACf,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,EACN;AACA,QAAM,eAAe,SAAS,WAAW,KAAK;AAC9C,QAAM,mBAAmB,SAAS,gBAAgB,KAAK;AACvD,QAAM,YAAY,GAAG,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,UAAU,IAAI,WAAW;AAE9E,MAAI,KAAK,SAAS,UAAU;AAC3B,QAAI;AACH,YAAM;AAAA,QACL;AAAA,QACA,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,YAAY;AAAA,MACb;AAAA,IACD,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,qDAAqD,KAAK,UAAU,KAAK,OAAO,CAAC,IAAI;AAAA,QACpG,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,QAAI;AACH,YAAM,SAAS,QAAQ,6CAA6C,KAAK,OAAO;AAAA,IACjF,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,wCAAwC,KAAK,UAAU,KAAK,OAAO,CAAC,IAAI;AAAA,QACvF,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA;AAAA,EACD;AAEA,MAAI,KAAK,SAAS,YAAY,SAAS;AACtC,QAAI;AACH,YAAM;AAAA,QACL;AAAA,QACA,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL;AAAA,QACA,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,QAAQ,WAAW;AAAA,QACnB;AAAA,MACD;AAAA,IACD,SAAS,OAAO;AACf,YAAM,IAAI;AAAA,QACT,0DAA0D,KAAK,UAAU,KAAK,OAAO,CAAC;AAAA,QACtF,EAAE,OAAO,MAAM;AAAA,MAChB;AAAA,IACD;AACA;AAAA,EACD;AAEA,QAAM,MAAe;AAAA,IACpB,UAAU,KAAK;AAAA,IACf,OAAO;AAAA,IACP,SAAS;AAAA,IACT,cAAc;AAAA,IACd,kBAAkB,SAASA,QAAO;AAAA,IAClC,eAAeA,SAAQ;AAAA,EACxB;AAEA,MAAI;AACH,UAAM;AAAA,MACL;AAAA,MACA,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA,IAAI;AAAA,MACJ,IAAI;AAAA,MACJ,IAAI;AAAA,MACJA;AAAA,MACA;AAAA,IACD;AAAA,EACD,SAAS,OAAO;AACf,UAAM,IAAI;AAAA,MACT,+CAA+C,KAAK,IAAI,QAAQ,KAAK,UAAU,KAAK,OAAO,CAAC,iBAAiB,KAAK,YAAY;AAAA,MAC9H,EAAE,OAAO,MAAM;AAAA,IAChB;AAAA,EACD;AACA,MAAI;AACH,UAAM,eAAe,UAAU,KAAKA,QAAO;AAAA,EAC5C,SAAS,OAAO;AACf,UAAM,IAAI;AAAA,MACT,kDAAkD,KAAK,IAAI,QAAQ,KAAK,UAAU,KAAK,OAAO,CAAC,iBAAiB,KAAK,YAAY;AAAA,MACjI,EAAE,OAAO,MAAM;AAAA,IAChB;AAAA,EACD;AACD;AAEA,eAAe,gBACd,UACA,MAQgB;AAChB,WAAS,IAAI,GAAG,IAAI,KAAK,SAAS,KAAK,GAAG;AACzC,QAAI;AACH,YAAM,mBAAmB,UAAU;AAAA,QAClC,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,QACZ,YAAY,KAAK,aAAa,MAAO;AAAA,QACrC,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,cAAc,KAAK;AAAA,MACpB,CAAC;AAAA,IACF,SAAS,OAAO;AACf,YAAM,IAAI;AAAA,QACT,yBAAyB,KAAK,OAAO,kBAAkB,IAAI,CAAC,IAAI,KAAK,OAAO,sBAAsB,KAAK,YAAY;AAAA,QACnH,EAAE,OAAO,MAAM;AAAA,MAChB;AAAA,IACD;AAAA,EACD;AACD;AAEA,eAAe,cACd,UACA,MAOgB;AAChB,QAAM,YAAY,UAAU,YAAY;AACvC,UAAM,SAAS,MAAM;AAAA,MACpB;AAAA,MACA;AAAA,IACD;AACA,UAAM,SAAS;AAAA,MACd;AAAA,MACA,KAAK;AAAA,MACL,KAAK;AAAA,IACN;AACA,UAAM,SAAS;AAAA,MACd;AAAA,MACA,KAAK;AAAA,MACL,KAAK;AAAA,IACN;AACA,UAAM,QAAQ,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,IACD;AACA,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA;AAAA,MAIA,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,QAAQ,SAAS;AAAA,MACjB,OAAO,SAAS;AAAA,MAChB,KAAK,IAAI;AAAA,IACV;AAAA,EACD,CAAC;AACF;AAEA,eAAe,kBACd,UACA,MAKkB;AAClB,QAAM,mBAAmB,OACxB,IACA,MACAA,UACA,cACmB;AACnB,UAAM,kBAAkB,SAASA,QAAO;AACxC,UAAM,eAAeA,SAAQ;AAC7B,QAAI;AACH,YAAM,SAAS,QAAQ,OAAO;AAAA,IAC/B,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,iCAAiC,SAAS,IAAI;AAAA,QAC7D,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,QAAI;AACH,UAAI;AACH,cAAM,SAAS;AAAA,UACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,UASA;AAAA,UACA;AAAA,UACAA;AAAA,UACA;AAAA,UACA;AAAA,UACA,KAAK,IAAI;AAAA,QACV;AAAA,MACD,SAAS,OAAO;AACf,cAAM,IAAI,MAAM,sCAAsC,SAAS,IAAI;AAAA,UAClE,OAAO;AAAA,QACR,CAAC;AAAA,MACF;AACA,UAAI;AACH,cAAM,SAAS;AAAA,UACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,UAOA;AAAA,UACA;AAAA,UACA;AAAA,QACD;AAAA,MACD,SAAS,OAAO;AACf,cAAM,IAAI,MAAM,8CAA8C,SAAS,IAAI;AAAA,UAC1E,OAAO;AAAA,QACR,CAAC;AAAA,MACF;AACA,UAAI;AACH,cAAM,SAAS,QAAQ,QAAQ;AAAA,MAChC,SAAS,OAAO;AACf,cAAM,IAAI,MAAM,kCAAkC,SAAS,IAAI;AAAA,UAC9D,OAAO;AAAA,QACR,CAAC;AAAA,MACF;AAAA,IACD,SAAS,OAAO;AACf,YAAM,SAAS,QAAQ,UAAU,EAAE,MAAM,MAAM,MAAS;AACxD,YAAM;AAAA,IACP;AAAA,EACD;AAEA,QAAM,QAAQ,oBAAoB,OAAO,CAAC,SAAS,QAAQ,KAAK,eAAe;AAC/E,MAAI,CAAC,MAAM,SAAS,KAAK,eAAe,EAAG,OAAM,KAAK,KAAK,eAAe;AAE1E,MAAI,MAAM;AACV,aAAW,QAAQ,OAAO;AACzB,UAAM,KAAK,QAAQ,KAAK,KAAK,IAAI,IAAI;AACrC,UAAMA,WAAU,WAAW,KAAK,MAAM,KAAK,OAAO,MAAM,IAAI;AAC5D,QAAI;AACH,YAAM,iBAAiB,IAAI,YAAYA,UAAS,QAAQ,IAAI,EAAE;AAAA,IAC/D,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,sCAAsC,IAAI,IAAI;AAAA,QAC7D,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,WAAO;AAAA,EACR;AAEA,QAAM,iBAAiB,8CAAoC,KAAK,KAAK,SAAS,KAAK,IAAI;AACvF,QAAM,YAAY,QAAQ,KAAK,KAAK;AACpC,MAAI;AACH,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,EACD,SAAS,OAAO;AACf,UAAM,IAAI,MAAM,6DAA6D;AAAA,MAC5E,OAAO;AAAA,IACR,CAAC;AAAA,EACF;AAEA,SAAO,MAAM;AACd;AAEA,eAAe,sBACd,UACA,MAIkB;AAClB,QAAMA,WAAU,uBAAuB,KAAK,KAAK,SAAS,KAAK,IAAI;AACnE,QAAM,KAAK,cAAc,KAAK,KAAK;AACnC,QAAM,YAAY,UAAU,YAAY;AACvC,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAQA;AAAA,MACAA;AAAA,MACA,SAASA,QAAO;AAAA,MAChBA,SAAQ;AAAA,MACR,KAAK,IAAI;AAAA,IACV;AACA,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOA;AAAA,MACA,SAASA,QAAO;AAAA,MAChBA,SAAQ;AAAA,IACT;AAAA,EACD,CAAC;AACD,SAAO;AACR;AAEA,eAAe,wBACd,UACA,MAOkB;AAClB,QAAM,OAAO,KAAK,IAAI,IAAI,KAAK,MAAM,KAAK,aAAa,CAAC,CAAC;AACzD,MAAI,MAAM;AAEV,WAAS,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;AACjC,UAAM,OAAO,WAAW,KAAK,KAAK,IAAI,KAAK,IAAI,IAAI,KAAK,eAAe,CAAC;AACxE,UAAM,KAAK,QAAQ,KAAK,KAAK,IAAI,CAAC;AAClC,UAAMA,WAAU,WAAW,KAAK,MAAM,KAAK,OAAO,MAAS,GAAG,IAAI;AAClE,QAAI;AACH,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAQA;AAAA,QACAA;AAAA,QACA,SAASA,QAAO;AAAA,QAChBA,SAAQ;AAAA,QACR,KAAK,IAAI;AAAA,MACV;AAAA,IACD,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,2CAA2C,EAAE,YAAY,IAAI,IAAI;AAAA,QAChF,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,QAAI;AACH,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAOA;AAAA,QACA,SAASA,QAAO;AAAA,QAChBA,SAAQ;AAAA,MACT;AAAA,IACD,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,+CAA+C,EAAE,YAAY,IAAI,IAAI;AAAA,QACpF,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,WAAO;AAAA,EACR;AAEA,WAAS,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;AACjC,UAAM,KAAK,QAAQ,KAAK,KAAK,IAAI,CAAC;AAClC,QAAI;AACH,YAAM,SAAS,QAAQ,+CAA+C,EAAE;AAAA,IACzE,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,mCAAmC,EAAE,IAAI;AAAA,QACxD,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,QAAI;AACH,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAOA;AAAA,MACD;AAAA,IACD,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,kDAAkD,EAAE,IAAI;AAAA,QACvE,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,WAAO;AAAA,EACR;AAEA,WAAS,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;AACjC,UAAM,OAAO,WAAW,KAAK,KAAK,GAAG,KAAK,IAAI,GAAG,KAAK,eAAe,CAAC;AACtE,UAAM,KAAK,QAAQ,KAAK,KAAK,IAAI,CAAC;AAClC,UAAMA,WAAU,WAAW,KAAK,MAAM,KAAK,OAAO,MAAS,GAAG,IAAI;AAClE,QAAI;AACH,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA,QAGAA;AAAA,QACA,SAASA,QAAO;AAAA,QAChBA,SAAQ;AAAA,QACR,KAAK,IAAI;AAAA,QACT;AAAA,MACD;AAAA,IACD,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,4CAA4C,EAAE,YAAY,IAAI,IAAI;AAAA,QACjF,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,QAAI;AACH,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA,QAGA,SAASA,QAAO;AAAA,QAChBA,SAAQ;AAAA,QACR;AAAA,MACD;AAAA,IACD,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,gDAAgD,EAAE,YAAY,IAAI,IAAI;AAAA,QACrF,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,WAAO;AAAA,EACR;AAEA,MAAI,KAAK,QAAQ,MAAM,GAAG;AACzB,QAAI;AACH,YAAM,SAAS,QAAQ,QAAQ;AAAA,IAChC,SAAS,OAAO;AACf,YAAM,IAAI,MAAM,yCAAyC,KAAK,KAAK,IAAI;AAAA,QACtE,OAAO;AAAA,MACR,CAAC;AAAA,IACF;AACA,WAAO;AAAA,EACR;AAEA,SAAO;AACR;AAEA,eAAe,iBACd,UACA,OACkB;AAClB,QAAM,QAAQ,qBAAqB,KAAK;AACxC,QAAM,QAAQ,yBAAyB,KAAK;AAC5C,QAAM,OAAO,0BAA0B,KAAK;AAC5C,QAAM,YAAY,8BAA8B,KAAK;AAErD,QAAM,SAAS;AAAA,IACd,8BAA8B,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMpC;AACA,QAAM,SAAS,QAAQ,8BAA8B,KAAK,OAAO,KAAK,eAAe;AACrF,MAAI;AACH,UAAM,SAAS,QAAQ,eAAe,KAAK,uBAAuB,KAAK,yBAAyB;AAAA,EACjG,QAAQ;AACP,UAAM,SAAS,MAAM;AAAA,MACpB;AAAA,MACA,oDAAoD,KAAK;AAAA,MACzD,WAAW,KAAK;AAAA,IACjB;AACA,SAAK,QAAQ,SAAS,OAAO,EAAG,OAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE;AAAA,EACjF;AACA,QAAM,SAAS,QAAQ,6BAA6B,IAAI,mCAAmC,KAAK,EAAE;AAClG,QAAM,SAAS;AAAA,IACd,eAAe,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA,IAKpB,UAAU,KAAK;AAAA,IACf;AAAA,IACA,SAAS,KAAK;AAAA,EACf;AACA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA;AAAA,EAID;AACA,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUtB;AACD,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA,IAGA,SAAS,KAAK;AAAA,IACd;AAAA,EACD;AAEA,aAAW,CAAC,MAAM,IAAI,KAAK;AAAA,IAC1B,CAAC,OAAO,OAAO;AAAA,IACf,CAAC,OAAO,OAAO;AAAA,IACf,CAAC,MAAM,MAAM;AAAA,IACb,CAAC,gCAAgC,SAAS;AAAA,IAC1C,CAAC,sBAAsB,OAAO;AAAA,EAC/B,GAAY;AACX,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA,MAGA;AAAA,MACA;AAAA,IACD;AAAA,EACD;AAEA,QAAM,SAAS,QAAQ,sFAAsF;AAC7G,QAAM,SAAS,QAAQ,kDAAkD,QAAQ,KAAK,EAAE;AACxF,QAAM,SAAS,QAAQ,4BAA4B;AACnD,QAAM,SAAS,QAAQ,8BAA8B,SAAS,gCAAgC;AAC9F,QAAM,SAAS,QAAQ,wBAAwB,SAAS,EAAE;AAC1D,QAAM,UAAU,MAAM;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,SAAS;AAAA,KACjB,SAAS,SAAS,QAAQ;AAAA,EAC5B;AAEA,SAAO;AACR;AAEA,eAAe,gBACd,UACA,MAMkB;AAClB,QAAM,OAAO,KAAK,IAAI,IAAI,KAAK,UAAU;AACzC,QAAM,YAAY,UAAU,YAAY;AACvC,aAAS,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;AACjC,YAAM,SAAS,UAAU,WAAW,KAAK,KAAK,GAAG,CAAC,CAAC;AACnD,YAAM,SAAS,WAAW,KAAK,KAAK,GAAG,EAAE;AACzC,YAAM,QAAQ,WAAW,KAAK,KAAK,MAAM,GAAG;AAC5C,YAAM,QAAQ,GAAG,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,CAAC;AAC7C,YAAM,SAAS;AAAA,QACd;AAAA;AAAA,QAEA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,WAAW,KAAK,MAAM,KAAK,OAAO,MAAS,GAAG,WAAW,KAAK,KAAK,GAAG,GAAG,CAAC;AAAA,MAC3E;AAAA,IACD;AAAA,EACD,CAAC;AACD,SAAO;AACR;AAEA,eAAe,mBACd,UACA,MAMkB;AAClB,QAAM,OAAO,KAAK,IAAI,IAAI,KAAK,UAAU;AACzC,WAAS,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;AACjC,UAAM,KAAK,QAAQ,KAAK,KAAK,IAAI,CAAC;AAClC,UAAMA,WAAU;AAAA,MACf,KAAK;AAAA,MACL,KAAK;AAAA,MACL,MAAS;AAAA,MACT,KAAK,IAAI,KAAK,iBAAiB,KAAM,IAAI,GAAI;AAAA,IAC9C;AACA,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,wBAMqB,KAAK,KAAK,IAAI,CAAC;AAAA,MACpC;AAAA,MACA;AAAA,MACAA;AAAA,MACA,SAASA,QAAO;AAAA,IACjB;AACA,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA;AAAA;AAAA,MAKA;AAAA,MACA;AAAA,MACA,SAASA,QAAO;AAAA,IACjB;AAAA,EACD;AAEA,QAAM,aAAa,eAAe,KAAK,KAAK;AAC5C,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA,IAGA;AAAA,EACD;AACA,WAAS,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;AACjC,UAAMA,WAAU,WAAW,KAAK,MAAM,KAAK,OAAO,MAAS,GAAG,KAAK,IAAI,KAAK,KAAK,eAAe,CAAC;AACjG,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA,MAGA;AAAA,MACAA;AAAA,MACA,SAASA,QAAO;AAAA,MAChB;AAAA,IACD;AAAA,EACD;AACA,QAAM,eAAe,WAAW,KAAK,MAAM,KAAK,OAAO,MAAS,OAAO,GAAG,KAAK,IAAI,KAAK,KAAK,eAAe,CAAC;AAC7G,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA;AAAA;AAAA,IAKA;AAAA,IACA;AAAA,IACA,SAAS,YAAY;AAAA,EACtB;AAEA,SAAO,OAAO,IAAI;AACnB;AAEA,eAAe,oBACd,UACA,MAMkB;AAClB,OAAK,MAAM,SAA4B,UAAU,4CAA4C,IAAI,UAAU,GAAG;AAC7G,UAAM,gBAAgB,UAAU,IAAI;AAAA,EACrC;AAEA,QAAM,OAAO,SAAS;AAAA,IACrB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMD;AACA,QAAM,QAAQ,gBAAgB,UAAU;AAAA,IACvC,MAAM,KAAK;AAAA,IACX,OAAO,KAAK;AAAA,IACZ,KAAK,KAAK;AAAA,IACV,YAAY,KAAK,IAAI,IAAI,KAAK,MAAM,KAAK,aAAa,CAAC,CAAC;AAAA,EACzD,CAAC;AACD,QAAM,CAAC,UAAU,QAAQ,IAAI,MAAM,QAAQ,IAAI,CAAC,MAAM,KAAK,CAAC;AAC5D,QAAM,MAAM,SAAS,CAAC;AACtB,QAAM,aAAa,OAAO,KAAK,eAAe,EAAE;AAChD,QAAM,WAAW,OAAO,KAAK,aAAa,OAAO,GAAG;AACpD,QAAM;AAAA,IACL;AAAA,IACA,KAAK;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,UAAU,IAAI,QAAQ;AAAA,IACzB,aAAa,KAAK,CAAC,OAAO,SAAS,QAAQ;AAAA,EAC5C;AACA,SAAO,WAAW;AACnB;AAEA,eAAe,kBACd,UACA,MAKkB;AAClB,QAAM,OAAO;AAAA,IACZ;AAAA,IACA;AAAA,IACA,QAAQ,IAAI,OAAO,IAAI,CAAC;AAAA,IACxB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,MAAI,MAAM;AACV,aAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AAC1C,QAAI;AACH,YAAM,mBAAmB,UAAU;AAAA,QAClC,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,QACZ,YAAY,MAAS;AAAA,QACrB,MAAM;AAAA,QACN,SAAS;AAAA,QACT,cAAc,KAAK,IAAI,KAAK,iBAAiB,MAAM,KAAK;AAAA,MACzD,CAAC;AAAA,IACF,SAAS,OAAO;AACf,YAAM,IAAI;AAAA,QACT,6CAA6C,KAAK,UAAU,GAAG,CAAC,aAAa,KAAK;AAAA,QAClF,EAAE,OAAO,MAAM;AAAA,MAChB;AAAA,IACD;AACA,WAAO;AAAA,EACR;AACA,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK,GAAG;AAChC,UAAM,UAAU,OAAO,KAAK,KAAK,IAAI,EAAE,SAAS,EAAE,SAAS,GAAG,GAAG,CAAC;AAClE,QAAI;AACH,YAAM,mBAAmB,UAAU;AAAA,QAClC,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,QACZ,YAAY,OAAS;AAAA,QACrB,MAAM,IAAI,MAAM,IAAI,WAAW;AAAA,QAC/B;AAAA,QACA,cAAc,KAAK,IAAI,KAAK,iBAAiB,KAAM,IAAI,EAAG;AAAA,MAC3D,CAAC;AAAA,IACF,SAAS,OAAO;AACf,YAAM,IAAI;AAAA,QACT,gDAAgD,KAAK,UAAU,OAAO,CAAC,aAAa,CAAC;AAAA,QACrF,EAAE,OAAO,MAAM;AAAA,MAChB;AAAA,IACD;AACA,WAAO;AAAA,EACR;AACA,QAAM,YAAY,UAAU,KAAK,OAAO,iBAAiB,gBAAgB,KAAK,KAAK,QAAQ,GAAG;AAC9F,SAAO;AACR;AAEA,eAAe,iBACd,UACA,MAMkB;AAClB,QAAM,aAAa,KAAK,IAAI,GAAG,KAAK,IAAI,yBAAyB,KAAK,eAAe,CAAC;AACtF,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,KAAK,KAAK,oBAAoB,UAAU,CAAC;AACvE,MAAI,UAAU;AACd,WAAS,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;AACjC,UAAM,OAAO,KAAK,IAAI,YAAY,KAAK,oBAAoB,OAAO;AAClE,UAAM,KAAK,UAAU,KAAK,KAAK,IAAI,KAAK,iBAAiB,IAAI,CAAC;AAC9D,UAAMA,WAAU,WAAW,KAAK,MAAM,KAAK,OAAO,MAAU,GAAG,IAAI;AACnE,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAQA;AAAA,MACAA;AAAA,MACA,SAASA,QAAO;AAAA,MAChBA,SAAQ;AAAA,MACR,KAAK,IAAI;AAAA,IACV;AACA,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOA;AAAA,MACA,SAASA,QAAO;AAAA,MAChBA,SAAQ;AAAA,IACT;AACA,eAAW;AAAA,EACZ;AACA,QAAM;AAAA,IACL;AAAA,IACA,KAAK;AAAA,IACL;AAAA,IACA;AAAA,IACA,KAAK;AAAA,IACL;AAAA,IACA,YAAY,KAAK;AAAA,EAClB;AACA,SAAO;AACR;AAEA,eAAe,2BACd,UACA,MAKkB;AAClB,QAAM,KAAK,YAAY,KAAK,KAAK;AACjC,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,iBAAiB,MAAM,CAAC;AACpE,QAAM,eAAe,WAAW,KAAK,MAAM,KAAK,OAAO,MAAS,SAAS;AACzE,QAAM,cAAc,WAAW,KAAK,MAAM,KAAK,OAAO,QAAS,CAAC;AAChE,QAAM,mBAAmB,WAAW,KAAK,MAAM,KAAK,OAAO,QAAS,KAAK,IAAI,MAAM,SAAS,CAAC;AAE7F,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQA;AAAA,IACA;AAAA,IACA,SAAS,YAAY;AAAA,IACrB,aAAa;AAAA,IACb,KAAK,IAAI;AAAA,EACV;AACA,QAAM,SAAS;AAAA,IACd;AAAA,IACA;AAAA,IACA,SAAS,WAAW;AAAA,IACpB,YAAY;AAAA,IACZ,KAAK,IAAI;AAAA,IACT;AAAA,EACD;AACA,QAAM,SAAS,QAAQ,+CAA+C,EAAE;AACxE,QAAM,SAAS,QAAQ,QAAQ;AAC/B,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB;AAAA,IACzB,iBAAiB;AAAA,IACjB,KAAK,IAAI;AAAA,EACV;AACA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOA;AAAA,IACA,SAAS,gBAAgB;AAAA,IACzB,iBAAiB;AAAA,EAClB;AACA,SAAO;AACR;AAEA,eAAe,sBACd,UACA,OACkB;AAClB,QAAM,OAAO,MAAM;AAAA,IAClB;AAAA,IACA;AAAA;AAAA,EAED;AACA,QAAM,OAAO,MAAM;AAAA,IAClB;AAAA,IACA;AAAA;AAAA,EAED;AACA,QAAM,YAAY,UAAU,YAAY;AACvC,eAAW,CAAC,MAAM,MAAM,KAAK,KAAK;AAAA,MACjC,CAAC,SAAS,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC;AAAA,MAC3C,CAAC,QAAQ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC;AAAA,IAC3C,GAAY;AACX,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA;AAAA;AAAA,QAKA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AACD,QAAM,YAAY,UAAU,OAAO,UAAU,kBAAkB,GAAG,GAAG,KAAK;AAC1E,SAAO;AACR;AAEA,eAAe,qBACd,UACA,OACkB;AAClB,QAAM,SAAS,QAAQ,0BAA0B;AACjD,QAAM,cAAc,SAAS,KAAK;AAClC,QAAM,oBAAoB,MAAM;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,GAAG,WAAW;AAAA,EACf;AACA,QAAM,SAAS,mBAAmB,SAAS;AAC3C,QAAM,UAAU,GAAG,WAAW,IAAI,MAAM;AACxC,QAAM,YAAY,QAAQ,KAAK,IAAI,MAAM;AACzC,QAAM,SAAS,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,WAID;AAAA,IACJ;AAAA,MACC,MAAM,YAAY,KAAK,IAAI,MAAM;AAAA,MACjC,KAAK;AAAA,MACL,MAAM,CAAC,YAAY,KAAK,IAAI,MAAM,IAAI,MAAM,GAAG,YAAY,KAAK,IAAI,MAAM,EAAE;AAAA,IAC7E;AAAA,IACA;AAAA,MACC,MAAM,SAAS,KAAK,IAAI,MAAM;AAAA,MAC9B,KAAK;AAAA,MACL,MAAM,CAAC,aAAa,KAAK,IAAI,MAAM,IAAI,MAAM,IAAI,aAAa,KAAK,IAAI,MAAM,EAAE;AAAA,IAChF;AAAA,IACA;AAAA,MACC,MAAM,UAAU,KAAK,IAAI,MAAM;AAAA,MAC/B,KAAK;AAAA,MACL,MAAM,CAAC,cAAc,KAAK,IAAI,MAAM,IAAI,MAAM,GAAG,SAAS;AAAA,IAC3D;AAAA,EACD;AAEA,aAAW,WAAW,UAAU;AAC/B,UAAM,gBAAgB,MAAM;AAAA,MAC3B;AAAA,MACA;AAAA,IACD;AACA,QAAI,SAAS;AACb,QAAI;AACH,YAAM,SAAS,QAAQ,QAAQ,KAAK,GAAG,QAAQ,IAAI;AAAA,IACpD,QAAQ;AACP,eAAS;AAAA,IACV;AACA,UAAM,eAAe,MAAM;AAAA,MAC1B;AAAA,MACA;AAAA,IACD;AACA,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA,MAGA,QAAQ;AAAA,MACR,SAAS,IAAI;AAAA,MACb,eAAe,SAAS;AAAA,MACxB,cAAc,SAAS;AAAA,IACxB;AAAA,EACD;AAEA,QAAM,QAAQ,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,EACD;AACA,OAAK,OAAO,SAAS,QAAQ,QAAQ,SAAS,KAAK,GAAG;AACrD,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA,MAGA,eAAe,KAAK,IAAI,MAAM;AAAA,MAC9B,QAAQ,SAAS;AAAA,MACjB,OAAO,SAAS;AAAA,IACjB;AAAA,EACD;AAEA,QAAM,WAAW,aAAa,KAAK,IAAI,MAAM;AAC7C,QAAM,UAAU,YAAY,KAAK,IAAI,MAAM;AAC3C,QAAM,SAAS;AAAA,IACd;AAAA,IACA;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACD;AACA,QAAM,oBAAoB,MAAM;AAAA,IAC/B;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM,SAAS,QAAQ,2CAA2C,QAAQ;AAC1E,QAAM,mBAAmB,MAAM;AAAA,IAC9B;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,kBAAkB,SAAS;AAAA,KAC1B,mBAAmB,SAAS,OAAO,MAAM,kBAAkB,SAAS,QAAQ;AAAA,EAC9E;AAEA,QAAM,WAAW,MAAM;AAAA,IACtB;AAAA,IACA;AAAA,EACD;AACA,MAAI,WAAW;AACf,MAAI;AACH,UAAM,SAAS;AAAA,MACd;AAAA,MACA,aAAa,KAAK,IAAI,MAAM;AAAA,MAC5B,kBAAkB,KAAK,IAAI,MAAM;AAAA,IAClC;AAAA,EACD,QAAQ;AACP,eAAW;AAAA,EACZ;AACA,QAAM,UAAU,MAAM;AAAA,IACrB;AAAA,IACA;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,UAAU,SAAS,CAAC;AAAA,IACvB,GAAG,SAAS,SAAS,CAAC,IAAI,WAAW,WAAW,UAAU;AAAA,IAC1D,CAAC,aAAa,SAAS,SAAS,QAAQ,UAAU,SAAS;AAAA,EAC5D;AAEA,SAAO,SAAS,SAAS;AAC1B;AAEA,eAAe,iBACd,UACA,OACkB;AAClB,MAAI,MAAM;AACV,aAAW,CAAC,MAAM,UAAU,UAAU,QAAQ,KAAK;AAAA,IAClD,CAAC,gBAAgB,gCAAgC,uBAAuB,UAAU;AAAA,IAClF,CAAC,eAAe,+BAA+B,sBAAsB,UAAU;AAAA,IAC/E,CAAC,cAAc,6BAA6B,qBAAqB,OAAO;AAAA,IACxE,CAAC,gBAAgB,4BAA4B,uBAAuB,GAAG;AAAA,IACvE,CAAC,eAAe,sBAAsB,sBAAsB,UAAU;AAAA,EACvE,GAAY;AACX,QAAI;AACH,YAAM,SAAS,QAAQ,QAAQ;AAC/B,YAAM,OAAO,MAAM,SAAS,QAAQ,QAAQ;AAC5C,YAAM,SAAS,OAAO,YAAY,KAAK,CAAC,CAAC,KAAK,EAAE;AAChD,YAAM;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,aAAa,aAAa,OAAO,WAAW,IAAI,WAAW;AAAA,MAC5D;AAAA,IACD,SAAS,KAAK;AACb,YAAM;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,eAAe,QAAQ,IAAI,UAAU;AAAA,QACrC;AAAA,MACD;AAAA,IACD;AACA,WAAO;AAAA,EACR;AACA,SAAO;AACR;AAEA,eAAe,uBACd,UACA,OACkB;AAClB,QAAM,SAAS,aAAa,KAAK;AACjC,QAAM,eAAe,oBAAoB,KAAK;AAE9C,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI;AACH,UAAM,SAAS;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,SAAS,QAAQ,6BAA6B;AACpD,UAAM,SAAS;AAAA,MACd;AAAA,MACA;AAAA,MACA,QAAU;AAAA,IACX;AACA,UAAM,SAAS;AAAA,MACd;AAAA,MACA;AAAA,IACD;AACA,UAAM,SAAS,QAAQ,+BAA+B;AACtD,UAAM,SAAS,QAAQ,2BAA2B;AAClD,UAAM,SAAS,QAAQ,QAAQ;AAAA,EAChC,SAAS,KAAK;AACb,UAAM,SAAS,QAAQ,UAAU,EAAE,MAAM,MAAM,MAAS;AACxD,UAAM;AAAA,EACP;AAEA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA,IAGA;AAAA,EACD;AAEA,SAAO;AACR;AAEA,eAAe,sBACd,UACA,OACkB;AAClB,QAAM,WAAW,eAAe,QAAQ,CAAC;AACzC,QAAM,SAAS;AAAA,IACd;AAAA,IACA;AAAA,EACD;AAEA,WAAS,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG;AAC9B,UAAM,OAAO,QAAQ,KAAK,IAAI,CAAC;AAC/B,UAAM,SAAS,QAAQ,IAAI;AAC3B,aAAS,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG;AAChD,YAAM,YAAY,UAAU,YAAY;AACvC,cAAM,WAAW,MAAM;AAAA,UACtB;AAAA,UACA;AAAA,UACA;AAAA,QACD;AACA,YAAI,CAAC,UAAU;AACd,gBAAM,SAAS;AAAA,YACd;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACD;AACA,gBAAM,SAAS;AAAA,YACd;AAAA,YACA;AAAA,YACA;AAAA,UACD;AAAA,QACD;AAAA,MACD,CAAC;AAAA,IACF;AAAA,EACD;AAEA,SAAO;AACR;AAEA,eAAe,qBACd,UACgB;AAChB,QAAM,YAAY,UAAU,YAAY;AACvC,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG;AAC9B,YAAM,SAAS;AAAA,QACd;AAAA,QACA,QAAQ,CAAC;AAAA,QACT,QAAQ,CAAC;AAAA,MACV;AAAA,IACD;AACA,aAAS,IAAI,GAAG,IAAI,IAAI,KAAK,GAAG;AAC/B,YAAM,YAAY,WAAW,CAAC;AAC9B,YAAM,aAAa;AACnB,YAAM,SAAS;AAAA,QACd;AAAA,QACA;AAAA,SACC,IAAI,KAAK;AAAA,MACX;AACA,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA,QAGA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AACF;AAEA,eAAe,qBACd,UACA,MAKkB;AAClB,QAAM,qBAAqB,QAAQ;AACnC,QAAM,cAAc,SAAS,KAAK,KAAK,IAAI,KAAK,UAAU;AAC1D,QAAM,iBAAiB,MAAM;AAAA,IAC5B;AAAA,IACA;AAAA,IACA,GAAG,WAAW;AAAA,EACf;AACA,QAAM,UAAU,GAAG,WAAW,IAAI,gBAAgB,SAAS,CAAC;AAC5D,QAAM,SAAS,QAAQ,WAAW,KAAK,KAAK,GAAG,CAAC,CAAC;AACjD,QAAM,YAAY,WAAW,KAAK,KAAK,GAAG,CAAC;AAC3C,MAAI,QAAQ;AAEZ,QAAM,YAAY,UAAU,YAAY;AACvC,UAAM,SAAS;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK,GAAG;AACtC,YAAM,YAAY,WAAW,WAAW,KAAK,KAAK,GAAG,EAAE,CAAC;AACxD,YAAM,UAAU,MAAM;AAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,YAAM,WAAW,WAAW,KAAK,KAAK,GAAG,CAAC;AAC1C,YAAM,QAAQ,SAAS,SAAS;AAChC,eAAS,QAAQ;AACjB,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA,QAGA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA,QAGA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AAAA,IACD;AACA,UAAM,SAAS;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,SAAS;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,EACD,CAAC;AAED,SAAO,YAAY;AACpB;AAEA,eAAe,mBACd,UACA,OACA,WAAW,IACO;AAClB,QAAM,SAAS,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,IACA,YAAY,KAAK;AAAA,EAClB;AACA,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI;AACH,aAAS,IAAI,GAAG,IAAI,UAAU,KAAK,GAAG;AACrC,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA;AAAA,QAIA,YAAY,KAAK,IAAI,CAAC;AAAA,QACtB;AAAA,QACA;AAAA,QACA,SAAS,kBAAkB;AAAA,QAC3B,mBAAmB;AAAA,QACnB,KAAK,IAAI;AAAA,MACV;AAAA,IACD;AACA,UAAM,IAAI,MAAM,4BAA4B;AAAA,EAC7C,QAAQ;AACP,UAAM,SAAS,QAAQ,UAAU;AAAA,EAClC;AACA,QAAM,QAAQ,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA,YAAY,KAAK;AAAA,EAClB;AACA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA,IAGA,kBAAkB,KAAK;AAAA,IACvB,QAAQ,SAAS;AAAA,IACjB,OAAO,SAAS;AAAA,EACjB;AACA,SAAO;AACR;AAEA,eAAe,iBACd,UACA,MAMkB;AAClB,QAAM,UAAU,cAAc,KAAK,KAAK;AACxC,MAAI,MAAM;AACV,QAAM,UAAU,KAAK,IAAI,KAAK,iBAAiB,MAAM;AACrD,QAAM,YAAY,oBAAoB,OAAO,CAAC,SAAS,QAAQ,OAAO;AACtE,MAAI,CAAC,UAAU,SAAS,CAAC,EAAG,WAAU,QAAQ,CAAC;AAC/C,MAAI,CAAC,UAAU,SAAS,OAAO,EAAG,WAAU,KAAK,OAAO;AACxD,aAAW,QAAQ,WAAW;AAC7B,UAAM,mBAAmB,UAAU;AAAA,MAClC,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,YAAY,MAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,MACT,cAAc,KAAK,IAAI,MAAM,KAAK,eAAe;AAAA,IAClD,CAAC;AACD,WAAO;AAAA,EACR;AAEA,QAAM,aAAa,KAAK,UAAU,MAAS;AAC3C,QAAM,gBAAgB,UAAU;AAAA,IAC/B,MAAM,KAAK;AAAA,IACX,OAAO,KAAK;AAAA,IACZ,YAAY;AAAA,IACZ,SAAS,aAAa,KAAK,KAAK;AAAA,IAChC,SAAS;AAAA,IACT,cAAc,KAAK,IAAI,MAAM,KAAK,eAAe;AAAA,EAClD,CAAC;AACD,SAAO;AAEP,MAAI,KAAK,SAAS;AACjB,UAAM,SAAS,QAAQ,gFAAgF;AACvG,aAAS,IAAI,GAAG,IAAI,KAAQ,KAAK,GAAG;AACnC,YAAM,mBAAmB,UAAU;AAAA,QAClC,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,QACZ,YAAY,OAAU;AAAA,QACtB,MAAM;AAAA,QACN,SAAS,cAAc,KAAK,KAAK,IAAI,CAAC;AAAA,QACtC,cAAc,KAAK,IAAI,KAAK,KAAK,eAAe;AAAA,MACjD,CAAC;AACD,aAAO;AAAA,IACR;AACA,aAAS,IAAI,GAAG,IAAI,KAAQ,KAAK,GAAG;AACnC,YAAM,mBAAmB,UAAU;AAAA,QAClC,MAAM,KAAK;AAAA,QACX,OAAO,KAAK;AAAA,QACZ,YAAY,OAAU;AAAA,QACtB,MAAM;AAAA,QACN,SAAS,cAAc,KAAK,KAAK,IAAI,CAAC;AAAA,QACtC,cAAc;AAAA,MACf,CAAC;AACD,aAAO;AAAA,IACR;AACA,UAAM,SAAS,QAAQ,4CAA4C;AAAA,EACpE;AAEA,QAAM,eAAe,KAAK,UAAU,MAAO;AAC3C,QAAM,mBAAmB,UAAU,KAAK,OAAO,YAAY;AAC3D,SAAO;AAEP,SAAO;AACR;AAEA,eAAe,8BACd,UACA,MAKkB;AAClB,MAAI,MAAM;AACV,QAAM,SAAS,qBAAqB,KAAK,KAAK;AAC9C,QAAM,eAAe,KAAK,IAAI,KAAK,iBAAiB,MAAM;AAC1D,MAAI,mBAAmB;AACvB,QAAM,YAAY,UAAU,YAAY;AACvC,aAAS,IAAI,GAAG,IAAI,KAAK,KAAK,GAAG;AAChC,YAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,KAAM,eAAe,KAAK,IAAK,GAAG,CAAC;AACvE,YAAMA,WAAU,WAAW,KAAK,MAAM,KAAK,OAAO,OAAU,GAAG,IAAI;AACnE,yBAAmBA;AACnB,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAQA;AAAA,QACAA;AAAA,QACA,SAASA,QAAO;AAAA,QAChBA,SAAQ;AAAA,QACR,KAAK,IAAI;AAAA,MACV;AACA,aAAO;AAAA,IACR;AACA,UAAM,SAAS;AAAA,MACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOA;AAAA,MACA,SAAS,gBAAgB;AAAA,MACzB,iBAAiB;AAAA,IAClB;AAAA,EACD,CAAC;AAED,QAAM,YAAY,iBAAiB,KAAK,KAAK;AAC7C,QAAM,SAAS;AAAA,IACd;AAAA,IACA;AAAA,EACD;AACA,QAAM,YAAY,UAAU,YAAY;AACvC,aAAS,IAAI,GAAG,IAAI,KAAQ,KAAK,GAAG;AACnC,YAAM,SAAS;AAAA,QACd;AAAA,QACA;AAAA,MACD;AACA,aAAO;AAAA,IACR;AAAA,EACD,CAAC;AACD,QAAMC,WAAU,MAAM;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA,KAAK;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACAA,UAAS,SAAS;AAAA,KACjBA,UAAS,SAAS,QAAQ;AAAA,EAC5B;AAEA,QAAM,UAAU,cAAc,KAAK,KAAK;AACxC,QAAM,SAAS,QAAQ,wFAAwF;AAC/G,QAAM,YAAY,UAAU,YAAY;AACvC,aAAS,IAAI,GAAG,IAAI,KAAQ,KAAK,GAAG;AACnC,YAAM,SAAS;AAAA,QACd;AAAA;AAAA;AAAA,QAGA;AAAA,QACA;AAAA,QACA,WAAW,KAAK,MAAM,KAAK,OAAO,OAAU,GAAG,EAAE;AAAA,MAClD;AACA,aAAO;AAAA,IACR;AACA,UAAM,SAAS,QAAQ,gEAAgE,OAAO;AAC9F,WAAO;AAAA,EACR,CAAC;AACD,QAAM,SAAS,QAAQ,kDAAkD;AACzE,QAAM,YAAY,MAAM;AAAA,IACvB;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA,KAAK;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,SAAS;AAAA,KACnB,WAAW,SAAS,QAAQ;AAAA,EAC9B;AACA,QAAM,YAAY,MAAM;AAAA,IACvB;AAAA,IACA;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA,KAAK;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,SAAS;AAAA,KACnB,WAAW,SAAS,QAAQ;AAAA,EAC9B;AAEA,QAAM,kBAAkB,kBAAkB,KAAK,KAAK;AACpD,QAAM,iBAAiB,MAAM;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI;AACH,aAAS,IAAI,GAAG,IAAI,KAAM,KAAK,GAAG;AACjC,YAAM,SAAS;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,aAAO;AAAA,IACR;AACA,UAAM,SAAS,QAAQ,UAAU;AAAA,EAClC,SAAS,KAAK;AACb,UAAM,SAAS,QAAQ,UAAU,EAAE,MAAM,MAAM,MAAS;AACxD,UAAM;AAAA,EACP;AACA,QAAM,gBAAgB,MAAM;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA,KAAK;AAAA,IACL;AAAA,IACA;AAAA,IACA,gBAAgB,SAAS;AAAA,IACzB,eAAe,SAAS;AAAA,KACvB,eAAe,SAAS,SAAS,gBAAgB,SAAS;AAAA,EAC5D;AAEA,SAAO;AACR;AAEA,SAAS,sBAAsBC,OAAoB,UAAiC;AACnF,SAAOA,UAAS,YAAYA,UAAS,kBAAkBA,UAAS;AACjE;AAEA,eAAe,mBACd,UACA,MAUgB;AAChB,QAAM,cAAc,OAAU,MAAc,OAAqC;AAChF,QAAI;AACH,aAAO,MAAM,GAAG;AAAA,IACjB,SAAS,OAAO;AACf,YAAM,SACL,iBAAiB,QAAQ,MAAM,UAAU,OAAO,UAAU,WAAW,QAAQ,OAAO,KAAK;AAC1F,YAAM,IAAI;AAAA,QACT,iBAAiB,IAAI,mBAAmB,KAAK,IAAI,iBAAiB,KAAK,KAAK,KAAK,MAAM;AAAA,QACvF,EAAE,OAAO,MAAM;AAAA,MAChB;AAAA,IACD;AAAA,EACD;AAEA,MAAI,sBAAsB,KAAK,MAAM,MAAM,KAAK,KAAK,SAAS,YAAY;AACzE,SAAK,IAAI,eAAe,KAAK,IAAI,eAAe,KAC/C,MAAM,YAAY,QAAQ,MAAM,kBAAkB,UAAU,IAAI,CAAC;AAAA,EACnE;AACA,MAAI,KAAK,SAAS,cAAc;AAC/B,SAAK,IAAI,aAAa,KAAK,IAAI,aAAa,KAC3C,MAAM,YAAY,cAAc,MAAM,sBAAsB,UAAU,IAAI,CAAC;AAAA,EAC7E;AACA,MAAI,sBAAsB,KAAK,MAAM,eAAe,GAAG;AACtD,SAAK,IAAI,iBAAiB,KAAK,IAAI,iBAAiB,KACnD,MAAM,YAAY,iBAAiB,MAAM,wBAAwB,UAAU,IAAI,CAAC;AAAA,EAClF;AACA,MAAI,sBAAsB,KAAK,MAAM,QAAQ,GAAG;AAC/C,SAAK,IAAI,UAAU,KAAK,IAAI,UAAU,KACrC,MAAM,YAAY,UAAU,MAAM,iBAAiB,UAAU,KAAK,KAAK,CAAC;AAAA,EAC1E;AACA,MAAI,sBAAsB,KAAK,MAAM,OAAO,GAAG;AAC9C,SAAK,IAAI,SAAS,KAAK,IAAI,SAAS,KACnC,MAAM,YAAY,SAAS,MAAM,gBAAgB,UAAU,IAAI,CAAC;AAAA,EAClE;AACA,MAAI,sBAAsB,KAAK,MAAM,aAAa,GAAG;AACpD,SAAK,IAAI,eAAe,KAAK,IAAI,eAAe,KAC/C,MAAM,YAAY,eAAe,MAAM,qBAAqB,UAAU,KAAK,KAAK,CAAC;AAAA,EACnF;AACA,MAAI,sBAAsB,KAAK,MAAM,YAAY,GAAG;AACnD,SAAK,IAAI,cAAc,KAAK,IAAI,cAAc,KAC7C,MAAM,YAAY,cAAc,MAAM,uBAAuB,UAAU,KAAK,KAAK,CAAC;AAAA,EACpF;AACA,MAAI,sBAAsB,KAAK,MAAM,QAAQ,GAAG;AAC/C,SAAK,IAAI,UAAU,KAAK,IAAI,UAAU,KACrC,MAAM,YAAY,UAAU,MAAM,iBAAiB,UAAU,KAAK,KAAK,CAAC;AAAA,EAC1E;AACA,MAAI,sBAAsB,KAAK,MAAM,UAAU,GAAG;AACjD,SAAK,IAAI,YAAY,KAAK,IAAI,YAAY,KACzC,MAAM,YAAY,YAAY,MAAM,mBAAmB,UAAU,IAAI,CAAC;AAAA,EACxE;AACA,MAAI,sBAAsB,KAAK,MAAM,QAAQ,GAAG;AAC/C,SAAK,IAAI,UAAU,KAAK,IAAI,UAAU,KACrC,MAAM,YAAY,UAAU,MAAM,iBAAiB,UAAU,IAAI,CAAC;AAAA,EACpE;AACA,MAAI,sBAAsB,KAAK,MAAM,WAAW,GAAG;AAClD,SAAK,IAAI,aAAa,KAAK,IAAI,aAAa,KAC3C,MAAM,YAAY,aAAa,MAAM,oBAAoB,UAAU,IAAI,CAAC;AAAA,EAC1E;AACA,MAAI,sBAAsB,KAAK,MAAM,UAAU,GAAG;AACjD,SAAK,IAAI,YAAY,KAAK,IAAI,YAAY,KACzC,MAAM,YAAY,YAAY,MAAM,2BAA2B,UAAU,IAAI,CAAC;AAAA,EAChF;AACA,MAAI,sBAAsB,KAAK,MAAM,eAAe,GAAG;AACtD,SAAK,IAAI,gBAAgB,KAAK,IAAI,gBAAgB,KACjD,MAAM,YAAY,iBAAiB,MAAM,kBAAkB,UAAU,IAAI,CAAC;AAAA,EAC5E;AACA,MAAI,sBAAsB,KAAK,MAAM,YAAY,GAAG;AACnD,UAAM,SAAS,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,aAAa,EAAE,CAAC;AAC3D,aAAS,IAAI,GAAG,IAAI,QAAQ,KAAK,GAAG;AACnC,WAAK,IAAI,cAAc,KAAK,IAAI,cAAc,KAC7C,MAAM;AAAA,QAAY;AAAA,QAAc,MAC/B,qBAAqB,UAAU;AAAA,UAC9B,OAAO,KAAK;AAAA,UACZ,YAAY;AAAA,UACZ,KAAK,KAAK;AAAA,QACX,CAAC;AAAA,MACF;AAAA,IACF;AAAA,EACD;AACA,MAAI,KAAK,SAAS,kBAAkB,KAAK,SAAS,SAAS;AAC1D,SAAK,IAAI,cAAc,KAAK,IAAI,cAAc,KAC7C,MAAM,YAAY,cAAc,MAAM,sBAAsB,UAAU,KAAK,KAAK,CAAC;AAClF,SAAK,IAAI,SAAS,KAAK,IAAI,SAAS,KACnC,MAAM;AAAA,MAAY;AAAA,MAAS,MAC1B,iBAAiB,UAAU,EAAE,GAAG,MAAM,SAAS,KAAK,SAAS,QAAQ,CAAC;AAAA,IACvE;AAAA,EACF;AACA,MAAI,KAAK,SAAS,gBAAgB;AACjC,SAAK,IAAI,SAAS,KAAK,IAAI,SAAS,KACnC,MAAM,YAAY,gBAAgB,MAAM,8BAA8B,UAAU,IAAI,CAAC;AAAA,EACvF;AACA,MAAI,sBAAsB,KAAK,MAAM,QAAQ,GAAG;AAC/C,SAAK,IAAI,UAAU,KAAK,IAAI,UAAU,KACrC,MAAM,YAAY,UAAU,MAAM,sBAAsB,UAAU,KAAK,KAAK,CAAC;AAAA,EAC/E;AACD;AAEA,SAAS,WACRA,OACA,KACiE;AACjE,QAAM,OAAO,IAAI;AACjB,MAAIA,UAAS,gBAAgB;AAC5B,QAAI,OAAO,KAAM,QAAO;AACxB,QAAI,OAAO,KAAM,QAAO;AACxB,QAAI,OAAO,IAAK,QAAO;AACvB,WAAO;AAAA,EACR;AACA,MAAIA,UAAS,OAAO;AACnB,QAAI,OAAO,IAAK,QAAO;AACvB,QAAI,OAAO,KAAM,QAAO;AACxB,QAAI,OAAO,IAAK,QAAO;AACvB,WAAO;AAAA,EACR;AACA,MAAIA,UAAS,YAAY;AACxB,QAAI,OAAO,IAAK,QAAO;AACvB,QAAI,OAAO,IAAK,QAAO;AACvB,QAAI,OAAO,IAAK,QAAO;AACvB,WAAO;AAAA,EACR;AACA,MAAI,OAAO,IAAK,QAAO;AACvB,MAAI,OAAO,KAAM,QAAO;AACxB,MAAI,OAAO,KAAM,QAAO;AACxB,MAAI,OAAO,KAAM,QAAO;AACxB,MAAI,OAAO,KAAM,QAAO;AACxB,SAAO;AACR;AAEA,eAAe,SACd,UAC6B;AAC7B,QAAM,YAAY,MAAM;AAAA,IACvB;AAAA,IACA;AAAA,EACD;AACA,QAAM,QAAQ,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,EACD;AACA,QAAM,SAAS,MAAM;AAAA,IASpB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAiBD;AACA,QAAM,aAAa,MAAM;AAAA,IAMxB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA4CD;AACA,QAAM,WAAW,MAAM;AAAA,IAKtB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IASA,gBAAgB;AAAA,IAChB,gBAAgB;AAAA,EACjB;AACA,QAAM,OAAO,MAAM;AAAA,IAKlB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeD;AACA,QAAM,aAAa,MAAM;AAAA,IAIxB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAqBD;AACA,QAAM,aAAa,MAAM;AAAA,IAIxB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA0BD;AACA,QAAM,cAAc,MAAM;AAAA,IAIzB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUD;AACA,QAAM,aAAa,MAAM;AAAA,IAIxB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWD;AACA,QAAM,cAAc,MAAM;AAAA,IAIzB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYD;AACA,QAAM,SAAS,MAAM;AAAA,IAIpB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA,EAKD;AACA,QAAM,SAAS,MAAM;AAAA,IAIpB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA,EAID;AACA,QAAM,WAAW,MAAM;AAAA,IAItB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWD;AACA,QAAM,SAAS,MAAM;AAAA,IAIpB;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAmBD;AAEA,QAAM,UAA6B;AAAA,IAClC,aAAa,QAAQ,gBAAgB;AAAA,IACrC,YAAY,QAAQ,eAAe;AAAA,IACnC,cAAc,QAAQ,iBAAiB;AAAA,IACvC,aAAa,YAAY,gBAAgB;AAAA,IACzC,WAAW,YAAY,cAAc;AAAA,IACrC,gBAAgB,YAAY,mBAAmB;AAAA,IAC/C,eAAe,YAAY,kBAAkB;AAAA,IAC7C,kBAAkB,QAAQ,sBAAsB;AAAA,IAChD,oBAAoB,QAAQ,wBAAwB;AAAA,IACpD,0BAA0B,QAAQ,+BAA+B;AAAA,IACjE,4BAA4B,QAAQ,iCAAiC;AAAA,IACrE,cAAc,UAAU,iBAAiB;AAAA,IACzC,mBAAmB,UAAU,uBAAuB;AAAA,IACpD,2BAA2B,gBAAgB;AAAA,IAC3C,wBAAwB,UAAU,4BAA4B;AAAA,IAC9D,gBAAgB,WAAW,mBAAmB;AAAA,IAC9C,YAAY,OAAO,eAAe;AAAA,IAClC,UAAU,MAAM,aAAa;AAAA,IAC7B,kBAAkB,MAAM,sBAAsB;AAAA,IAC9C,gBAAgB,MAAM,mBAAmB;AAAA,IACzC,WAAW,YAAY,cAAc;AAAA,IACrC,iBAAiB,YAAY,oBAAoB;AAAA,IACjD,kBAAkB,YAAY,qBAAqB;AAAA,IACnD,sBAAsB,YAAY,yBAAyB;AAAA,IAC3D,oBAAoB,aAAa,uBAAuB;AAAA,IACxD,iBAAiB,aAAa,oBAAoB;AAAA,IAClD,eAAe,YAAY,kBAAkB;AAAA,IAC7C,qBAAqB,YAAY,wBAAwB;AAAA,IACzD,eAAe,aAAa,kBAAkB;AAAA,IAC9C,sBAAsB,aAAa,yBAAyB;AAAA,IAC5D,eAAe,QAAQ,kBAAkB;AAAA,IACzC,sBAAsB,QAAQ,0BAA0B;AAAA,IACxD,WAAW,QAAQ,cAAc;AAAA,IACjC,iBAAiB,QAAQ,oBAAoB;AAAA,IAC7C,cAAc,UAAU,iBAAiB;AAAA,IACzC,oBAAoB,UAAU,uBAAuB;AAAA,IACrD,YAAY,QAAQ,eAAe;AAAA,IACnC,kBAAkB,QAAQ,qBAAqB;AAAA,EAChD;AAEA,SAAO;AACR;AAEA,eAAe,oBACd,UACA,QAAQ,GAIN;AACF,QAAM,OAAQ,MAAM,SAAS;AAAA,IAC5B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAqCA;AAAA,EACD;AAcA,QAAM,iBAAiB,KAAK,IAAI,CAAC,SAAS;AAAA,IACzC,SAAS,IAAI;AAAA,IACb,aAAa,IAAI;AAAA,IACjB,eAAe,IAAI;AAAA,IACnB,eAAe,IAAI;AAAA,IACnB,iBAAiB,IAAI;AAAA,IACrB,mBAAmB,IAAI;AAAA,IACvB,qBAAqB,IAAI;AAAA,IACzB,uBAAuB,IAAI;AAAA,IAC3B,yBAAyB,IAAI;AAAA,IAC7B,oBAAoB,IAAI;AAAA,IACxB,sBAAsB,IAAI;AAAA,EAC3B,EAAE;AAEF,QAAM,oBAAyD,CAAC;AAChE,aAAW,OAAO,gBAAgB;AACjC,UAAM,SAAU,MAAM,SAAS;AAAA,MAC9B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAgBA,IAAI;AAAA,IACL;AAaA,sBAAkB,IAAI,OAAO,IAAI,OAAO,IAAI,CAACC,aAAW;AAAA,MACvD,KAAKA,QAAM;AAAA,MACX,OAAOA,QAAM;AAAA,MACb,YAAYA,QAAM;AAAA,MAClB,MAAMA,QAAM;AAAA,MACZ,SAASA,QAAM;AAAA,MACf,OAAOA,QAAM;AAAA,MACb,SAASA,QAAM;AAAA,MACf,aAAaA,QAAM;AAAA,MACnB,iBAAiBA,QAAM;AAAA,MACvB,cAAcA,QAAM;AAAA,MACpB,SAASA,QAAM;AAAA,IAChB,EAAE;AAAA,EACH;AAEA,SAAO,EAAE,gBAAgB,kBAAkB;AAC5C;AAEO,IAAM,kBAAkBL,QAAM;AAAA,EACpC,SAAS;AAAA,IACR,eAAe;AAAA,EAChB;AAAA,EACA,IAAIC,IAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAgBtB;AACD,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAYtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAStB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQtB;AACD,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAStB;AACD,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAStB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA,IAItB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOtB;AAAA,IACF;AAAA,EACD,CAAC;AAAA,EACD,SAAS;AAAA,IACR,OAAO,OAAO,MAAM;AACnB,YAAM,EAAE,GAAG,QAAQ,6BAA6B;AAChD,YAAM,EAAE,GAAG,QAAQ,gCAAgC;AACnD,YAAM,EAAE,GAAG,QAAQ,mCAAmC;AACtD,YAAM,EAAE,GAAG,QAAQ,wCAAwC;AAC3D,YAAM,EAAE,GAAG,QAAQ,iCAAiC;AACpD,YAAM,EAAE,GAAG,QAAQ,gCAAgC;AACnD,YAAM,EAAE,GAAG,QAAQ,kCAAkC;AACrD,YAAM,EAAE,GAAG,QAAQ,qCAAqC;AACxD,YAAM,EAAE,GAAG,QAAQ,iCAAiC;AACpD,YAAM,EAAE,GAAG,QAAQ,yCAAyC;AAC5D,YAAM,EAAE,GAAG,QAAQ,6BAA6B;AAChD,YAAM,EAAE,GAAG,QAAQ,2BAA2B;AAC9C,YAAM,EAAE,GAAG,QAAQ,4BAA4B;AAC/C,YAAM,EAAE,GAAG,QAAQ,sCAAsC;AACzD,YAAM,EAAE,GAAG,QAAQ,8BAA8B;AACjD,YAAM,EAAE,GAAG,QAAQ,2BAA2B;AAC9C,YAAM,EAAE,GAAG,QAAQ,8BAA8B;AACjD,YAAM,EAAE,GAAG,QAAQ,yBAAyB;AAC5C,YAAM,EAAE,GAAG,QAAQ,4BAA4B;AAC/C,YAAM,EAAE,GAAG,QAAQ,+BAA+B;AAClD,YAAM,EAAE,GAAG,QAAQ,4BAA4B;AAC/C,YAAM,EAAE,GAAG,QAAQ,0BAA0B;AAC7C,YAAM,EAAE,GAAG,QAAQ,gCAAgC;AACnD,YAAM,EAAE,GAAG,QAAQ,oCAAoC;AACvD,YAAM,EAAE,GAAG,QAAQ,gCAAgC;AACnD,YAAM,EAAE,GAAG,QAAQ,kCAAkC;AACrD,YAAM,EAAE,GAAG,QAAQ,2BAA2B;AAC9C,YAAM,EAAE,GAAG,QAAQ,8BAA8B;AACjD,YAAM,EAAE,GAAG,QAAQ,wBAAwB;AAC3C,YAAM,eAAe,EAAE,EAAE;AACzB,aAAO,MAAM,SAAS,EAAE,EAAE;AAAA,IAC3B;AAAA,IAEA,UAAU,OAAO,GAAG,UAA+C;AAClE,YAAMG,QAAO,MAAM,QAAQ;AAC3B,YAAM,aAAa,KAAK,IAAI,GAAG,KAAK,MAAM,MAAM,UAAU,CAAC;AAC3D,YAAM,WAAW,KAAK,IAAI,GAAG,KAAK,MAAM,MAAM,YAAY,iBAAiB,CAAC;AAC5E,YAAM,kBAAkB,KAAK;AAAA,QAC5B;AAAA,QACA,KAAK,MAAM,MAAM,mBAAmB,yBAAyB;AAAA,MAC9D;AACA,YAAM,oBAAoB,KAAK;AAAA,QAC9B;AAAA,QACA,KAAK,MAAM,MAAM,qBAAqB,2BAA2B;AAAA,MAClE;AACA,YAAM,MAAM,QAAQ,GAAG,MAAM,IAAI,IAAI,MAAM,KAAK,IAAIA,KAAI,EAAE;AAC1D,YAAM,MAA8B,CAAC;AACrC,UAAI,QAAQ;AAEZ,UAAI;AACH,cAAM,eAAe,EAAE,EAAE;AAEzB,iBAAS,IAAI,GAAG,IAAI,YAAY,KAAK,GAAG;AACvC,gBAAM,OAAO,WAAWA,OAAM,GAAG;AACjC,cAAI,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK;AAC/B,kBAAQ,QAAQ,IAAI,cAAc,CAAC;AAEnC,cAAI,SAAS,YAAY;AACxB,kBAAM,YAAY,WAAW,KAAK,GAAG,gBAAgB,CAAC;AACtD,gBAAI,UAAU,WAAW,KAAK,GAAG,gBAAgB,CAAC;AAClD,gBAAI,YAAY,UAAW,YAAW,UAAU,KAAK;AACrD,kBAAM,cAAc,QAAQ,SAAS;AACrC,kBAAM,YAAY,QAAQ,OAAO;AACjC,gBAAI;AACH,oBAAM,cAAc,EAAE,IAAI;AAAA,gBACzB,OAAO,MAAM;AAAA,gBACb,YAAY;AAAA,gBACZ;AAAA,gBACA;AAAA,gBACA,QAAQ,WAAW,KAAK,GAAG,GAAG;AAAA,cAC/B,CAAC;AAAA,YACF,SAAS,OAAO;AACf,oBAAM,IAAI;AAAA,gBACT,+CAA+C,CAAC,SAAS,WAAW,OAAO,SAAS;AAAA,gBACpF,EAAE,OAAO,MAAM;AAAA,cAChB;AAAA,YACD;AAAA,UACD,WAAW,SAAS,OAAO;AAC1B,kBAAM,UAAU,OAAO,WAAW,KAAK,GAAG,CAAC,CAAC;AAC5C,kBAAM,UAAU,WAAW,KAAK,GAAGA,UAAS,QAAQ,KAAK,CAAC;AAC1D,gBAAI;AACH,oBAAM,gBAAgB,EAAE,IAAI;AAAA,gBAC3B,MAAM,MAAM;AAAA,gBACZ,OAAO,MAAM;AAAA,gBACb,YAAY;AAAA,gBACZ;AAAA,gBACA;AAAA,gBACA,cAAc,WAAW,KAAK,GAAG,eAAe;AAAA,cACjD,CAAC;AAAA,YACF,SAAS,OAAO;AACf,oBAAM,IAAI;AAAA,gBACT,0CAA0C,CAAC,QAAQ,OAAO,SAAS,OAAO;AAAA,gBAC1E,EAAE,OAAO,MAAM;AAAA,cAChB;AAAA,YACD;AAAA,UACD,OAAO;AACN,kBAAM,UACLA,UAAS,SAAS,IAAI,IAAI,MACvB,OAAO,WAAW,KAAK,GAAG,CAAC,CAAC,KAC5B,QAAQ,WAAW,KAAK,GAAG,WAAW,CAAC,CAAC;AAC5C,kBAAM,eACLA,UAAS,aACN,WAAW,KAAK,KAAK,IAAI,KAAK,eAAe,GAAG,eAAe,IAC/D,WAAW,KAAK,GAAG,eAAe;AACtC,gBAAI;AACH,oBAAM,mBAAmB,EAAE,IAAI;AAAA,gBAC9B,MAAM,MAAM;AAAA,gBACZ,OAAO,MAAM;AAAA,gBACb,YAAY;AAAA,gBACZ;AAAA,gBACA;AAAA,gBACA;AAAA,cACD,CAAC;AAAA,YACF,SAAS,OAAO;AACf,oBAAM,IAAI;AAAA,gBACT,kBAAkB,IAAI,wBAAwB,CAAC,QAAQ,KAAK,UAAU,OAAO,CAAC,sBAAsB,YAAY;AAAA,gBAChH,EAAE,OAAO,MAAM;AAAA,cAChB;AAAA,YACD;AAAA,UACD;AAAA,QACD;AAEA,gBAAQ;AACR,cAAM,mBAAmB,EAAE,IAAI;AAAA,UAC9B,MAAM,MAAM;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,MAAAA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACD,CAAC;AAED,gBAAQ;AACR,eAAO;AAAA,UACN,MAAM,MAAM;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,MAAAA;AAAA,UACA;AAAA,UACA;AAAA,UACA,YAAY,MAAM,SAAS,EAAE,EAAE;AAAA,QAChC;AAAA,MACD,SAAS,OAAO;AACf,cAAM,SACL,iBAAiB,QAAQ,MAAM,UAAU,OAAO,UAAU,WAAW,QAAQ,OAAO,KAAK;AAC1F,cAAM,IAAI;AAAA,UACT,0BAA0B,KAAK,aAAaA,KAAI,UAAU,MAAM,KAAK,SAAS,MAAM,IAAI,KAAK,MAAM;AAAA,UACnG,EAAE,OAAO,MAAM;AAAA,QAChB;AAAA,MACD;AAAA,IACD;AAAA,IAEA,UAAU,OAAO,MAAM;AACtB,YAAM,eAAe,EAAE,EAAE;AACzB,aAAO,MAAM,SAAS,EAAE,EAAE;AAAA,IAC3B;AAAA,IAEA,qBAAqB,OAAO,GAAG,UAAmB;AACjD,YAAM,eAAe,EAAE,EAAE;AACzB,aAAO,MAAM,oBAAoB,EAAE,IAAI,SAAS,CAAC;AAAA,IAClD;AAAA,IAEA,WAAW,CAAC,MAAM;AACjB,QAAE,MAAM;AACR,aAAO,EAAE,IAAI,KAAK;AAAA,IACnB;AAAA,EACD;AACD,CAAC;;;ACn8FD,SAAS,SAAAE,eAAa;AAEtB,SAAS,MAAAC,YAAU;AAuBnB,IAAM,sBAAsB;AAC5B,IAAMC,qBAAoB,KAAK;AAG/B,IAAM,oBAAoB;AAC1B,IAAM,oBAAoB;AAE1B,SAAS,UAAU,OAA2B,UAA0B;AACvE,MAAI,UAAU,OAAW,QAAO;AAChC,MAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GAAG;AACzC,UAAM,IAAI,MAAM,8CAA8C,KAAK,EAAE;AAAA,EACtE;AACA,SAAO,KAAK,MAAM,KAAK;AACxB;AAEA,SAAS,kBACR,SAC6B;AAC7B,MAAI,CAAC,QAAS,QAAO;AACrB,QAAM,MAAM;AACZ,QAAMC,eAAc,CAAC,OAAe,UACnC,OAAO,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,CAAC;AACrC,SAAO;AAAA,IACN,gBAAgBA,aAAY,kBAAkB,kBAAkB;AAAA,IAChE,aAAaA,aAAY,eAAe,cAAc;AAAA,IACtD,aAAaA,aAAY,eAAe,cAAc;AAAA,IACtD,eAAeA,aAAY,iBAAiB,iBAAiB;AAAA,IAC7D,SAASA,aAAY,WAAW,UAAU;AAAA,IAC1C,aAAaA,aAAY,eAAe,cAAc;AAAA,IACtD,kBAAkBA,aAAY,oBAAoB,oBAAoB;AAAA,IACtE,uBAAuBA;AAAA,MACtB;AAAA,MACA;AAAA,IACD;AAAA,IACA,wBAAwBA;AAAA,MACvB;AAAA,MACA;AAAA,IACD;AAAA,IACA,uBAAuBA;AAAA,MACtB;AAAA,MACA;AAAA,IACD;AAAA,IACA,aAAaA,aAAY,eAAe,eAAe;AAAA,EACxD;AACD;AAEA,eAAeC,UACd,UACA,QACG,MACU;AACb,QAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,GAAG,IAAI;AAChD,MAAI,CAAC,KAAK,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B,GAAG,EAAE;AAC9D,SAAO,KAAK,CAAC;AACd;AAEA,eAAe,aAAa,UAMJ;AACvB,QAAM,CAAC,WAAW,eAAe,QAAQ,IAAI,MAAM,QAAQ,IAAI;AAAA,IAC9DA,UAAiC,UAAU,mBAAmB;AAAA,IAC9DA,UAAqC,UAAU,uBAAuB;AAAA,IACtEA,UAAgC,UAAU,kBAAkB;AAAA,EAC7D,CAAC;AAED,QAAM,gBAAgB,MAAM,SAAS,gBAAgB;AACrD,QAAM,gBAAgB,kBAAkB,aAAa;AAErD,SAAO;AAAA,IACN,YAAY,UAAU;AAAA,IACtB,gBAAgB,cAAc;AAAA,IAC9B,WAAW,SAAS;AAAA,IACpB,KAAK;AAAA,EACN;AACD;AAEO,IAAM,uBAAuBJ,QAAM;AAAA,EACzC,SAAS;AAAA,IACR,eAAe;AAAA,EAChB;AAAA,EACA,OAAO;AAAA,IACN,YAAY;AAAA,EACb;AAAA,EACA,IAAIC,KAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUtB;AACD,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWtB;AAAA,IACF;AAAA,EACD,CAAC;AAAA,EACD,SAAS,CAAC,MAAM;AACf,MAAE,MAAM,cAAc;AACtB,YAAQ;AAAA,MACP,KAAK,UAAU;AAAA,QACd,MAAM;AAAA,QACN,SAAS,EAAE;AAAA,QACX,YAAY,EAAE,MAAM;AAAA,QACpB,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MACnC,CAAC;AAAA,IACF;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,OAAO,OAAO,MAAM;AACnB,YAAM,EAAE,GAAG,QAAQ,6BAA6B;AAChD,YAAM,EAAE,GAAG,QAAQ,2BAA2B;AAC9C,YAAM,EAAE,GAAG,QAAQ,QAAQ;AAC3B,aAAO;AAAA,QACN,IAAI;AAAA,QACJ,SAAS,MAAM,aAAa,EAAE,EAAE;AAAA,MACjC;AAAA,IACD;AAAA,IAEA,WAAW,CAAC,MAAM;AACjB,QAAE,MAAM;AACR,aAAO,EAAE,IAAI,KAAK;AAAA,IACnB;AAAA,IAEA,gBAAgB,OAAO,MAAM;AAC5B,YAAM,SAAS,MAAM,aAAa,EAAE,EAAE;AAKtC,aAAO;AAAA,QACN,IAAI;AAAA,QACJ;AAAA,QACA,OAAO,MAAM,aAAa,EAAE,EAAE;AAAA,MAC/B;AAAA,IACD;AAAA,IAEA,OAAO,OAAO,MAAM;AACnB,YAAM,WAAW,MAAMG;AAAA,QAKtB,EAAE;AAAA,QACF;AAAA,MACD;AACA,YAAM,SAAS,MAAMA;AAAA,QACpB,EAAE;AAAA,QACF;AAAA,MACD;AACA,YAAM,YAAY,MAAMA;AAAA,QACvB,EAAE;AAAA,QACF;AAAA,MACD;AAEA,aAAO;AAAA,QACN,YAAY,SAAS;AAAA,QACrB,aAAa,SAAS,gBAAgB;AAAA,QACtC,cAAc,SAAS,eAAe;AAAA,QACtC,QAAQ,OAAO;AAAA,QACf,gBAAgB,UAAU;AAAA,QAC1B,SAAS,MAAM,aAAa,EAAE,EAAE;AAAA,MACjC;AAAA,IACD;AAAA,IAEA,UAAU,OAAO,GAAG,UAAyB;AAC5C,YAAM,YAAY,YAAY,IAAI;AAClC,YAAM,aAAa,UAAU,MAAM,YAAY,mBAAmB;AAClE,YAAM,WAAW,UAAU,MAAM,UAAUF,kBAAiB;AAM5D,YAAM,WAAW,KAAK,IAAI,GAAG,UAAU,MAAM,UAAU,iBAAiB,CAAC;AACzE,YAAM,MAAM,KAAK,IAAI;AACrB,UAAI,eAAe;AACnB,YAAM,WAAW,CAChB,OACA,OACA,SAAkC,CAAC,MAC/B;AACJ,gBAAQ;AAAA,UACP,KAAK,UAAU;AAAA,YACd,MAAM;AAAA,YACN,SAAS,EAAE;AAAA,YACX,MAAM,MAAM;AAAA,YACZ,OAAO,MAAM;AAAA,YACb;AAAA,YACA;AAAA,YACA,WAAW,YAAY,IAAI,IAAI;AAAA,YAC/B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,YAClC,GAAG;AAAA,UACJ,CAAC;AAAA,QACF;AAAA,MACD;AACA,YAAM,eAAe,OACpB,OACA,QACG,SACC;AACJ,cAAM,iBAAiB,YAAY,IAAI;AACvC,iBAAS,OAAO,SAAS,EAAE,UAAU,KAAK,OAAO,CAAC;AAClD,YAAI;AACH,gBAAM,OAAO,MAAM,EAAE,GAAG,QAAQ,KAAK,GAAG,IAAI;AAC5C,mBAAS,OAAO,OAAO;AAAA,YACtB,YAAY,YAAY,IAAI,IAAI;AAAA,YAChC,UAAU,KAAK;AAAA,UAChB,CAAC;AACD,iBAAO;AAAA,QACR,SAAS,KAAK;AACb,mBAAS,OAAO,SAAS;AAAA,YACxB,YAAY,YAAY,IAAI,IAAI;AAAA,YAChC,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,UACvD,CAAC;AACD,gBAAM;AAAA,QACP;AAAA,MACD;AACA,eAAS,aAAa,SAAS;AAAA,QAC9B;AAAA,QACA;AAAA,QACA;AAAA,MACD,CAAC;AAED,YAAM,aAAa,SAAS,OAAO;AACnC,UAAI;AACH,eAAO,eAAe,YAAY;AACjC,gBAAM,YAAY,KAAK;AAAA,YACtB;AAAA,YACA,aAAa;AAAA,UACd;AACA,gBAAM,eAAyB,CAAC;AAChC,gBAAM,OAAkB,CAAC;AAEzB,mBAAS,IAAI,GAAG,IAAI,WAAW,KAAK,GAAG;AACtC,kBAAM,WAAW,eAAe;AAChC,yBAAa,KAAK,gCAAgC;AAClD,iBAAK;AAAA,cACJ,MAAM;AAAA,cACN,MAAM;AAAA,eACL,MAAM,QAAQ,YAAY;AAAA,cAC3B;AAAA,cACA,MAAM;AAAA,YACP;AAAA,UACD;AAEA,gBAAM;AAAA,YACL;AAAA,YACA,8FAA8F,aAAa,KAAK,IAAI,CAAC;AAAA,YACrH,GAAG;AAAA,UACJ;AACA,0BAAgB;AAChB,mBAAS,yBAAyB,OAAO;AAAA,YACxC;AAAA,YACA;AAAA,UACD,CAAC;AAAA,QACF;AACA,cAAM,aAAa,UAAU,QAAQ;AAAA,MACtC,SAAS,KAAK;AACb,cAAM,aAAa,YAAY,UAAU,EAAE,MAAM,MAAM,MAAS;AAChE,cAAM;AAAA,MACP;AAEA,YAAM,OAAO,MAAM;AAAA,QAClB;AAAA,QACA;AAAA,QACA;AAAA,MACD;AACA,YAAM,YAAY,MAAM;AAAA,QACvB;AAAA,QACA;AAAA,QACA,MAAM,QAAQ;AAAA,QACb,MAAM,QAAQ,KAAM;AAAA,MACtB;AACA,YAAM;AAAA,QACL;AAAA,QACA;AAAA,QACA,KAAK,IAAI,UAAU,UAAU;AAAA,MAC9B;AAEA,UAAI,cAAc;AAmBlB,YAAM,eAAe,MAAM;AAAA,QAC1B;AAAA,QACA;AAAA,MACD;AACA,YAAM,WAAW,aAAa,CAAC;AAM/B,UAAI,CAAC,SAAU,OAAM,IAAI,MAAM,mCAAmC;AAClE,YAAM,gBAAgB,MAAM;AAAA,QAC3B;AAAA,QACA;AAAA,MACD;AACA,YAAM,YAAY,cAAc,CAAC;AAGjC,UAAI,CAAC,WAAW;AACf,cAAM,IAAI,MAAM,yCAAyC;AAAA,MAC1D;AACA,YAAM,aAAa,YAAY,IAAI,IAAI;AAEvC,YAAM;AAAA,QACL;AAAA,QACA;AAAA,QACA,MAAM;AAAA,QACN,MAAM;AAAA,QACN;AAAA,QACA;AAAA,QACA,SAAS;AAAA,QACT,SAAS,gBAAgB;AAAA,QACzB;AAAA,QACA;AAAA,MACD;AAEA,YAAM,mBAAmB,YAAY,IAAI;AACzC,eAAS,iBAAiB,OAAO;AACjC,YAAM,UAAU,MAAM,aAAa,EAAE,EAAE;AACvC,eAAS,iBAAiB,OAAO;AAAA,QAChC,YAAY,YAAY,IAAI,IAAI;AAAA,QAChC,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ,KAAK,eAAe;AAAA,QACzC,kBAAkB,QAAQ,KAAK,oBAAoB;AAAA,MACpD,CAAC;AACD,eAAS,aAAa,OAAO;AAAA,QAC5B;AAAA,QACA,YAAY,SAAS;AAAA,QACrB,aAAa,SAAS,gBAAgB;AAAA,QACtC,WAAW,QAAQ;AAAA,MACpB,CAAC;AAED,aAAO;AAAA,QACN,MAAM,MAAM;AAAA,QACZ,OAAO,MAAM;AAAA,QACb;AAAA,QACA;AAAA,QACA,YAAY,SAAS;AAAA,QACrB,aAAa,SAAS,gBAAgB;AAAA,QACtC,aAAa,KAAK;AAAA,QAClB,aAAa,UAAU;AAAA,QACvB,gBAAgB,UAAU;AAAA,QAC1B;AAAA,QACA;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;AC9ZD;AAAA,EACC,SAAAG;AAAA,OAGM;AACP,SAAS,MAAAC,YAAU;AAEnB,IAAM,gCAAgC;AACtC,IAAM,4BAA4B;AAiDlC,IAAM,wBAAwB,oBAAI,IAAqC;AAEvE,SAASC,OAAM,IAA2B;AACzC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACxD;AAEA,SAASC,iBAAgB,OAAgB,MAAc;AACtD,MAAI,CAAC,OAAO,UAAU,KAAK,KAAM,QAAmB,GAAG;AACtD,UAAM,IAAI,MAAM,GAAG,IAAI,6BAA6B;AAAA,EACrD;AAEA,SAAO;AACR;AAEA,SAAS,YAAY,OAAgB,MAAc;AAClD,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACpD,UAAM,IAAI,MAAM,GAAG,IAAI,6BAA6B;AAAA,EACrD;AAEA,SAAO;AACR;AAEA,SAASC,WAAa,MAAsB;AAC3C,SAAO;AACR;AAEA,SAAS,cAAc,MAAc,UAA0B;AAC9D,QAAM,QAAQ,QAAQ,IAAI,IAAI;AAC9B,MAAI,UAAU,UAAa,UAAU,GAAI,QAAO;AAEhD,QAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,GAAG;AAC3C,UAAM,IAAI,MAAM,GAAG,IAAI,uCAAuC;AAAA,EAC/D;AAEA,SAAO;AACR;AAEA,SAAS,KAAK,WAA+BC,UAAkB;AAC9D,MAAI,UAAU,eAAe,EAAG;AAChC,YAAU,KAAK,KAAK,UAAUA,QAAO,CAAC;AACvC;AAEA,SAAS,aAAa,KAAoB,UAAmB;AAC5D,SAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS,IAAI;AAAA,IACb,MAAM,IAAI;AAAA,IACV,SAAS,IAAI;AAAA,IACb,cAAc,IAAI;AAAA,IAClB,WAAW,IAAI;AAAA,IACf,SAAS,KAAK,MAAM,IAAI,YAAY;AAAA,IACpC,WAAW,IAAI;AAAA,IACf;AAAA,EACD;AACD;AAEA,SAAS,kBAAkB,KAAoB;AAC9C,QAAM,UAAU,sBAAsB,IAAI,IAAI,QAAQ;AACtD,MAAI,CAAC,QAAS;AAEd,aAAW,UAAU,SAAS;AAC7B,SAAK,QAAQ,aAAa,KAAK,KAAK,CAAC;AAAA,EACtC;AACD;AAEA,SAAS,eAAe,SAAiB,WAA+B;AACvE,QAAM,UAAU,sBAAsB,IAAI,OAAO,KAAK,oBAAI,IAAI;AAC9D,UAAQ,IAAI,SAAS;AACrB,wBAAsB,IAAI,SAAS,OAAO;AAE1C,SAAO,MAAM;AACZ,YAAQ,OAAO,SAAS;AACxB,QAAI,QAAQ,SAAS,GAAG;AACvB,4BAAsB,OAAO,OAAO;AAAA,IACrC;AAAA,EACD;AACD;AAEA,eAAe,iBAAiB,GAAiB,OAAwB;AACxE,QAAM,MAAqB;AAAA,IAC1B,UAAU,OAAO,WAAW;AAAA,IAC5B,MAAM,MAAM;AAAA,IACZ,UAAU,EAAE;AAAA,IACZ,eAAe,MAAM,gBAAgB;AAAA,IACrC,YAAY,MAAM,aAAa;AAAA,IAC/B,cAAc,KAAK,UAAU,MAAM,WAAW,CAAC,CAAC;AAAA,IAChD,YAAY,MAAM,aAAa,KAAK,IAAI;AAAA,EACzC;AAEA,MAAI;AACH,UAAM,EAAE,GAAG;AAAA,MACV;AAAA,MACA,IAAI;AAAA,MACJ,IAAI;AAAA,MACJ,IAAI;AAAA,MACJ,IAAI;AAAA,MACJ,IAAI;AAAA,MACJ,IAAI;AAAA,MACJ,IAAI;AAAA,IACL;AACA,sBAAkB,GAAG;AAAA,EACtB,SAAS,OAAO;AACf,MAAE,IAAI,KAAK;AAAA,MACV,KAAK;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,KAAK,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC3D,CAAC;AAAA,EACF;AACD;AAEA,eAAe,kBACd,UACA,WACC;AACD,QAAM,OAAOD;AAAA,IACZ,MAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAStB;AAAA,EACF;AAEA,aAAW,OAAO,MAAM;AACvB,SAAK,WAAW,aAAa,KAAK,IAAI,CAAC;AAAA,EACxC;AACD;AAEA,SAAS,gBAAgB,MAAkB,iBAAyB;AACnE,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,UAAU,KAAK,IAAI,CAAC,QAAQ,IAAI,GAAG,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/D,aAAW,OAAO,QAAS,MAAK,IAAI,GAAG;AAEvC,QAAM,UAAoB,CAAC;AAC3B,WAAS,MAAM,GAAG,OAAO,iBAAiB,OAAO,GAAG;AACnD,QAAI,CAAC,KAAK,IAAI,GAAG,EAAG,SAAQ,KAAK,GAAG;AAAA,EACrC;AAEA,QAAM,aACL,KAAK,WAAW,mBAChB,QAAQ,WAAW,KACnB,QAAQ,MAAM,CAAC,KAAK,WAAW,QAAQ,SAAS,CAAC;AAElD,SAAO;AAAA,IACN;AAAA,IACA,OAAO,KAAK;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA,IAAI;AAAA,EACL;AACD;AAEA,SAAS,cAAc,MAAkB,kBAAqC;AAC7E,QAAM,oBAAoB,IAAI;AAAA,IAC7B,iBAAiB,IAAI,CAAC,YAAY,CAAC,QAAQ,WAAW,QAAQ,OAAO,CAAC;AAAA,EACvE;AACA,QAAM,gBAAgB,oBAAI,IAAwB;AAElD,aAAW,OAAO,MAAM;AACvB,UAAM,cAAc,cAAc,IAAI,IAAI,UAAU,KAAK,CAAC;AAC1D,gBAAY,KAAK,GAAG;AACpB,kBAAc,IAAI,IAAI,YAAY,WAAW;AAAA,EAC9C;AAEA,QAAM,WAAW,iBAAiB,IAAI,CAAC,YAAY;AAClD,UAAM,SAAS;AAAA,MACd,cAAc,IAAI,QAAQ,SAAS,KAAK,CAAC;AAAA,MACzC,QAAQ;AAAA,IACT;AACA,WAAO;AAAA,MACN,WAAW,QAAQ;AAAA,MACnB,GAAG;AAAA,IACJ;AAAA,EACD,CAAC;AAED,QAAM,uBAAuB,CAAC,GAAG,cAAc,KAAK,CAAC,EACnD,OAAO,CAAC,cAAc,CAAC,kBAAkB,IAAI,SAAS,CAAC,EACvD,KAAK;AACP,QAAM,oBAAoB,iBAAiB;AAAA,IAC1C,CAAC,OAAO,YAAY,QAAQ,QAAQ;AAAA,IACpC;AAAA,EACD;AACA,QAAM,KACL,qBAAqB,WAAW,KAChC,KAAK,WAAW,qBAChB,SAAS,MAAM,CAAC,YAAY,QAAQ,EAAE;AAEvC,SAAO;AAAA,IACN,MAAM;AAAA,IACN,kBAAkB,iBAAiB;AAAA,IACnC;AAAA,IACA,WAAW,KAAK;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACD;AAEO,IAAM,kBAAkBJ,QAAM;AAAA,EACpC,SAAS;AAAA,IACR,uBAAuB;AAAA,IACvB,kBAAkB;AAAA,EACnB;AAAA,EACA,IAAIC,KAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAOtB;AACD,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AACA,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAKtB;AACD,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUtB;AACD,YAAM,SAAS;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AAAA,EACD,MAAM,OAAO,GAAG;AACf,UAAM,iBAAiB,GAAG;AAAA,MACzB,MAAM;AAAA,MACN,SAAS;AAAA,QACR,KAAK,EAAE;AAAA,QACP,MAAM,EAAE;AAAA,MACT;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EACA,MAAM,QAAQ,GAAG;AAChB,UAAM,UAAU;AAAA,MACf;AAAA,MACA;AAAA,IACD;AACA,UAAM,iBAAiB,KAAK,IAAI;AAChC,UAAM,iBAAiB,GAAG;AAAA,MACzB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,SAAS;AAAA,QACR;AAAA,MACD;AAAA,IACD,CAAC;AACD,UAAM,EAAE,GAAG;AAAA,MACV;AAAA,MACA;AAAA,IACD;AACA,MAAE,IAAI,KAAK;AAAA,MACV,KAAK;AAAA,MACL;AAAA,MACA;AAAA,IACD,CAAC;AACD,UAAMC,OAAM,OAAO;AACnB,UAAM,iBAAiB,GAAG;AAAA,MACzB,MAAM;AAAA,MACN,SAAS;AAAA,QACR;AAAA,QACA;AAAA,QACA,WAAW,KAAK,IAAI,IAAI;AAAA,MACzB;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EACA,MAAM,UAAU,GAAG,SAAS;AAC3B,UAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,QAAI,IAAI,aAAa,aAAa,IAAI,aAAa,mBAAmB;AACrE,YAAM,CAAC,UAAU,IAAIE;AAAA,QACpB,MAAM,EAAE,GAAG;AAAA,UACV;AAAA,QACD;AAAA,MACD;AACA,aAAO,IAAI,SAAS,KAAK,UAAU;AAAA,QAClC,MAAM;AAAA,QACN,WAAW;AAAA,QACX,cAAc,eAAe;AAAA,QAC7B,gBAAgB,YAAY,oBAAoB;AAAA,QAChD,WAAW,KAAK,IAAI;AAAA,MACrB,CAAC,GAAG;AAAA,QACH,SAAS;AAAA,UACR,gBAAgB;AAAA,QACjB;AAAA,MACD,CAAC;AAAA,IACF;AAEA,WAAO,IAAI,SAAS,aAAa,EAAE,QAAQ,IAAI,CAAC;AAAA,EACjD;AAAA,EACA,YAAY,GAAG,WAA+B;AAC7C,UAAM,eAAe,OAAO,WAAW;AACvC,QAAI;AACJ,UAAM,oBAAoB,eAAe,EAAE,SAAS,SAAS;AAE7D,SAAK,WAAW;AAAA,MACf,MAAM;AAAA,MACN;AAAA,MACA,WAAW,KAAK,IAAI;AAAA,IACrB,CAAC;AACD,UAAM,YAAY;AACjB,UAAI;AACH,cAAM,kBAAkB,EAAE,IAAI,SAAS;AAAA,MACxC,SAAS,OAAO;AACf,UAAE,IAAI,KAAK;AAAA,UACV,KAAK;AAAA,UACL,KAAK,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC3D,CAAC;AAAA,MACF;AACA,YAAM,iBAAiB,GAAG;AAAA,QACzB,MAAM;AAAA,QACN;AAAA,MACD,CAAC;AAAA,IACF,GAAG;AAEH,UAAM,SAAS,OAAO,WAAmB,oBAA4B;AACpE,YAAM,OAAOA;AAAA,QACZ,MAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,aAAO;AAAA,QACN,MAAM;AAAA,QACN;AAAA,QACA,GAAG,gBAAgB,MAAM,eAAe;AAAA,MACzC;AAAA,IACD;AAEA,UAAM,cAAc,YAAY;AAC/B,YAAM,CAAC,UAAU,IAAIA;AAAA,QACpB,MAAM,EAAE,GAAG;AAAA,UACV;AAAA,QACD;AAAA,MACD;AACA,aAAO;AAAA,QACN,cAAc,eAAe;AAAA,QAC7B,gBAAgB,YAAY,oBAAoB;AAAA,MACjD;AAAA,IACD;AAEA,UAAM,eAAe,OAAO,WAAmB,YAAoB;AAClE,WAAK,WAAW;AAAA,QACf,MAAM;AAAA,QACN;AAAA,QACA;AAAA,QACA,WAAW,KAAK,IAAI;AAAA,MACrB,CAAC;AAED,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,MACD;AAEA,eAAS,MAAM,GAAG,OAAO,SAAS,OAAO,GAAG;AAC3C,cAAMF,OAAM,GAAK;AACjB,cAAM,YAAY,KAAK,IAAI;AAC3B,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACD;AACA,aAAK,WAAW;AAAA,UACf,MAAM;AAAA,UACN;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,eAAe,MAAM,OAAO,WAAW,OAAO;AACpD,WAAK,WAAW;AAAA,QACf,MAAM;AAAA,QACN;AAAA,QACA;AAAA,QACA,WAAW,KAAK,IAAI;AAAA,QACpB;AAAA,MACD,CAAC;AAAA,IACF;AAEA,cAAU,iBAAiB,WAAW,OAAOI,YAA6B;AACzE,UAAI;AACH,YAAI,OAAOA,QAAM,SAAS,UAAU;AACnC,gBAAM,IAAI,MAAM,oCAAoC;AAAA,QACrD;AAEA,cAAM,UAAU,KAAK,MAAMA,QAAM,IAAI;AACrC,cAAM,OAAO,YAAY,QAAQ,MAAM,MAAM;AAE7C,YAAI,SAAS,WAAW;AACvB,gBAAM,OAAOF;AAAA,YACZ,MAAM,EAAE,GAAG;AAAA,cACV;AAAA,YACD;AAAA,UACD;AACA,gBAAM,CAAC,KAAK,IAAIA;AAAA,YACf,MAAM,EAAE,GAAG;AAAA,cACV;AAAA,YACD;AAAA,UACD;AACA,eAAK,WAAW;AAAA,YACf,MAAM;AAAA,YACN,WAAW,OAAO,SAAS,KAAK;AAAA,YAChC,SAAS;AAAA,YACT,WAAW,KAAK,IAAI;AAAA,UACrB,CAAC;AACD;AAAA,QACD;AAEA,YAAI,SAAS,QAAQ;AACpB,eAAK,WAAW;AAAA,YACf,MAAM;AAAA,YACN,SAAS,YAAY,QAAQ,SAAS,SAAS;AAAA,YAC/C,GAAI,MAAM,YAAY;AAAA,YACtB,WAAW,KAAK,IAAI;AAAA,UACrB,CAAC;AACD;AAAA,QACD;AAEA,YAAI,SAAS,UAAU;AACtB,gBAAM,YAAY,YAAY,QAAQ,WAAW,WAAW;AAC5D,gBAAM,kBAAkBD;AAAA,YACvB,QAAQ;AAAA,YACR;AAAA,UACD;AACA,eAAK,WAAW,MAAM,OAAO,WAAW,eAAe,CAAC;AACxD;AAAA,QACD;AAEA,YAAI,SAAS,SAAS;AACrB,gBAAM,YAAY,YAAY,QAAQ,WAAW,WAAW;AAC5D,gBAAM,UAAUA,iBAAgB,QAAQ,SAAS,SAAS;AAC1D,gBAAM,iBAAiB,GAAG;AAAA,YACzB,MAAM;AAAA,YACN;AAAA,YACA;AAAA,YACA,SAAS;AAAA,cACR;AAAA,YACD;AAAA,UACD,CAAC;AACD,gBAAM,oBAAoB;AAC1B,gBAAM,aAAa,YAAY;AAC9B,kBAAM,mBAAmB,MAAM,MAAM,MAAS;AAC9C,kBAAM,aAAa,WAAW,OAAO;AAAA,UACtC,GAAG;AACH,4BAAkB;AAClB,gBAAM,EAAE,UAAU,SAAS;AAC3B,cAAI,oBAAoB,WAAW;AAClC,8BAAkB;AAAA,UACnB;AACA;AAAA,QACD;AAEA,cAAM,IAAI,MAAM,yBAAyB,IAAI,EAAE;AAAA,MAChD,SAAS,OAAO;AACf,aAAK,WAAW;AAAA,UACf,MAAM;AAAA,UACN,SACC,iBAAiB,QACd,MAAM,UACN;AAAA,UACJ,WAAW,KAAK,IAAI;AAAA,QACrB,CAAC;AAAA,MACF;AAAA,IACD,CAAC;AAED,cAAU,iBAAiB,SAAS,YAAY;AAC/C,wBAAkB;AAClB,YAAM,iBAAiB,GAAG;AAAA,QACzB,MAAM;AAAA,QACN;AAAA,MACD,CAAC;AAAA,IACF,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,QAAQ,OAAO,GAAG,WAAmB,oBAA4B;AAChE,YAAM,OAAOC;AAAA,QACZ,MAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,QACD;AAAA,MACD;AACA,aAAO;AAAA,QACN;AAAA,QACA;AAAA,QACA,OAAO,KAAK;AAAA,QACZ,SAAS,KAAK,IAAI,CAAC,QAAQ,IAAI,GAAG;AAAA,MACnC;AAAA,IACD;AAAA,IACA,WAAW,OAAO,GAAG,qBAAwC;AAC5D,UAAI,CAAC,MAAM,QAAQ,gBAAgB,GAAG;AACrC,cAAM,IAAI,MAAM,mCAAmC;AAAA,MACpD;AAEA,iBAAW,WAAW,kBAAkB;AACvC,oBAAY,QAAQ,WAAW,WAAW;AAC1C,QAAAD,iBAAgB,QAAQ,SAAS,SAAS;AAAA,MAC3C;AAEA,YAAM,OAAOC;AAAA,QACZ,MAAM,EAAE,GAAG;AAAA,UACV;AAAA,QACD;AAAA,MACD;AACA,aAAO,cAAc,MAAM,gBAAgB;AAAA,IAC5C;AAAA,EACD;AACD,CAAC;;;ACzkBD,SAAS,SAAAG,eAA8D;AAKhE,IAAM,iBAAiBA,QAAM;AAAA,EACnC,SAAS;AAAA,IACR,uBAAuB;AAAA,EACxB;AAAA,EACA,OAAO;AAAA,IACN,iBAAiB;AAAA,IACjB,cAAc;AAAA,EACf;AAAA,EACA,YAAY,GAAG,WAA+B;AAC7C,MAAE,MAAM,mBAAmB;AAC3B,UAAM,eAAe,OAAO,WAAW;AAEvC,cAAU;AAAA,MACT,KAAK,UAAU;AAAA,QACd,MAAM;AAAA,QACN;AAAA,QACA,iBAAiB,EAAE,MAAM;AAAA,MAC1B,CAAC;AAAA,IACF;AAEA,UAAM,WAAW,YAAY,MAAM;AAClC,UAAI,UAAU,eAAe,EAAG;AAChC,gBAAU;AAAA,QACT,KAAK,UAAU;AAAA,UACd,MAAM;AAAA,UACN;AAAA,UACA,WAAW,KAAK,IAAI;AAAA,QACrB,CAAC;AAAA,MACF;AAAA,IACD,GAAG,GAAG;AAEN,cAAU,iBAAiB,WAAW,CAACC,YAA6B;AACnE,QAAE,MAAM,gBAAgB;AACxB,gBAAU;AAAA,QACT,KAAK,UAAU;AAAA,UACd,MAAM;AAAA,UACN;AAAA,UACA,UAAUA,QAAM;AAAA,QACjB,CAAC;AAAA,MACF;AAAA,IACD,CAAC;AAED,cAAU,iBAAiB,SAAS,MAAM;AACzC,oBAAc,QAAQ;AACtB,QAAE,MAAM,mBAAmB;AAAA,IAC5B,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,SAAS,GAAG;AACX,aAAO;AAAA,QACN,iBAAiB,EAAE,MAAM;AAAA,QACzB,cAAc,EAAE,MAAM;AAAA,MACvB;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;AC5DD,SAAS,SAAAC,eAA8D;AACvE,SAAS,MAAAC,YAAU;AAEnB,IAAM,4BAA4B;AAClC,IAAM,sBAAsB;AAE5B,SAASC,MAAK,WAA+BC,UAAwB;AACpE,MAAI,UAAU,eAAe,EAAG;AAChC,YAAU,KAAK,KAAK,UAAUA,QAAO,CAAC;AACvC;AAEA,SAAS,oBACR,OACA,MACA,UACS;AACT,MAAI,UAAU,UAAa,UAAU,KAAM,QAAO;AAClD,QAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,CAAC,OAAO,SAAS,MAAM,KAAK,UAAU,GAAG;AAC5C,UAAM,IAAI,MAAM,GAAG,IAAI,4BAA4B;AAAA,EACpD;AACA,SAAO;AACR;AAEA,SAASC,OAAM,IAAY,QAAoC;AAC9D,MAAI,OAAO,QAAS,QAAO,QAAQ,QAAQ;AAC3C,SAAO,IAAI,QAAQ,CAAC,YAAY;AAC/B,UAAM,UAAU,WAAW,SAAS,EAAE;AACtC,WAAO;AAAA,MACN;AAAA,MACA,MAAM;AACL,qBAAa,OAAO;AACpB,gBAAQ;AAAA,MACT;AAAA,MACA,EAAE,MAAM,KAAK;AAAA,IACd;AAAA,EACD,CAAC;AACF;AAEO,IAAM,gBAAgBJ,QAAM;AAAA,EAClC,SAAS;AAAA,IACR,uBAAuB;AAAA,IACvB,kBAAkB;AAAA,EACnB;AAAA,EACA,IAAIC,KAAG;AAAA,IACN,WAAW,OAAOA,SAAO;AACxB,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAShB;AACD,YAAMA,KAAG,QAAQ;AAAA;AAAA;AAAA,IAGhB;AAAA,IACF;AAAA,EACD,CAAC;AAAA,EACD,OAAO;AAAA,IACN,iBAAiB;AAAA,IACjB,gBAAgB;AAAA,IAChB,YAAY;AAAA,EACb;AAAA,EACA,YAAY,GAAG,WAA+B;AAC7C,MAAE,MAAM,mBAAmB;AAC3B,UAAM,eAAe,OAAO,WAAW;AAEvC,IAAAC,MAAK,WAAW;AAAA,MACf,MAAM;AAAA,MACN;AAAA,MACA,iBAAiB,EAAE,MAAM;AAAA,MACzB,WAAW,KAAK,IAAI;AAAA,IACrB,CAAC;AAED,cAAU,iBAAiB,WAAW,OAAOG,YAA6B;AACzE,UAAI;AACH,cAAM,UACL,OAAOA,QAAM,SAAS,WACnB,KAAK,MAAMA,QAAM,IAAI,IACrB;AAKJ,YAAI,WAAW,QAAQ,SAAS,QAAQ;AACvC,UAAAH,MAAK,WAAW;AAAA,YACf,MAAM;AAAA,YACN;AAAA,YACA,IAAI,QAAQ;AAAA,YACZ,WAAW,KAAK,IAAI;AAAA,UACrB,CAAC;AACD;AAAA,QACD;AAEA,YAAI,CAAC,WAAW,QAAQ,SAAS,aAAa;AAC7C,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC7C;AAEA,cAAM,YACL,OAAO,QAAQ,cAAc,YAAY,QAAQ,YAC9C,QAAQ,YACR,OAAO,WAAW;AACtB,cAAM,kBAAkB;AAAA,UACvB,QAAQ;AAAA,UACR;AAAA,UACA;AAAA,QACD;AACA,cAAM,aAAa;AAAA,UAClB,QAAQ;AAAA,UACR;AAAA,UACA;AAAA,QACD;AACA,cAAM,aAAa,MAAQ;AAC3B,cAAM,eAAe,KAAK;AAAA,UACzB;AAAA,UACA,KAAK,MAAO,aAAa,MAAS,eAAe;AAAA,QAClD;AAEA,cAAM,aAAa,YAAY;AAC9B,YAAE,MAAM,kBAAkB;AAC1B,UAAAA,MAAK,WAAW;AAAA,YACf,MAAM;AAAA,YACN;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,WAAW,KAAK,IAAI;AAAA,UACrB,CAAC;AAED,gBAAM,YAAY,YAAY,IAAI;AAClC,mBAAS,IAAI,GAAG,IAAI,cAAc,KAAK;AACtC,gBAAI,EAAE,YAAY,WAAW,UAAU,eAAe,GAAG;AACxD;AAAA,YACD;AAEA,kBAAM,aAAa,IAAI;AACvB,kBAAM,QAAQ,SAAS,UAAU;AACjC,kBAAM,YAAY,KAAK,IAAI;AAC3B,kBAAM,EAAE,GAAG;AAAA,cACV;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACD;AACA,cAAE,MAAM,cAAc;AAEtB,YAAAA,MAAK,WAAW;AAAA,cACf,MAAM;AAAA,cACN;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW;AAAA,YACZ,CAAC;AAED,kBAAM,SAAS,YAAY,aAAa;AACxC,kBAAM,UAAU,KAAK,IAAI,GAAG,SAAS,YAAY,IAAI,CAAC;AACtD,gBAAI,UAAU,GAAG;AAChB,oBAAME,OAAM,SAAS,EAAE,WAAW;AAAA,YACnC;AAAA,UACD;AAEA,UAAAF,MAAK,WAAW;AAAA,YACf,MAAM;AAAA,YACN;AAAA,YACA;AAAA,YACA,YAAY;AAAA,YACZ,WAAW,KAAK,IAAI;AAAA,UACrB,CAAC;AAAA,QACF,GAAG;AAEH,cAAM,EAAE,UAAU,SAAS;AAAA,MAC5B,SAAS,OAAO;AACf,QAAAA,MAAK,WAAW;AAAA,UACf,MAAM;AAAA,UACN,SACC,iBAAiB,QACd,MAAM,UACN;AAAA,UACJ,WAAW,KAAK,IAAI;AAAA,QACrB,CAAC;AAAA,MACF;AAAA,IACD,CAAC;AAED,cAAU,iBAAiB,SAAS,MAAM;AACzC,QAAE,MAAM,mBAAmB;AAAA,IAC5B,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,SAAS,GAAG;AACX,aAAO;AAAA,QACN,iBAAiB,EAAE,MAAM;AAAA,QACzB,gBAAgB,EAAE,MAAM;AAAA,QACxB,YAAY,EAAE,MAAM;AAAA,MACrB;AAAA,IACD;AAAA,EACD;AACD,CAAC;;;AC5MD,SAAS,SAAAI,eAA8D;AACvE,SAAS,MAAAC,YAAU;AAuGnB,IAAM,aAAN,MAAiB;AAAA,EACR,SAAS;AAAA,EACT,UAA6B,CAAC;AAAA,EAEtC,MAAM,UAAyB;AAC9B,QAAI,CAAC,KAAK,QAAQ;AACjB,WAAK,SAAS;AACd;AAAA,IACD;AACA,UAAM,IAAI,QAAc,CAAC,YAAY,KAAK,QAAQ,KAAK,OAAO,CAAC;AAC/D,SAAK,SAAS;AAAA,EACf;AAAA,EAEA,UAAgB;AACf,UAAM,OAAO,KAAK,QAAQ,MAAM;AAChC,QAAI,MAAM;AACT,WAAK;AACL;AAAA,IACD;AACA,SAAK,SAAS;AAAA,EACf;AACD;AAEA,SAAS,mBACR,SAIqB;AACrB,QAAM,QAAQ,IAAI,WAAW;AAC7B,MAAI,oBAA+C;AAEnD,QAAM,sBAAsB,MAA0B;AACrD,UAAM,KAAK,OAAO;AAAA,MACjB,CACC,UACG,WACC,QAAW,OAAO,GAAG,MAAM;AAAA,MAChC;AAAA,QACC,iBAAiB,OAChB,QACA,OACgB,GAAG,EAAE;AAAA,MACvB;AAAA,IACD;AACA,WAAO;AAAA,EACR;AAEA,QAAM,iBAAiB,OACtB,UACG,WACe;AAClB,UAAM,MAAM,QAAQ;AACpB,QAAI;AACH,aAAO,MAAM,QAAW,OAAO,GAAG,MAAM;AAAA,IACzC,UAAE;AACD,YAAM,QAAQ;AAAA,IACf;AAAA,EACD;AAEA,SAAO,OAAO,OAAO,gBAAgB;AAAA,IACpC,iBAAiB,OAChB,OACA,OACgB;AAChB,UAAI,mBAAmB;AACtB,eAAO,GAAG,iBAAiB;AAAA,MAC5B;AACA,YAAM,MAAM,QAAQ;AACpB,YAAM,KAAK,oBAAoB;AAC/B,UAAI;AACH,cAAM,oBAAoB,SAAS,OAAO,qBAAqB,OAAO;AACtE,4BAAoB;AACpB,YAAI;AACH,gBAAM,SAAS,MAAM,GAAG,EAAE;AAC1B,8BAAoB;AACpB,gBAAM,oBAAoB,SAAS,OAAO,sBAAsB,QAAQ;AACxE,iBAAO;AAAA,QACR,SAAS,OAAO;AACf,8BAAoB;AACpB,gBAAM;AAAA,YACL;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACD;AACA,gBAAM;AAAA,QACP;AAAA,MACD,UAAE;AACD,4BAAoB;AACpB,cAAM,QAAQ;AAAA,MACf;AAAA,IACD;AAAA,EACD,CAAC;AACF;AAEA,IAAM,gBAAgB;AACtB,IAAM,yBAAyB;AAC/B,IAAM,kBAAkB;AACxB,IAAM,sBAAsB;AAC5B,IAAM,qBAAqB;AAE3B,IAAM,wBAAwB;AAC9B,IAAM,6BAA6B;AACnC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AACnC,IAAM,gBAAgB;AAEtB,SAASC,MACR,WACA,SACO;AACP,MAAI,UAAU,eAAe,GAAG;AAC/B,cAAU,KAAK,KAAK,UAAU,OAAO,CAAC;AAAA,EACvC;AACD;AAEO,IAAM,iBAAiBF,QAAM;AAAA,EACnC,SAAS;AAAA,IACR,uBAAuB;AAAA,IACvB,kBAAkB;AAAA,EACnB;AAAA,EACA,OAAO;AAAA,IACN,UAAU;AAAA,IACV,WAAW;AAAA,IACX,YAAY,iCAAiC;AAAA,EAC9C;AAAA,EACA,IAAIC,KAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,6BAA6B,QAAQ;AAC3C,YAAM,yBAAyB,QAAQ;AAAA,IACxC;AAAA,EACD,CAAC;AAAA,EACD,MAAM;AAAA,IACL,KAAK;AAAA,IACL,WAAW;AAAA,IACX,eAAe;AAAA,IACf,eAAe;AAAA,EAChB;AAAA,EACA,aAAa,CAAC,GAAG,cAAkC;AAClD,IAAAC,MAAK,WAAW;AAAA,MACf,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,IACrB,CAAC;AAED,cAAU,iBAAiB,WAAW,CAACC,YAA6B;AACnE,YAAM,UAAU,8BAA8B,GAAG,WAAWA,QAAM,IAAI;AACtE,WAAK,EAAE,UAAU,OAAO;AAAA,IACzB,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,KAAK,OAAO,GAAG,aAAsB;AACpC,YAAM,UAAU,8BAA8B,CAAC;AAC/C,QAAE,MAAM;AACR,cAAQ,KAAK;AACb,YAAM,aAAa,iCAAiC;AACpD,YAAM,QAAQ;AAAA,QACb;AAAA,QACA,QAAQ;AAAA,QACR,EAAE,MAAM;AAAA,MACT;AACA,YAAM,SAAS,MAAM;AAAA,QACpB,QAAQ;AAAA,QACR,YAAY,iBAAiB,EAAE,MAAM,QAAQ;AAAA,QAC7C;AAAA,QACA;AAAA,MACD;AACA,aAAO;AAAA,QACN,GAAG;AAAA,QACH,OAAO,8BAA8B,GAAG,UAAU;AAAA,MACnD;AAAA,IACD;AAAA,IACA,aAAa,CAAC,MAAM,EAAE,MAAM;AAAA,IAC5B,OAAO,CAAC,MAAM;AACb,QAAE,MAAM;AACR,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;AAED,eAAe,8BACd,GAMA,WACA,MACgB;AAChB,MAAI,UAAuD;AAC3D,MAAI,aAAgD;AACpD,MAAI;AACH,UAAM,UAAU,6BAA6B,IAAI;AACjD,cAAU,QAAQ;AAElB,QAAI,QAAQ,SAAS,QAAQ;AAC5B,MAAAD,MAAK,WAAW;AAAA,QACf,MAAM;AAAA,QACN,IAAI,QAAQ;AAAA,QACZ,WAAW,KAAK,IAAI;AAAA,MACrB,CAAC;AACD;AAAA,IACD;AAEA,QAAI,QAAQ,SAAS,eAAe;AACnC,MAAAA,MAAK,WAAW,EAAE,MAAM,YAAY,WAAW,KAAK,IAAI,EAAE,CAAC;AAC3D,QAAE,MAAM;AACR;AAAA,IACD;AAEA,UAAM,UAAU,8BAA8B,CAAC;AAC/C,MAAE,MAAM;AACR,YAAQ,KAAK;AACb,iBAAa,iCAAiC;AAC9C,UAAM,QAAQ;AAAA,MACb;AAAA,MACA,QAAQ;AAAA,MACR,EAAE,MAAM;AAAA,IACT;AAEA,QAAI,QAAQ,SAAS,iBAAiB;AACrC,YAAM,YAAY,YAAY,IAAI;AAClC,YAAME,UAAS,MAAM;AAAA,QACpB,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR;AAAA,MACD;AACA,MAAAF,MAAK,WAAW;AAAA,QACf,MAAM;AAAA,QACN,SAAS,QAAQ;AAAA,QACjB,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,QACjD,SAAS,CAACE,OAAM;AAAA,QAChB,OAAO,8BAA8B,GAAG,UAAU;AAAA,MACnD,CAAC;AACD;AAAA,IACD;AAEA,UAAM,SAAS,MAAM;AAAA,MACpB,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,QAAQ,mBAAmB;AAAA,MAC3B;AAAA,IACD;AACA,IAAAF,MAAK,WAAW;AAAA,MACf,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,MACjB,GAAG;AAAA,MACH,OAAO,8BAA8B,GAAG,UAAU;AAAA,IACnD,CAAC;AAAA,EACF,SAAS,OAAO;AACf,IAAAA,MAAK,WAAW;AAAA,MACf,MAAM;AAAA,MACN;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC5D,GAAI,aAAa,EAAE,OAAO,8BAA8B,GAAG,UAAU,EAAE,IAAI,CAAC;AAAA,IAC7E,CAAC;AAAA,EACF;AACD;AAEA,SAAS,6BAA6B,MAAwC;AAC7E,MAAI,OAAO,SAAS,UAAU;AAC7B,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC9D;AACA,QAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AAC1C,UAAM,IAAI,MAAM,8CAA8C;AAAA,EAC/D;AACA,QAAM,UAAU;AAEhB,MAAI,QAAQ,SAAS,QAAQ;AAC5B,WAAO;AAAA,MACN,MAAM;AAAA,MACN,GAAI,OAAO,QAAQ,OAAO,WAAW,EAAE,IAAI,QAAQ,GAAG,IAAI,CAAC;AAAA,IAC5D;AAAA,EACD;AACA,MAAI,QAAQ,SAAS,eAAe;AACnC,WAAO,EAAE,MAAM,cAAc;AAAA,EAC9B;AACA,MAAI,QAAQ,SAAS,iBAAiB;AACrC,WAAO,EAAE,MAAM,iBAAiB,SAAS,YAAY,SAAS,SAAS,EAAE;AAAA,EAC1E;AACA,MAAI,QAAQ,SAAS,kBAAkB;AACtC,WAAO;AAAA,MACN,MAAM;AAAA,MACN,UAAU,YAAY,SAAS,UAAU;AAAA,MACzC,GAAI,OAAO,QAAQ,oBAAoB,WACpC,EAAE,iBAAiB,QAAQ,gBAAgB,IAC3C,CAAC;AAAA,IACL;AAAA,EACD;AACA,QAAM,IAAI,MAAM,4CAA4C,OAAO,QAAQ,IAAI,CAAC,EAAE;AACnF;AAEA,SAAS,YAAY,QAAiC,OAAuB;AAC5E,QAAM,QAAQ,OAAO,KAAK;AAC1B,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACpD,UAAM,IAAI,MAAM,8BAA8B,KAAK,mBAAmB;AAAA,EACvE;AACA,SAAO;AACR;AAEA,SAAS,YAAY,QAAiC,OAAuB;AAC5E,QAAM,QAAQ,OAAO,KAAK;AAC1B,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;AACzD,UAAM,IAAI,MAAM,8BAA8B,KAAK,0BAA0B;AAAA,EAC9E;AACA,SAAO;AACR;AAEA,SAAS,yBAAyBD,MAAoC;AACrE,SAAO,mBAAmB,OACzB,UACG,WACe;AAClB,UAAM,YAAY,OAAO;AAAA,MAAI,CAAC,UAC7B,OAAO,UAAU,YAAa,QAAQ,IAAI,IAAK;AAAA,IAChD;AACA,WAAQ,MAAMA,KAAG,QAAQ,OAAO,GAAG,SAAS;AAAA,EAC7C,CAAC;AACF;AAEA,SAAS,8BAA8B,GAIX;AAC3B,IAAE,KAAK,QAAQ,yBAAyB,EAAE,EAAE;AAC5C,IAAE,MAAM,eAAe,iCAAiC;AACxD,IAAE,MAAM,cAAc;AACtB,MAAI,CAAC,EAAE,KAAK,WAAW;AACtB,MAAE,KAAK,YAAY,iCAAiC;AACpD,MAAE,KAAK,gBAAgB,KAAK,IAAI;AAChC,MAAE,KAAK,gBAAgB;AACvB,MAAE,MAAM;AAAA,EACT;AACA,SAAO;AAAA,IACN,KAAK,EAAE,KAAK;AAAA,IACZ,WAAW,EAAE,KAAK;AAAA,IAClB,MAAM,EAAE;AAAA,EACT;AACD;AAEA,SAAS,mCAA+D;AACvE,SAAO;AAAA,IACN,OAAO;AAAA,IACP,OAAO;AAAA,IACP,WAAW;AAAA,IACX,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,OAAO;AAAA,IACP,SAAS;AAAA,IACT,aAAa,CAAC;AAAA,IACd,SAAS,CAAC;AAAA,EACX;AACD;AAEA,SAAS,+BACR,OACA,MACAD,SACgC;AAChC,SAAO,EAAE,OAAO,MAAM,OAAAA,QAAM;AAC7B;AAEA,SAAS,8BACR,GACA,OACgC;AAChC,SAAO;AAAA,IACN,WAAW,EAAE,MAAM;AAAA,IACnB,gBAAgB,EAAE,MAAM;AAAA,IACxB,eAAe,EAAE,KAAK;AAAA,IACtB,OAAO,gCAAgC,KAAK;AAAA,IAC5C,MAAM;AAAA,MACL,EAAE,KAAK,aAAa,iCAAiC;AAAA,IACtD;AAAA,IACA,OAAO,gCAAgC,EAAE,MAAM,UAAU;AAAA,EAC1D;AACD;AAEA,SAAS,gCACR,OAC6B;AAC7B,SAAO;AAAA,IACN,OAAO,MAAM;AAAA,IACb,OAAO,MAAM;AAAA,IACb,WAAW,MAAM;AAAA,IACjB,IAAI,MAAM;AAAA,IACV,OAAO,MAAM;AAAA,IACb,MAAM,MAAM;AAAA,IACZ,QAAQ,MAAM;AAAA,IACd,MAAM,MAAM;AAAA,IACZ,OAAO,MAAM;AAAA,IACb,SAAS,MAAM;AAAA,IACf,aAAa,EAAE,GAAG,MAAM,YAAY;AAAA,IACpC,SAAS,EAAE,GAAG,MAAM,QAAQ;AAAA,EAC7B;AACD;AAEA,eAAe,4BACd,KACA,UACA,iBACA,OAC6E;AAC7E,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,uBAAuB,wBAAwB,KAAK,KAAK;AAC/D,QAAM,kBAAkB,mBAAmB,KAAK,GAAG,KAAK;AACxD,QAAM,mBAAmB,oBAAoB,KAAK,KAAK;AACvD,QAAM,cAAc,eAAe,KAAK,UAAU,KAAK;AACvD,QAAM,wBAAwBK,OAAM,eAAe,EAAE;AAAA,IAAK,MACzD,uBAAuB,KAAK,UAAU,KAAK;AAAA,EAC5C;AAEA,QAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,IACjC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,CAAC;AACD,SAAO;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAe,uBACd,KACA,UACA,OAC0C;AAC1C,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAAgC,CAAC;AACvC,QAAM,UAAU,MAAM,IAAI,gBAAgB,OAAO,OAAO,OAAO;AAC9D,UAAM,iBAAiB,MAAM;AAAA,MAC5B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,mBAAmB;AAAA,MACxB,eAAe,CAAC,GAAG,eAAe;AAAA,IACnC;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,eAAe,MAAM;AAAA,MAC1B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,QAAI,CAAC,aAAa,CAAC,GAAG,OAAO;AAC5B,YAAM;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,SACA,oBAAI,KAAK,GAAE,YAAY;AAAA,MACxB;AAAA,IACD;AACA,UAAM,gBAAgB,MAAM;AAAA,MAC3B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,QAAI,iBAAiB,cAAc,CAAC,GAAG,KAAK,GAAG;AAC9C,YAAM;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,KAAK,UAAU,EAAE,MAAM,MAAM,eAAe,KAAK,CAAC;AAAA,SAClD,oBAAI,KAAK,GAAE,YAAY;AAAA,MACxB;AAAA,IACD;AACA,UAAM,UAAU,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,MAAM,OAAO,QAAQ,CAAC,GAAG,OAAO,CAAC;AACvC,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,UAAU,EAAE,MAAM,oBAAoB,SAAS,CAAC;AAAA,OACrD,oBAAI,KAAK,GAAE,YAAY;AAAA,IACxB;AACA,WAAO;AAAA,EACR,CAAC;AACD,QAAM,KAAK;AAAA,IACV,MAAM;AAAA,IACN,YAAY,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACpD,UAAU;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAe,wBACd,KACA,OAC0C;AAC1C,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAAgC,CAAC;AACvC,QAAM,iBAAiB,MAAM;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM,mBAAmB,OAAO,eAAe,CAAC,GAAG,eAAe,eAAe;AACjF,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM,iBAAiB,MAAM;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuBD;AACA,QAAM,sBAAsB,eAAe,CAAC,GAAG;AAC/C,MAAI,OAAO,wBAAwB,UAAU;AAC5C,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAQA;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,EACD;AACA,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA;AAAA,EAID;AACA,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA;AAAA,EAID;AACA,SAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAe,mBACd,KACA,SACA,OAC0C;AAC1C,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAAgC,CAAC;AACvC,QAAM,QAAQ,IAAI;AAAA,IACjB;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,EACD,CAAC;AACD,QAAM,KAAK,CAAC,GAAG,MAAM,EAAE,aAAa,EAAE,UAAU;AAChD,SAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAe,oBACd,KACA,OAC0C;AAC1C,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAAgC,CAAC;AACvC,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA;AAAA,EAID;AACA,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA,EAGD;AACA,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA,EAGD;AACA,SAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAe,eACd,KACA,UACA,OAC0C;AAC1C,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAAgC,CAAC;AACvC,QAAM,aAAa,MAAM,IAAI,gBAAgB,OAAO,OAAO,OAAO;AACjE,UAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AACnC,UAAM,SAAS,OAAO,QAAQ;AAC9B,UAAM,UAAU,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,MAAM,OAAO,QAAQ,CAAC,GAAG,OAAO,CAAC;AACvC,UAAM,kBAAkB,MAAM;AAAA,MAC7B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,iBAAiB,MAAM;AAAA,MAC5B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA;AAAA;AAAA,IAGD;AAEA,UAAM,iBAAiB,kBAAkB,MAAM,IAAI,GAAG;AACtD,UAAM,iBAAiB,eAAe,MAAM,IAAI,GAAG;AACnD,UAAM,kBAAkB,eAAe,MAAM,IAAI,GAAG;AACpD,UAAM,mBAAmB,OAAO,eAAe,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC;AACrE,UAAM,gBAAgB,OAAO,gBAAgB,CAAC,GAAG,cAAc,GAAG;AAElE,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,UAAU,EAAE,QAAQ,WAAW,UAAU,cAAc,CAAC;AAAA,MAC7D;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,UAAU,EAAE,MAAM,iBAAiB,WAAW,eAAe,CAAC;AAAA,MACnE;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA;AAAA,MAEA;AAAA,MACA;AAAA,MACA,KAAK,UAAU,EAAE,UAAU,IAAI,CAAC;AAAA,MAChC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA;AAAA,MAEA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA;AAAA,MAEA;AAAA,MACA,YAAY,eAAe;AAAA,MAC3B;AAAA,MACA,KAAK,UAAU,EAAE,MAAM,QAAQ,eAAe,GAAG,CAAC;AAAA,MAClD;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,UAAU,EAAE,KAAK,KAAK,SAAS,CAAC;AAAA,MACrC;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,UAAU,EAAE,KAAK,MAAM,UAAU,WAAW,IAAI,CAAC;AAAA,MACtD;AAAA,IACD;AACA,UAAM;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,UAAU,EAAE,KAAK,MAAM,UAAU,WAAW,IAAI,CAAC;AAAA,MACtD;AAAA,IACD;AAEA,WAAO;AAAA,EACR,CAAC;AACD,QAAM,KAAK;AAAA,IACV,MAAM;AAAA,IACN,YAAY,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACpD,UAAU;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAe,WACd,KACA,OACA,OACA,MACA,UACG,QACY;AACf,QAAM,YAAY,YAAY,IAAI;AAClC,MAAI;AACH,UAAM,OAAO,MAAM,IAAO,OAAO,GAAG,MAAM;AAC1C,UAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAC3D,gCAA4B,OAAO,MAAM,OAAO,YAAY,KAAK,QAAQ,KAAK;AAC9E,UAAM,KAAK;AAAA,MACV;AAAA,MACA;AAAA,MACA,UAAU,KAAK;AAAA,IAChB,CAAC;AACD,WAAO;AAAA,EACR,SAAS,OAAO;AACf,UAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAC3D,gCAA4B,OAAO,MAAM,OAAO,YAAY,GAAG,IAAI;AACnE,UAAM;AAAA,EACP;AACD;AAEA,eAAe,oBACd,SAIA,OACA,MACA,UACG,QACY;AACf,QAAM,YAAY,YAAY,IAAI;AAClC,MAAI;AACH,UAAM,OAAO,MAAM,QAAW,OAAO,GAAG,MAAM;AAC9C;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,MACxC,KAAK;AAAA,MACL;AAAA,IACD;AACA,WAAO;AAAA,EACR,SAAS,OAAO;AACf;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,MACxC;AAAA,MACA;AAAA,IACD;AACA,UAAM;AAAA,EACP;AACD;AAEA,SAAS,4BACR,OACA,MACA,OACA,YACA,UACA,QACO;AACP,QAAM,iBAAiB,8BAA8B,KAAK;AAC1D,aAAW,UAAU,CAAC,MAAM,OAAO,MAAM,MAAM,MAAM,KAAK,GAAG;AAC5D,WAAO;AACP,WAAO,QAAQ;AACf,QAAI,OAAQ,QAAO;AACnB,QAAI,cAAc,cAAe,QAAO;AACxC,QAAI,aAAa,OAAO,OAAO;AAC9B,aAAO,QAAQ;AACf,aAAO,UAAU,GAAG,IAAI,IAAI,eAAe,KAAK;AAAA,IACjD;AACA,WAAO,YAAY,eAAe,SAAS,KACzC,OAAO,YAAY,eAAe,SAAS,KAAK,KAAK;AACvD,WAAO,QAAQ,eAAe,KAAK,KACjC,OAAO,QAAQ,eAAe,KAAK,KAAK,KAAK;AAC/C,QAAI,eAAe,SAAS,QAAQ;AACnC,aAAO;AAAA,IACR,WAAW,eAAe,SAAS,YAAY;AAC9C,aAAO;AAAA,IACR,WAAW,eAAe,SAAS,MAAM;AACxC,aAAO;AAAA,IACR,OAAO;AACN,aAAO;AAAA,IACR;AAAA,EACD;AACD;AAEA,SAAS,8BAA8B,OAIrC;AACD,QAAM,aAAa,MAAM,KAAK,EAAE,QAAQ,QAAQ,GAAG;AACnD,QAAM,YAAY,WAAW,MAAM,YAAY,IAAI,CAAC,GAAG,YAAY,KAAK;AACxE,QAAM,QAAQ,6BAA6B,YAAY,SAAS;AAChE,MAAI,cAAc,UAAU;AAC3B,WAAO,EAAE,WAAW,MAAM,QAAQ,MAAM;AAAA,EACzC;AACA,MACC,cAAc,YACd,cAAc,YACd,cAAc,YACd,cAAc,WACb;AACD,WAAO,EAAE,WAAW,MAAM,YAAY,MAAM;AAAA,EAC7C;AACA,MAAI,cAAc,WAAW,cAAc,YAAY,cAAc,YAAY;AAChF,WAAO,EAAE,WAAW,MAAM,MAAM,MAAM;AAAA,EACvC;AACA,SAAO,EAAE,WAAW,MAAM,SAAS,MAAM;AAC1C;AAEA,SAAS,6BAA6B,OAAe,WAA2B;AAC/E,QAAM,QAAQ,MAAM,YAAY;AAChC,MAAI,cAAc,UAAU;AAC3B,WAAO,WAAW,OAAO,uBAAuB,KAAK;AAAA,EACtD;AACA,MAAI,cAAc,YAAY,cAAc,WAAW;AACtD,WAAO,WAAW,OAAO,uBAAuB,KAAK;AAAA,EACtD;AACA,MAAI,cAAc,UAAU;AAC3B,WAAO,WAAW,OAAO,yBAAyB,KAAK;AAAA,EACxD;AACA,MAAI,cAAc,UAAU;AAC3B,WAAO,WAAW,OAAO,uBAAuB,KAAK;AAAA,EACtD;AACA,MAAI,cAAc,WAAW,cAAc,YAAY,cAAc,YAAY;AAChF,WAAO;AAAA,EACR;AACA,SAAO;AACR;AAEA,SAAS,WAAW,OAAe,SAAgC;AAClE,SAAO,QAAQ,KAAK,KAAK,IAAI,CAAC,KAAK;AACpC;AAEA,SAAS,iBAAiB,OAAyB;AAClD,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACpD,WAAO;AAAA,EACR;AACA,MAAI;AACH,UAAM,SAAS,KAAK,MAAM,KAAK;AAC/B,WAAO,OAAO,kBAAkB,QAAQ,OAAO,kBAAkB;AAAA,EAClE,QAAQ;AACP,WAAO;AAAA,EACR;AACD;AAEA,SAASA,OAAM,IAA2B;AACzC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC;AACrE;AAEA,eAAe,6BAA6B,UAAqC;AAChF,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAMrB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA,GAIrB;AACF,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,GAKrB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA,GAGrB;AACF,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAWrB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOrB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAarB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACD;AAEA,eAAe,yBAAyB,UAAqC;AAC5E,QAAM,WAAW,MAAM,SAAS,QAAQ,wCAAwC;AAChF,MAAI,OAAO,SAAS,CAAC,GAAG,SAAS,CAAC,IAAI,GAAG;AACxC;AAAA,EACD;AAEA,QAAM,OAAM,oBAAI,KAAK,0BAA0B,GAAE,QAAQ;AACzD,QAAMC,QAAO,CAAC,SAAiB,IAAI,OAAO,IAAI;AAC9C,QAAM,QAAQ,CAAC,UAAkB,IAAI,KAAK,MAAM,QAAQ,GAAK,EAAE,YAAY;AAE3E,QAAMC,aAAY,UAAU,uDAAuD;AAAA,IAClF,CAAC,iBAAiB,gBAAgB,MAAM,CAAC,CAAC;AAAA,IAC1C,CAAC,oBAAoB,KAAK,UAAU,EAAE,MAAM,MAAM,eAAe,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,IAClF,CAAC,mBAAmB,KAAK,UAAU,EAAE,WAAW,MAAM,SAAS,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,EACpF,CAAC;AAED,QAAM,cAA2B,CAAC;AAClC,WAAS,QAAQ,GAAG,SAAS,eAAe,SAAS;AACpD,UAAM,OAAO,QAAQ,MAAM,IAAI,cAAc;AAC7C,gBAAY,KAAK;AAAA,MAChB,UAAU,KAAK;AAAA,MACf;AAAA,MACAD,MAAK,qBAAqB;AAAA,MAC1B;AAAA,MACA;AAAA,MACA,MAAM,KAAK;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAMC;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,qBAAkC,CAAC;AACzC,WAAS,QAAQ,GAAG,QAAQ,yBAAyB,GAAG,SAAS;AAChE,UAAM,iBAAiB,IAAK,QAAQ,KAAM;AAC1C,UAAM,cAAc,KAAK,IAAI,GAAG,iBAAiB,CAAC;AAClD,UAAM,cAAc,KAAK,IAAI,eAAe,iBAAiB,CAAC;AAC9D,UAAM,YAAY,UAAU,QAAQ,CAAC;AACrC,uBAAmB,KAAK;AAAA,MACvB,UAAU,WAAW;AAAA,MACrB,UAAU,cAAc;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,IACD,CAAC;AACD,uBAAmB,KAAK;AAAA,MACvB,UAAU,WAAW;AAAA,MACrB,UAAU,cAAc;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAMA;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,eAA4B,CAAC;AACnC,WAAS,QAAQ,GAAG,SAAS,iBAAiB,SAAS;AACtD,UAAM,iBAAiB,KAAM,QAAQ,KAAK,KAAM;AAChD,iBAAa,KAAK;AAAA,MACjB,UAAU,KAAK;AAAA,MACf,YAAY,KAAK;AAAA,MACjB,QAAQ,QAAQ,EAAE;AAAA,MAClB,KAAK,UAAU,EAAE,MAAM,aAAa,KAAK,GAAG,CAAC;AAAA,MAC7C;AAAA,MACA,UAAU,cAAc;AAAA,MACxB,MAAM,KAAK;AAAA,MACX;AAAA,MACA;AAAA,MACA,KAAK,UAAU;AAAA,QACd,IAAI;AAAA,QACJ,KAAK,EAAE,QAAQ,QAAQ,QAAQD,MAAK,sBAAsB,EAAE;AAAA,MAC7D,CAAC;AAAA,MACD;AAAA,MACA,MAAM,QAAQ,GAAG;AAAA,IAClB,CAAC;AAAA,EACF;AACA,QAAMC;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,mBAAgC,CAAC;AACvC,WAAS,QAAQ,GAAG,SAAS,qBAAqB,SAAS;AAC1D,UAAM,SAAS,KAAK,UAAU;AAAA,MAC7B,MAAM,QAAQ,KAAK;AAAA,MACnB,aAAaD,MAAK,0BAA0B;AAAA,MAC5C,cAAc,EAAE,MAAM,UAAU,YAAY,CAAC,EAAE;AAAA,IAChD,CAAC;AACD,qBAAiB,KAAK,CAAC,iBAAiB,QAAQ,KAAK,IAAI,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,EAC/E;AACA,QAAMC;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,kBAA+B,CAAC;AACtC,WAAS,QAAQ,GAAG,SAAS,oBAAoB,SAAS;AACzD,oBAAgB,KAAK;AAAA,MACpB;AAAA,MACA,QAAQ,MAAM,IAAI,kBAAkB;AAAA,MACpC,KAAK,UAAU,EAAE,MAAM,cAAc,MAAMD,MAAK,0BAA0B,EAAE,CAAC;AAAA,MAC7E,MAAM,KAAK;AAAA,IACZ,CAAC;AAAA,EACF;AACA,QAAMC;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,mBAAgC,CAAC;AACvC,WAAS,QAAQ,GAAG,SAAS,eAAe,SAAS;AACpD,qBAAiB,KAAK,CAAC,UAAU,KAAK,GAAG,KAAK,CAAC;AAAA,EAChD;AACA,QAAMA;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,SAAS;AAAA,IACd;AAAA,IACA,KAAK,UAAU,EAAE,KAAK,cAAc,MAAMD,MAAK,IAAK,EAAE,CAAC;AAAA,IACvD,MAAM,CAAC;AAAA,EACR;AACA,QAAM,SAAS;AAAA,IACd;AAAA,IACA,KAAK,UAAU,EAAE,WAAW,KAAQ,MAAMA,MAAK,EAAE,EAAE,CAAC;AAAA,IACpD,MAAM,CAAC;AAAA,EACR;AACA,QAAM,SAAS;AAAA,IACd;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,KAAK,UAAU,CAAC,CAAC;AAAA,IACjB,KAAK,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC;AAAA,IAC/B,MAAM,CAAC;AAAA,IACP,MAAM,CAAC;AAAA,EACR;AACD;AAEA,eAAeC,aACd,UACA,cACA,MACA,YAAY,KACI;AAChB,MAAI,KAAK,WAAW,GAAG;AACtB;AAAA,EACD;AACA,QAAM,cAAc,KAAK,CAAC,GAAG,UAAU;AACvC,MAAI,gBAAgB,GAAG;AACtB;AAAA,EACD;AACA,QAAM,iBAAiB,IAAI,KAAK,OAAO,WAAW,EAAE,MAAM,GAAG,EAAE,CAAC;AAChE,WAAS,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS,WAAW;AAC5D,UAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,SAAS;AACjD,UAAM,SAAS,MAAM,IAAI,MAAM,cAAc,EAAE,KAAK,GAAG;AACvD,UAAM,WAAW,MAAM,KAAK;AAC5B,UAAM,SAAS,QAAQ,GAAG,YAAY,WAAW,MAAM,IAAI,GAAG,QAAQ;AAAA,EACvE;AACD;AAEA,SAAS,UAAU,OAAuB;AACzC,SAAO,KAAK,OAAO,KAAK,EAAE,SAAS,IAAI,GAAG,CAAC;AAC5C;AAEA,SAAS,UAAU,OAAuB;AACzC,SAAO,SAAS,OAAO,KAAK,EAAE,SAAS,IAAI,GAAG,CAAC;AAChD;AAEA,SAAS,OAAO,OAAuB;AACtC,SAAO,MAAM,QAAQ,mBAAmB,GAAG,EAAE,MAAM,GAAG,EAAE;AACzD;;;ACn/CA,SAAS,SAAAC,eAA8D;AACvE,SAAS,MAAAC,YAAU;AAEZ,IAAM,+BAA+B;AACrC,IAAM,2BAA2B;AACxC,IAAM,mBAAmB,KAAK,KAAK;AACnC,IAAM,wBAAwB,KAAK,KAAK;AACxC,IAAM,2BAA2B;AACjC,IAAM,6BAA6B;AAEnC,SAASC,OAAM,IAA2B;AACzC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACxD;AAEA,SAAS,YAAY,OAAwB;AAC5C,MAAI,iBAAiB,MAAO,QAAO,MAAM,SAAS,MAAM;AACxD,SAAO,OAAO,KAAK;AACpB;AAEO,IAAM,oBAAoBF,QAAM;AAAA,EACtC,OAAO;AAAA,IACN,OAAO;AAAA,IACP,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,mBAAmB;AAAA,IACnB,eAAe;AAAA,IACf,iBAAiB;AAAA,IACjB,cAAc;AAAA,IACd,kBAAkB;AAAA,IAClB,wBAAwB;AAAA,IACxB,mBAAmB;AAAA,IACnB,kBAAkB;AAAA,EACnB;AAAA,EACA,YAAY,OAAO;AAAA,IAClB,YAAY,oBAAI,IAAwB;AAAA,EACzC;AAAA,EACA,IAAIC,KAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQtB;AAAA,IACF;AAAA,EACD,CAAC;AAAA,EACD,aAAa,CAAC,GAAG,cAAkC;AAClD,MAAE,KAAK,WAAW,IAAI,SAAS;AAC/B,MAAE,MAAM,mBAAmB;AAC3B,UAAM,eAAe,OAAO,WAAW;AAEvC,MAAE,IAAI,KAAK;AAAA,MACV,KAAK;AAAA,MACL,OAAO,EAAE,MAAM;AAAA,MACf;AAAA,MACA,iBAAiB,EAAE,MAAM;AAAA,IAC1B,CAAC;AAED,cAAU;AAAA,MACT,KAAK,UAAU;AAAA,QACd,MAAM;AAAA,QACN;AAAA,QACA,OAAO,EAAE,MAAM;AAAA,QACf,iBAAiB,EAAE,MAAM;AAAA,MAC1B,CAAC;AAAA,IACF;AAEA,cAAU,iBAAiB,WAAW,CAACE,YAA6B;AACnE,QAAE,MAAM,gBAAgB;AACxB,YAAM,OAAOA,QAAM;AACnB,UAAI,OAAO,SAAS,SAAU;AAE9B,UAAI;AACH,cAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,YAAI,OAAO,SAAS,QAAQ;AAC3B,oBAAU;AAAA,YACT,KAAK,UAAU;AAAA,cACd,MAAM;AAAA,cACN;AAAA,cACA,cAAc,EAAE,MAAM;AAAA,cACtB,WAAW,KAAK,IAAI;AAAA,YACrB,CAAC;AAAA,UACF;AACA;AAAA,QACD;AAAA,MACD,QAAQ;AAAA,MAAC;AAET,gBAAU;AAAA,QACT,KAAK,UAAU;AAAA,UACd,MAAM;AAAA,UACN;AAAA,UACA,UAAU;AAAA,UACV,cAAc,EAAE,MAAM;AAAA,UACtB,WAAW,KAAK,IAAI;AAAA,QACrB,CAAC;AAAA,MACF;AAAA,IACD,CAAC;AAED,cAAU,iBAAiB,SAAS,CAACA,YAAU;AAC9C,QAAE,KAAK,WAAW,OAAO,SAAS;AAClC,QAAE,MAAM,mBAAmB;AAC3B,QAAE,IAAI,KAAK;AAAA,QACV,KAAK;AAAA,QACL,OAAO,EAAE,MAAM;AAAA,QACf;AAAA,QACA,iBAAiB,EAAE,MAAM;AAAA,QACzB,MAAMA,QAAM;AAAA,QACZ,QAAQA,QAAM;AAAA,MACf,CAAC;AAAA,IACF,CAAC;AAAA,EACF;AAAA,EACA,QAAQ,OAAO,MAAM;AACpB,MAAE,MAAM,aAAa;AACrB,MAAE,IAAI,KAAK;AAAA,MACV,KAAK;AAAA,MACL,OAAO,EAAE,MAAM;AAAA,MACf,WAAW,EAAE,MAAM;AAAA,MACnB,YAAY,EAAE,MAAM;AAAA,IACrB,CAAC;AACD,UAAM,EAAE,GAAG;AAAA,MACV;AAAA,MACA;AAAA,MACA,EAAE,MAAM;AAAA,MACR,QAAQ,EAAE,MAAM,SAAS;AAAA,MACzB,KAAK,IAAI;AAAA,IACV;AAAA,EACD;AAAA,EACA,SAAS,OAAO,MAAM;AACrB,UAAM,aAAa,EAAE,MAAM,aAAa;AACxC,UAAM,YAAY,KAAK,IAAI;AAC3B,MAAE,MAAM,aAAa;AACrB,MAAE,MAAM,mBAAmB;AAC3B,MAAE,MAAM,yBAAyB;AACjC,MAAE,MAAM,oBAAoB;AAC5B,MAAE,MAAM,mBAAmB;AAE3B,MAAE,IAAI,KAAK;AAAA,MACV,KAAK;AAAA,MACL,OAAO,EAAE,MAAM;AAAA,MACf;AAAA,MACA,mBAAmB,EAAE,MAAM;AAAA,MAC3B,eAAe,EAAE,MAAM;AAAA,IACxB,CAAC;AAED,QAAI;AACH,iBAAW,aAAa,EAAE,KAAK,YAAY;AAC1C,YAAI,UAAU,eAAe,EAAG;AAChC,kBAAU;AAAA,UACT,KAAK,UAAU;AAAA,YACd,MAAM;AAAA,YACN;AAAA,YACA,mBAAmB,EAAE,MAAM;AAAA,YAC3B,eAAe,EAAE,MAAM;AAAA,YACvB,WAAW;AAAA,UACZ,CAAC;AAAA,QACF;AAAA,MACD;AAEA,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA,EAAE,MAAM;AAAA,QACR;AAAA,MACD;AAEA,YAAM,WAAW,YAAY,EAAE,MAAM;AACrC,UAAI,YAAY;AAChB,aAAO,KAAK,IAAI,IAAI,UAAU;AAC7B,cAAM,SAAS,KAAK;AAAA,UACnB,EAAE,MAAM;AAAA,UACR,KAAK,IAAI,GAAG,WAAW,KAAK,IAAI,CAAC;AAAA,QAClC;AACA,YAAI,SAAS,EAAG,OAAMD,OAAM,MAAM;AAElC,qBAAa;AACb,cAAM,SAAS,KAAK,IAAI;AACxB,cAAM,SAAS,QAAQ,SAAS,eAAe,SAAS,SAAS;AACjE,cAAM,EAAE,GAAG;AAAA,UACV;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACD;AACA,UAAE,IAAI,KAAK;AAAA,UACV,KAAK;AAAA,UACL,OAAO,EAAE,MAAM;AAAA,UACf;AAAA,UACA;AAAA,UACA,WAAW,SAAS;AAAA,QACrB,CAAC;AAED,mBAAW,aAAa,EAAE,KAAK,YAAY;AAC1C,cAAI,UAAU,eAAe,EAAG;AAChC,oBAAU;AAAA,YACT,KAAK,UAAU;AAAA,cACd,MAAM;AAAA,cACN;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,cACpB,WAAW;AAAA,YACZ,CAAC;AAAA,UACF;AAAA,QACD;AAAA,MACD;AAEA,YAAM,kBAAkB,KAAK,IAAI;AACjC,QAAE,MAAM,yBAAyB;AACjC,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA,YAAY,kBAAkB,SAAS;AAAA,QACvC;AAAA,MACD;AAEA,YAAM,aAAa,KAAK,IAAI;AAC5B,QAAE,MAAM,oBAAoB;AAC5B,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA,EAAE,MAAM;AAAA,QACR;AAAA,MACD;AAEA,iBAAW,aAAa,EAAE,KAAK,YAAY;AAC1C,YAAI,UAAU,eAAe,EAAG;AAChC,kBAAU;AAAA,UACT,KAAK,UAAU;AAAA,YACd,MAAM;AAAA,YACN;AAAA,YACA,WAAW,aAAa;AAAA,YACxB,WAAW;AAAA,UACZ,CAAC;AAAA,QACF;AACA,kBAAU;AAAA,UACT;AAAA,UACA;AAAA,QACD;AAAA,MACD;AAEA,QAAE,IAAI,KAAK;AAAA,QACV,KAAK;AAAA,QACL,OAAO,EAAE,MAAM;AAAA,QACf;AAAA,QACA,WAAW,aAAa;AAAA,MACzB,CAAC;AAAA,IACF,SAAS,OAAO;AACf,YAAM,UAAU,YAAY,KAAK;AACjC,QAAE,MAAM,mBAAmB;AAC3B,QAAE,IAAI,MAAM;AAAA,QACX,KAAK;AAAA,QACL,OAAO,EAAE,MAAM;AAAA,QACf;AAAA,QACA,OAAO;AAAA,MACR,CAAC;AACD,YAAM;AAAA,IACP;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,SAAS,OACR,GACA,QAAQ,uBAAuB,KAAK,IAAI,CAAC,IACzC,oBAAoB,8BACpB,gBAAgB,6BACZ;AACJ,UAAI,CAAC,OAAO,SAAS,iBAAiB,KAAK,oBAAoB,GAAG;AACjE,cAAM,IAAI,MAAM,wDAAwD;AAAA,MACzE;AACA,UAAI,CAAC,OAAO,SAAS,aAAa,KAAK,iBAAiB,GAAG;AAC1D,cAAM,IAAI,MAAM,gDAAgD;AAAA,MACjE;AACA,QAAE,MAAM,QAAQ;AAChB,QAAE,MAAM,oBAAoB;AAC5B,QAAE,MAAM,gBAAgB;AACxB,YAAM,EAAE,GAAG;AAAA,QACV;AAAA,QACA;AAAA,QACA,EAAE,MAAM;AAAA,QACR;AAAA,QACA,KAAK,IAAI;AAAA,MACV;AACA,aAAO;AAAA,QACN,OAAO,EAAE,MAAM;AAAA,QACf,mBAAmB,EAAE,MAAM;AAAA,QAC3B,eAAe,EAAE,MAAM;AAAA,QACvB,WAAW,EAAE,MAAM;AAAA,QACnB,YAAY,EAAE,MAAM;AAAA,QACpB,iBAAiB,EAAE,MAAM;AAAA,QACzB,cAAc,EAAE,MAAM;AAAA,MACvB;AAAA,IACD;AAAA,IACA,UAAU,OAAO,MAAM;AACtB,YAAM,OAAO,MAAM,EAAE,GAAG,QAMrB,6CAA6C;AAChD,aAAO;AAAA,QACN,OAAO;AAAA,UACN,OAAO,EAAE,MAAM;AAAA,UACf,WAAW,EAAE,MAAM;AAAA,UACnB,YAAY,EAAE,MAAM;AAAA,UACpB,mBAAmB,EAAE,MAAM;AAAA,UAC3B,eAAe,EAAE,MAAM;AAAA,UACvB,iBAAiB,EAAE,MAAM;AAAA,UACzB,cAAc,EAAE,MAAM;AAAA,UACtB,kBAAkB,EAAE,MAAM;AAAA,UAC1B,wBAAwB,EAAE,MAAM;AAAA,UAChC,mBAAmB,EAAE,MAAM;AAAA,UAC3B,kBAAkB,EAAE,MAAM;AAAA,QAC3B;AAAA,QACA;AAAA,MACD;AAAA,IACD;AAAA,EACD;AAAA,EACA,SAAS;AAAA,IACR,uBAAuB;AAAA,IACvB,cAAc;AAAA,IACd,kBAAkB;AAAA,EACnB;AACD,CAAC;;;ACxUD,SAAS,SAAAE,SAAO,aAAa;AAC7B,SAAS,MAAAC,YAAU;AAyDnB,IAAMC,cAAN,MAAiB;AAAA,EACR,SAAS;AAAA,EACT,UAA6B,CAAC;AAAA,EAEtC,MAAM,UAAyB;AAC9B,QAAI,CAAC,KAAK,QAAQ;AACjB,WAAK,SAAS;AACd;AAAA,IACD;AACA,UAAM,IAAI,QAAc,CAAC,YAAY,KAAK,QAAQ,KAAK,OAAO,CAAC;AAC/D,SAAK,SAAS;AAAA,EACf;AAAA,EAEA,UAAgB;AACf,UAAM,OAAO,KAAK,QAAQ,MAAM;AAChC,QAAI,MAAM;AACT,WAAK;AACL;AAAA,IACD;AACA,SAAK,SAAS;AAAA,EACf;AACD;AAEA,SAAS,SAAS,SAGK;AACtB,QAAM,QAAQ,IAAIA,YAAW;AAC7B,MAAI,oBAA+B;AAEnC,QAAM,sBAAsB,MAAU;AACrC,UAAM,KAAK,OAAO;AAAA,MACjB,CAAmC,UAAkB,WACpD,QAAW,OAAO,GAAG,MAAM;AAAA,MAC5B;AAAA,QACC,iBAAiB,OAAU,OAA2C,GAAG,EAAE;AAAA,MAC5E;AAAA,IACD;AACA,WAAO;AAAA,EACR;AAEA,QAAM,iBAAiB,OACtB,UACG,WACe;AAClB,QAAI,mBAAmB;AACtB,aAAO,kBAAqB,OAAO,GAAG,MAAM;AAAA,IAC7C;AACA,UAAM,MAAM,QAAQ;AACpB,QAAI;AACH,aAAO,MAAM,QAAW,OAAO,GAAG,MAAM;AAAA,IACzC,UAAE;AACD,YAAM,QAAQ;AAAA,IACf;AAAA,EACD;AAEA,QAAM,MAAM,OAAO,OAAO,gBAAgB;AAAA,IACzC,iBAAiB,OAAU,OAA2C;AACrE,UAAI,mBAAmB;AACtB,eAAO,GAAG,iBAAiB;AAAA,MAC5B;AACA,YAAM,MAAM,QAAQ;AACpB,YAAM,KAAK,oBAAoB;AAC/B,UAAI;AACH,cAAM,QAAQ,OAAO;AACrB,4BAAoB;AACpB,YAAI;AACH,gBAAM,SAAS,MAAM,GAAG,EAAE;AAC1B,8BAAoB;AACpB,gBAAM,QAAQ,QAAQ;AACtB,iBAAO;AAAA,QACR,SAAS,OAAO;AACf,8BAAoB;AACpB,gBAAM,QAAQ,UAAU;AACxB,gBAAM;AAAA,QACP;AAAA,MACD,UAAE;AACD,4BAAoB;AACpB,cAAM,QAAQ;AAAA,MACf;AAAA,IACD;AAAA,EACD,CAAC;AACD,SAAO;AACR;AAEA,IAAMC,iBAAgB;AACtB,IAAMC,0BAAyB;AAC/B,IAAMC,mBAAkB;AACxB,IAAMC,uBAAsB;AAC5B,IAAMC,sBAAqB;AAE3B,IAAMC,yBAAwB;AAC9B,IAAMC,8BAA6B;AACnC,IAAMC,0BAAyB;AAC/B,IAAMC,8BAA6B;AAE5B,IAAM,qBAAqBX,QAAM;AAAA,EACvC,OAAO,EAAE,UAAU,EAAE;AAAA,EACrB,IAAIC,KAAG;AAAA,IACN,WAAW,OAAO,aAAa;AAC9B,YAAM,0BAA0B,QAAQ;AAAA,IACzC;AAAA,EACD,CAAC;AAAA,EACD,MAAM,EAAE,KAAK,KAAK;AAAA,EAClB,aAAa,CAAC,GAAG,OAAO;AACvB,UAAM,OAAO;AACb,QAAI,KAAK,eAAe,UAAU,MAAM;AACvC,WAAK,KAAK,MAAM;AAAA,IACjB;AAEA,OAAG,iBAAiB,WAAW,CAACW,YAAU;AACzC,YAAM,UAAU,oCAAoC,GAAG,MAAMA,QAAM,IAAI;AACvE,WAAK,EAAE,UAAU,OAAO;AAAA,IACzB,CAAC;AAAA,EACF;AAAA,EACA,SAAS;AAAA,IACR,SAAS,OAAO,MAAM;AACrB,YAAM,0BAA0B,EAAE,EAAE;AACpC,aAAO,MAAM,sBAAsB,EAAE,EAAE;AAAA,IACxC;AAAA,IACA,gBAAgB,OAAO,GAAG,aAAsB;AAC/C,QAAE,KAAK,QAAQ,sBAAsB,EAAE,EAAE;AACzC,QAAE,MAAM;AACR,aAAO,MAAM,kBAAkB,EAAE,KAAK,KAAK,YAAY,UAAU,EAAE,MAAM,QAAQ,IAAI,CAAC;AAAA,IACvF;AAAA,IACA,aAAa,CAAC,MAAM,EAAE,MAAM;AAAA,IAC5B,OAAO,CAAC,MAAM;AACb,QAAE,MAAM;AACR,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;AAED,eAAe,oCACd,GACA,MACA,MACgB;AAChB,MAAI,SAAS,QAAQ;AACpB,QAAI,KAAK,eAAe,UAAU,MAAM;AACvC,WAAK,KAAK,MAAM;AAAA,IACjB;AACA;AAAA,EACD;AAEA,MAAI,UAAoD;AACxD,MAAI;AACH,UAAM,UAAU,0BAA0B,IAAI;AAC9C,cAAU,QAAQ;AAClB,MAAE,KAAK,QAAQ,sBAAsB,EAAE,EAAE;AACzC,MAAE,MAAM;AAER,QAAI,QAAQ,SAAS,iBAAiB;AACrC,YAAM,YAAY,YAAY,IAAI;AAClC,YAAMC,UAAS,MAAMC,oBAAmB,EAAE,KAAK,KAAK,QAAQ,OAAO;AACnE,eAAS,MAAM;AAAA,QACd,MAAM;AAAA,QACN,SAAS,QAAQ;AAAA,QACjB,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,QACjD,SAAS,CAACD,OAAM;AAAA,MACjB,CAAC;AACD;AAAA,IACD;AAEA,UAAM,WACL,QAAQ,SAAS,qBACd,QAAQ,WACP,QAAQ,YAAY,kBAAkB,EAAE,MAAM,QAAQ;AAC3D,UAAM,kBAAkB,QAAQ,SAAS,oBAAqB,QAAQ,mBAAmB,IAAK;AAC9F,UAAM,SAAS,MAAM,kBAAkB,EAAE,KAAK,KAAK,UAAU,eAAe;AAE5E,QAAI,QAAQ,SAAS,oBAAoB;AACxC,eAAS,MAAM;AAAA,QACd,MAAM;AAAA,QACN,YAAY;AAAA,QACZ,qBAAqBP;AAAA,QACrB,mBAAmB,CAAC;AAAA,QACpB,iBAAiB;AAAA,MAClB,CAAC;AAAA,IACF;AAEA,aAAS,MAAM;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,MACjB,GAAG;AAAA,IACJ,CAAC;AAAA,EACF,SAAS,OAAO;AACf,aAAS,MAAM;AAAA,MACd,MAAM;AAAA,MACN;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC7D,CAAC;AAAA,EACF;AACD;AAEA,SAAS,0BAA0B,MAAqC;AACvE,MAAI,OAAO,SAAS,UAAU;AAC7B,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC9D;AACA,QAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AAC1C,UAAM,IAAI,MAAM,8CAA8C;AAAA,EAC/D;AACA,QAAM,UAAU;AAChB,MAAI,QAAQ,SAAS,iBAAiB;AACrC,WAAO,EAAE,MAAM,iBAAiB,SAASS,aAAY,SAAS,SAAS,EAAE;AAAA,EAC1E;AACA,MAAI,QAAQ,SAAS,oBAAoB;AACxC,UAAM,eAAe,QAAQ;AAC7B,WAAO;AAAA,MACN,MAAM;AAAA,MACN,UAAUC,aAAY,SAAS,UAAU;AAAA,MACzC,GAAI,iBAAiB,kBACrB,iBAAiB,aACjB,iBAAiB,YACd,EAAE,aAAa,IACf,CAAC;AAAA,IACL;AAAA,EACD;AACA,MAAI,QAAQ,SAAS,mBAAmB;AACvC,WAAO;AAAA,MACN,MAAM;AAAA,MACN,GAAI,OAAO,QAAQ,aAAa,WAAW,EAAE,UAAU,QAAQ,SAAS,IAAI,CAAC;AAAA,MAC7E,GAAI,OAAO,QAAQ,oBAAoB,WACpC,EAAE,iBAAiB,QAAQ,gBAAgB,IAC3C,CAAC;AAAA,IACL;AAAA,EACD;AACA,QAAM,IAAI,MAAM,4CAA4C,OAAO,QAAQ,IAAI,CAAC,EAAE;AACnF;AAEA,SAASA,aAAY,QAAiC,OAAuB;AAC5E,QAAM,QAAQ,OAAO,KAAK;AAC1B,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACpD,UAAM,IAAI,MAAM,8BAA8B,KAAK,6BAA6B;AAAA,EACjF;AACA,SAAO;AACR;AAEA,SAASD,aAAY,QAAiC,OAAuB;AAC5E,QAAM,QAAQ,OAAO,KAAK;AAC1B,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;AACzD,UAAM,IAAI,MAAM,8BAA8B,KAAK,0BAA0B;AAAA,EAC9E;AACA,SAAO;AACR;AAEA,SAAS,SACR,MACA,SACO;AACP,MAAI,KAAK,eAAe,UAAU,MAAM;AACvC,SAAK,KAAK,KAAK,UAAU,OAAO,CAAC;AAAA,EAClC;AACD;AAEA,SAAS,sBAAsBd,MAAoB;AAClD,SAAO,SAAS,OACf,UACG,WACe;AAClB,UAAM,YAAY,OAAO;AAAA,MAAI,CAAC,UAC7B,OAAO,UAAU,YAAa,QAAQ,IAAI,IAAK;AAAA,IAChD;AACA,WAAQ,MAAMA,KAAG,QAAQ,OAAO,GAAG,SAAS;AAAA,EAC7C,CAAC;AACF;AAEA,eAAe,kBACd,KACA,UACA,iBACgE;AAChE,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,uBAAuBgB,yBAAwB,GAAG;AACxD,QAAM,kBAAkBH,oBAAmB,KAAK,CAAC;AACjD,QAAM,mBAAmBI,qBAAoB,GAAG;AAChD,QAAM,wBAAwBC,OAAM,eAAe,EAAE;AAAA,IAAK,MACzD,yBAAyB,KAAK,QAAQ;AAAA,EACvC;AAEA,QAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,IACjC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,CAAC;AACD,SAAO;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAe,yBACd,KACA,UACuC;AACvC,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAA6B,CAAC;AACpC,QAAM,UAAU,MAAM,IAAI,gBAAgB,OAAO,OAAO;AACvD,UAAM,iBAAiB,MAAMC;AAAA,MAC5B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,mBAAmB,OAAO,eAAe,CAAC,GAAG,eAAe,eAAe;AACjF,UAAMA;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,eAAe,MAAMA;AAAA,MAC1B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,QAAI,CAAC,aAAa,CAAC,GAAG,OAAO;AAC5B,YAAMA;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,SACA,oBAAI,KAAK,GAAE,YAAY;AAAA,MACxB;AAAA,IACD;AACA,UAAM,gBAAgB,MAAMA;AAAA,MAC3B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,QAAIC,kBAAiB,cAAc,CAAC,GAAG,KAAK,GAAG;AAC9C,YAAMD;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,KAAK,UAAU,EAAE,MAAM,MAAM,eAAe,KAAK,CAAC;AAAA,SAClD,oBAAI,KAAK,GAAE,YAAY;AAAA,MACxB;AAAA,IACD;AACA,UAAM,UAAU,MAAMA;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,MAAM,OAAO,QAAQ,CAAC,GAAG,OAAO,CAAC;AACvC,UAAMA;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,UAAU,EAAE,MAAM,sBAAsB,YAAY,SAAS,CAAC;AAAA,OACnE,oBAAI,KAAK,GAAE,YAAY;AAAA,IACxB;AACA,WAAO;AAAA,EACR,CAAC;AACD,QAAM,KAAK;AAAA,IACV,MAAM;AAAA,IACN,YAAY,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACpD,UAAU;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAeH,yBAAwB,KAA+C;AACrF,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAA6B,CAAC;AACpC,QAAM,iBAAiB,MAAMG;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM,mBAAmB,OAAO,eAAe,CAAC,GAAG,eAAe,eAAe;AACjF,QAAMA;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAMA;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AACA,QAAM,iBAAiB,MAAMA;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuBD;AACA,QAAM,sBAAsB,eAAe,CAAC,GAAG;AAC/C,MAAI,OAAO,wBAAwB,UAAU;AAC5C,UAAMA;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAQA;AAAA,IACD;AACA,UAAMA;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,EACD;AACA,QAAMA;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA;AAAA,EAID;AACA,QAAMA;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA;AAAA,EAID;AACA,SAAO;AAAA,IACN,MAAM;AAAA,IACN,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACjD;AAAA,EACD;AACD;AAEA,eAAeN,oBAAmB,KAAS,SAAuD;AACjG,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAA6B,CAAC;AACpC,QAAM,QAAQ,IAAI;AAAA,IACjBM;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACAA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACAA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACAA,YAAW,KAAK,OAAO,eAAe,wCAAwC;AAAA,IAC9EA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACAA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACAA,YAAW,KAAK,OAAO,kBAAkB,qDAAqD;AAAA,IAC9FA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,IACAA;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAAA,EACD,CAAC;AACD,QAAM,KAAK,CAAC,GAAG,MAAM,EAAE,aAAa,EAAE,UAAU;AAChD,SAAO,EAAE,MAAM,oBAAoB,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS,GAAG,MAAM;AAC9F;AAEA,eAAeF,qBAAoB,KAA+C;AACjF,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,QAA6B,CAAC;AACpC,QAAME;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA;AAAA,EAID;AACA,QAAMA;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA,EAGD;AACA,QAAMA;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA;AAAA,EAGD;AACA,SAAO,EAAE,MAAM,sBAAsB,SAAS,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS,GAAG,MAAM;AAChG;AAEA,eAAeA,YACd,KACA,OACA,MACA,UACG,QACY;AACf,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,OAAO,MAAM,IAAO,OAAO,GAAG,MAAM;AAC1C,QAAM,KAAK;AAAA,IACV;AAAA,IACA,YAAY,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS;AAAA,IACpD,UAAU,KAAK;AAAA,EAChB,CAAC;AACD,SAAO;AACR;AAEA,SAASC,kBAAiB,OAAyB;AAClD,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACpD,WAAO;AAAA,EACR;AACA,MAAI;AACH,UAAM,SAAS,KAAK,MAAM,KAAK;AAC/B,WAAO,OAAO,kBAAkB,QAAQ,OAAO,kBAAkB;AAAA,EAClE,QAAQ;AACP,WAAO;AAAA,EACR;AACD;AAEA,SAASF,OAAM,IAA2B;AACzC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC;AACrE;AAEA,eAAe,0BAA0B,UAAqC;AAC7E,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAMrB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA,GAIrB;AACF,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,GAKrB;AACF,QAAM,SAAS,QAAQ,wEAAwE;AAC/F,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA,GAGrB;AACF,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAWrB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOrB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAarB;AACF,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS,QAAQ,sEAAsE;AAC7F,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACA,QAAM,SAAS;AAAA,IACd;AAAA,EACD;AACD;AAEA,eAAe,sBAAsB,UAKlC;AACF,QAAM,WAAW,MAAM,SAAS,QAAQ,wCAAwC;AAChF,MAAI,OAAO,SAAS,CAAC,GAAG,SAAS,CAAC,IAAI,GAAG;AACxC,WAAO;AAAA,MACN,QAAQ;AAAA,MACR,UAAUhB;AAAA,MACV,WAAWE;AAAA,MACX,cAAcE;AAAA,IACf;AAAA,EACD;AAEA,QAAM,OAAM,oBAAI,KAAK,0BAA0B,GAAE,QAAQ;AACzD,QAAMe,QAAO,CAAC,SAAiB,IAAI,OAAO,IAAI;AAC9C,QAAM,QAAQ,CAAC,UAAkB,IAAI,KAAK,MAAM,QAAQ,GAAK,EAAE,YAAY;AAE3E,QAAMC,aAAY,UAAU,uDAAuD;AAAA,IAClF,CAAC,iBAAiB,gBAAgB,MAAM,CAAC,CAAC;AAAA,IAC1C,CAAC,kBAAkB,KAAK,UAAU,EAAE,MAAM,MAAM,eAAe,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,IAChF,CAAC,mBAAmB,KAAK,UAAU,EAAE,WAAW,MAAM,SAAS,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,EACpF,CAAC;AAED,QAAM,cAA2B,CAAC;AAClC,WAAS,QAAQ,GAAG,SAASpB,gBAAe,SAAS;AACpD,UAAM,OAAO,QAAQ,MAAM,IAAI,cAAc;AAC7C,gBAAY,KAAK;AAAA,MAChBqB,WAAU,KAAK;AAAA,MACf;AAAA,MACAF,MAAKd,sBAAqB;AAAA,MAC1B;AAAA,MACA;AAAA,MACA,MAAM,KAAK;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAMe;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,qBAAkC,CAAC;AACzC,WAAS,QAAQ,GAAG,QAAQnB,0BAAyB,GAAG,SAAS;AAChE,UAAM,iBAAiB,IAAK,QAAQ,KAAM;AAC1C,UAAM,cAAc,KAAK,IAAI,GAAG,iBAAiB,CAAC;AAClD,UAAM,cAAc,KAAK,IAAID,gBAAe,iBAAiB,CAAC;AAC9D,UAAM,YAAYsB,WAAU,QAAQ,CAAC;AACrC,uBAAmB,KAAK;AAAA,MACvBD,WAAU,WAAW;AAAA,MACrBA,WAAU,cAAc;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,IACD,CAAC;AACD,uBAAmB,KAAK;AAAA,MACvBA,WAAU,WAAW;AAAA,MACrBA,WAAU,cAAc;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAMD;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,eAA4B,CAAC;AACnC,WAAS,QAAQ,GAAG,SAASlB,kBAAiB,SAAS;AACtD,UAAM,iBAAiB,KAAM,QAAQ,KAAK,KAAM;AAChD,iBAAa,KAAK;AAAA,MACjBoB,WAAU,KAAK;AAAA,MACf,YAAY,KAAK;AAAA,MACjB,QAAQ,QAAQ,EAAE;AAAA,MAClB,KAAK,UAAU,EAAE,MAAM,aAAa,KAAK,GAAG,CAAC;AAAA,MAC7C;AAAA,MACAD,WAAU,cAAc;AAAA,MACxB,MAAM,KAAK;AAAA,MACX;AAAA,MACA;AAAA,MACA,KAAK,UAAU;AAAA,QACd,IAAI;AAAA,QACJ,KAAK,EAAE,QAAQ,QAAQ,QAAQF,MAAKZ,uBAAsB,EAAE;AAAA,MAC7D,CAAC;AAAA,MACD;AAAA,MACA,MAAM,QAAQ,GAAG;AAAA,IAClB,CAAC;AAAA,EACF;AACA,QAAMa;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,mBAAgC,CAAC;AACvC,WAAS,QAAQ,GAAG,SAASjB,sBAAqB,SAAS;AAC1D,UAAM,SAAS,KAAK,UAAU;AAAA,MAC7B,MAAM,QAAQ,KAAK;AAAA,MACnB,aAAagB,MAAKX,2BAA0B;AAAA,MAC5C,cAAc,EAAE,MAAM,UAAU,YAAY,CAAC,EAAE;AAAA,IAChD,CAAC;AACD,qBAAiB,KAAK,CAAC,iBAAiB,QAAQ,KAAK,IAAI,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,EAC/E;AACA,QAAMY;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,kBAA+B,CAAC;AACtC,WAAS,QAAQ,GAAG,SAAShB,qBAAoB,SAAS;AACzD,oBAAgB,KAAK;AAAA,MACpB;AAAA,MACA,QAAQ,MAAM,IAAI,kBAAkB;AAAA,MACpC,KAAK,UAAU,EAAE,MAAM,cAAc,MAAMe,MAAKb,2BAA0B,EAAE,CAAC;AAAA,MAC7E,MAAM,KAAK;AAAA,IACZ,CAAC;AAAA,EACF;AACA,QAAMc;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,mBAAgC,CAAC;AACvC,WAAS,QAAQ,GAAG,SAASpB,gBAAe,SAAS;AACpD,qBAAiB,KAAK,CAACqB,WAAU,KAAK,GAAG,KAAK,CAAC;AAAA,EAChD;AACA,QAAMD;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD;AAEA,QAAM,SAAS;AAAA,IACd;AAAA,IACA,KAAK,UAAU,EAAE,KAAK,cAAc,MAAMD,MAAK,IAAK,EAAE,CAAC;AAAA,IACvD,MAAM,CAAC;AAAA,EACR;AACA,QAAM,SAAS;AAAA,IACd;AAAA,IACA,KAAK,UAAU,EAAE,WAAW,KAAQ,MAAMA,MAAK,EAAE,EAAE,CAAC;AAAA,IACpD,MAAM,CAAC;AAAA,EACR;AACA,SAAO;AAAA,IACN,QAAQ;AAAA,IACR,UAAUnB;AAAA,IACV,WAAWE;AAAA,IACX,cAAcE;AAAA,EACf;AACD;AAEA,eAAegB,aACd,UACA,cACA,MACA,YAAY,KACI;AAChB,MAAI,KAAK,WAAW,GAAG;AACtB;AAAA,EACD;AACA,QAAM,cAAc,KAAK,CAAC,GAAG,UAAU;AACvC,MAAI,gBAAgB,GAAG;AACtB;AAAA,EACD;AACA,QAAM,iBAAiB,IAAI,KAAK,OAAO,WAAW,EAAE,MAAM,GAAG,EAAE,CAAC;AAChE,WAAS,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS,WAAW;AAC5D,UAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,SAAS;AACjD,UAAM,SAAS,MAAM,IAAI,MAAM,cAAc,EAAE,KAAK,GAAG;AACvD,UAAM,WAAW,MAAM,KAAK;AAC5B,UAAM,SAAS,QAAQ,GAAG,YAAY,WAAW,MAAM,IAAI,GAAG,QAAQ;AAAA,EACvE;AACD;AAEA,SAASC,WAAU,OAAuB;AACzC,SAAO,KAAK,OAAO,KAAK,EAAE,SAAS,IAAI,GAAG,CAAC;AAC5C;AAEA,SAASC,WAAU,OAAuB;AACzC,SAAO,SAAS,OAAO,KAAK,EAAE,SAAS,IAAI,GAAG,CAAC;AAChD;AAEO,IAAM,WAAW,MAAM;AAAA,EAC7B,KAAK,EAAE,mBAAmB;AAAA,EAC1B,wBAAwB,IAAI,OAAO;AAAA,EACnC,wBAAwB,IAAI,OAAO;AACpC,CAAC;AAED,IAAI,YAAY,MAAM;AACrB,WAAS,MAAM;AAChB;;;AC/8BA,SAAS,cAAc;AACvB,SAAS,cAAc,YAAY;AACnC,SAAS,SAAAC,SAAO,SAAAC,eAAa;AAC7B,SAAS,SAAS;;;ACHlB,eAAsB,WAAW,UAAkB;AAElD,SAAO;AAAA,IACN;AAAA,IACA,aAAa,KAAK,MAAM,KAAK,OAAO,IAAI,EAAE,IAAI;AAAA,IAC9C,WAAW,CAAC,SAAS,UAAU,SAAS,OAAO,EAC9C,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC,CAC7B;AAAA,IACA,UAAU,KAAK,MAAM,KAAK,OAAO,IAAI,EAAE,IAAI;AAAA,EAC5C;AACD;;;ADHO,IAAM,UAAUC,QAAM;AAAA;AAAA,EAE5B,OAAO;AAAA,IACN,UAAU,CAAC;AAAA,EACZ;AAAA,EACA,QAAQ;AAAA,IACP,iBAAiBC,QAAe;AAAA,EACjC;AAAA,EAEA,SAAS;AAAA;AAAA,IAER,aAAa,CAAC,MAAM,EAAE,MAAM;AAAA,IAE5B,aAAa,OAAO,GAAG,gBAAwB;AAC9C,YAAM,UAAmB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS;AAAA,QACT,WAAW,KAAK,IAAI;AAAA,MACrB;AAEA,QAAE,MAAM,SAAS,KAAK,OAAO;AAE7B,YAAM,EAAE,MAAAC,MAAK,IAAI,MAAM,aAAa;AAAA,QACnC,OAAO,OAAO,aAAa;AAAA,QAC3B,QAAQ;AAAA,QACR,UAAU,EAAE,MAAM;AAAA,QAClB,OAAO;AAAA,UACN,SAAS,KAAK;AAAA,YACb,aAAa;AAAA,YACb,YAAY,EAAE,OAAO;AAAA,cACpB,UAAU,EACR,OAAO,EACP;AAAA,gBACA;AAAA,cACD;AAAA,YACF,CAAC;AAAA,YACD,SAAS,OAAO,EAAE,SAAS,MAAM;AAChC,qBAAO,MAAM,WAAW,QAAQ;AAAA,YACjC;AAAA,UACD,CAAC;AAAA,QACF;AAAA,MACD,CAAC;AAED,YAAM,eAAwB;AAAA,QAC7B,MAAM;AAAA,QACN,SAASA;AAAA,QACT,WAAW,KAAK,IAAI;AAAA,MACrB;AACA,QAAE,MAAM,SAAS,KAAK,YAAY;AAGlC,QAAE,UAAU,mBAAmB,YAAY;AAE3C,aAAO;AAAA,IACR;AAAA,EACD;AACD,CAAC;;;AlEwED,SAASC,eAAc,MAAc,UAA0B;AAC9D,QAAM,QAAQ,QAAQ,IAAI,IAAI;AAC9B,MAAI,UAAU,UAAa,UAAU,GAAI,QAAO;AAEhD,QAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,CAAC,OAAO,SAAS,MAAM,GAAG;AAC7B,UAAM,IAAI,MAAM,GAAG,IAAI,0BAA0B;AAAA,EAClD;AAEA,SAAO;AACR;AAEA,SAAS,uBAAuB;AAK/B,MAAI,YAAY,MAAM,mBAAoB,QAAO;AAEjD,QAAM,MACL,QAAQ,IAAI,wBACZ,QAAQ,IAAI,+BACZ;AAED,SAAO;AAAA,IACN,MAAM,QAAQ,IAAI;AAAA,IAClB;AAAA,IACA,iBAAiBA;AAAA,MAChB;AAAA,MACA,KAAK;AAAA,IACN;AAAA,IACA,kBAAkBA;AAAA,MACjB;AAAA,MACA,KAAK;AAAA,IACN;AAAA,IACA,sBAAsBA;AAAA,MACrB;AAAA,MACA;AAAA,IACD;AAAA,IACA,UAAU;AAAA,MACT,QAAQ;AAAA,MACR,OAAO;AAAA,IACR;AAAA,EACD;AACD;AAEO,IAAMC,YAAWC,OAAM;AAAA,EAC7B,eAAe,qBAAqB;AAAA,EACpC,YAAY;AAAA,IACX,aACC,QAAQ,IAAI,sBAAsB,QAAQ,IAAI,eAAe;AAAA,IAC9D,sBAAsBF;AAAA,MACrB;AAAA,MACA,KAAK,OAAO;AAAA,IACb;AAAA,EACD;AAAA,EACA,KAAK;AAAA;AAAA,IAEJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAAG;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,EACD;AACD,CAAC;;;AoEzSD,SAAS,aAAa;AACtB,SAAS,QAAAC,aAAY;AAErB,YAAY,QAAQ;AAEpB,IAAM,MAAM,IAAIA,MAAK;AACrB,IAAM,OAAO,OAAO,SAAS,QAAQ,IAAI,QAAQ,QAAQ,EAAE;AAC3D,IAAM,OAAO,YAAY;AAEzB,QAAQ,GAAG,QAAQ,CAAC,SAAS;AAC5B,UAAQ,IAAI,KAAK,UAAU,EAAE,MAAM,gBAAgB,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC;AAC7E,CAAC;AACD,IAAI,QAAQ,IAAI,mCAAmC,KAAK;AACvD,aAAW,UAAU,CAAC,UAAU,SAAS,GAAY;AACpD,YAAQ,GAAG,QAAQ,MAAM;AACxB,cAAQ;AAAA,QACP,KAAK,UAAU;AAAA,UACd,MAAM;AAAA,UACN;AAAA,UACA,KAAK,QAAQ;AAAA,UACb,MAAM,QAAQ;AAAA,UACd,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,QACnC,CAAC;AAAA,MACF;AACA,cAAQ,KAAK,WAAW,WAAW,MAAM,GAAG;AAAA,IAC7C,CAAC;AAAA,EACF;AACD;AACA,QAAQ,GAAG,cAAc,CAAC,SAAS;AAClC,UAAQ,IAAI,KAAK,UAAU,EAAE,MAAM,uBAAuB,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC;AACpF,CAAC;AACD,QAAQ,GAAG,qBAAqB,CAAC,UAAU;AAC1C,UAAQ;AAAA,IACP,KAAK,UAAU;AAAA,MACd,MAAM;AAAA,MACN,OAAO,MAAM,SAAS,MAAM;AAAA,IAC7B,CAAC;AAAA,EACF;AACD,CAAC;AACD,QAAQ,GAAG,sBAAsB,CAAC,WAAW;AAC5C,UAAQ;AAAA,IACP,KAAK,UAAU;AAAA,MACd,MAAM;AAAA,MACN,OAAO,kBAAkB,QAAQ,OAAO,SAAS,OAAO,UAAU,OAAO,MAAM;AAAA,IAChF,CAAC;AAAA,EACF;AACD,CAAC;AAED,eAAe,gBAAgB,SAAkB;AAChD,QAAM,KAAM,WAAuD;AACnE,MAAI,WAAW,OAAO,OAAO,WAAY,IAAG;AAE5C,QAAM,SAAS,QAAQ,YAAY;AACnC,QAAM,OAAU,qBAAkB;AAClC,QAAM,SAAY,0BAAuB;AACzC,QAAM,sBAAsB,KAAK,IAAI,GAAG,OAAO,MAAM,KAAK,eAAe;AAEzE,SAAO;AAAA,IACN,KAAK,QAAQ;AAAA,IACb,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC,eAAe,QAAQ,OAAO;AAAA,IAC9B,aAAa;AAAA,IACb,aAAa,OAAO,OAAO;AAAA,IAC3B,SAAS;AAAA,MACR,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO;AAAA,MACvB,eAAe,OAAO;AAAA,MACtB,eAAe,OAAO;AAAA,MACtB,mBAAmB,OAAO;AAAA,IAC3B;AAAA,IACA,IAAI;AAAA,MACH,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK;AAAA,MACxB,oBAAoB,KAAK;AAAA,MACzB,qBAAqB,KAAK;AAAA,MAC1B,qBAAqB,KAAK;AAAA,MAC1B,yBAAyB,KAAK;AAAA,MAC9B,QAAQ,OAAO,IAAI,CAAC,WAAW;AAAA,QAC9B,MAAM,MAAM;AAAA,QACZ,WAAW,MAAM;AAAA,QACjB,WAAW,MAAM;AAAA,QACjB,gBAAgB,MAAM;AAAA,QACtB,mBAAmB,MAAM;AAAA,MAC1B,EAAE;AAAA,IACH;AAAA,IACA,WAAW;AAAA,MACV,qBAAqB,OAAO;AAAA,MAC5B,iBAAiB,OAAO;AAAA,MACxB,iBAAiB,OAAO;AAAA,MACxB,kCAAkC;AAAA,IACnC;AAAA,IACA,eAAe,QAAQ,cAAc;AAAA,EACtC;AACD;AAeA,IAAI,IAAI,iBAAiB,OAAO,MAAM;AACrC,QAAM,UAAU,EAAE,IAAI,MAAM,IAAI,MAAM;AACtC,SAAO,EAAE,KAAK,MAAM,gBAAgB,OAAO,CAAC;AAC7C,CAAC;AAED,IAAI,IAAI,WAAW,MAAMC,UAAS,OAAO,OAAO,CAAC;AAEjD,IAAI,IAAI,aAAa,MAAMA,UAAS,OAAO,SAAS,CAAC;AAErD,IAAI,IAAI,YAAY,CAAC,MAAMA,UAAS,OAAO,kBAAkB,EAAE,IAAI,GAAG,CAAC;AAEvE,IAAI,KAAK,wBAAwB,CAAC,MAAM;AACvC,MAAI,QAAQ,IAAI,mCAAmC,KAAK;AACvD,WAAO,EAAE,KAAK,EAAE,OAAO,WAAW,GAAG,GAAG;AAAA,EACzC;AAEA,QAAM,OAAO,EAAE,IAAI,MAAM,MAAM;AAC/B,MAAI,CAAC,MAAM;AACV,WAAO,EAAE,KAAK,EAAE,OAAO,eAAe,GAAG,GAAG;AAAA,EAC7C;AAEA,QAAM,cAAiB,qBAAkB,IAAI;AAC7C,SAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACpC,CAAC;AAED,IAAI,IAAI,KAAK,OAAO,GAAG,SAAS;AAC/B,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK;AAWZ,CAAC;AAMD,IAAI,SAAS,aAAa;AACzB,EAAAA,UAAS,MAAM;AAChB,OAAO;AACN,MAAI,IAAI,gBAAgB,CAAC,MAAMA,UAAS,QAAQ,EAAE,IAAI,GAAG,CAAC;AAC1D,MAAI,IAAI,cAAc,CAAC,MAAMA,UAAS,QAAQ,EAAE,IAAI,GAAG,CAAC;AACzD;AAEA,IAAM,SAAS,MAAM,EAAE,OAAO,IAAI,OAAO,KAAK,GAAG,MAAM;AACtD,MAAI,SAAS,aAAa;AACzB,YAAQ;AAAA,MACP,0DAA0D,IAAI;AAAA,IAC/D;AAAA,EACD,OAAO;AACN,YAAQ;AAAA,MACP,iBAAiB,IAAI,mCAAmC,IAAI;AAAA,IAC7D;AAAA,EACD;AACD,CAAC;AACD,IAAM,aAAa;AACnB,WAAW,iBAAiB;AAC5B,WAAW,iBAAiB;AAC5B,WAAW,mBAAmB;AAC9B,WAAW,UAAU;","names":["setup","event","actor","event","actor","event","actor","actor","event","actor","actor","actor","actor","UserError","actor","actor","actor","actor","actor","actor","db","actor","db","todos","actor","db","actor","event","db","actor","event","c","actor","UserError","actor","event","actor","actor","text","actor","event","Hono","actor","app","actor","event","actor","event","actor","event","actor","actor","event","sleep","actor","event","actor","actor","actor","event","queue","actor","event","queue","payload","actor","event","queue","payload","actor","event","Loop","workflow","actor","event","workflow","Loop","actor","event","Loop","workflow","actor","event","workflow","Loop","actor","event","Loop","workflow","actor","event","workflow","batch","Loop","actor","event","queue","Loop","workflow","actor","queue","event","workflow","Loop","actor","event","queue","Loop","workflow","actor","queue","event","workflow","Loop","actor","event","Loop","workflow","actor","event","workflow","Loop","actor","event","Loop","workflow","actor","event","workflow","Loop","actor","queue","Loop","workflow","actor","workflow","Loop","queue","actor","actor","events","result1","result2","actor","actor","db","actor","db","actor","db","batch","actor","db","actor","db","DEFAULT_ROW_BYTES","CHAT_LOG_CHUNK_BYTES","CHAT_LOG_INSERT_BATCH_SIZE","positiveInteger","buildChatLogMessage","seedChatLog","checksum","actor","db","payload","counter","mode","event","actor","db","DEFAULT_ROW_BYTES","numberField","queryOne","actor","db","sleep","positiveInteger","typedRows","payload","event","actor","event","actor","db","send","payload","sleep","event","actor","db","send","event","result","delay","text","batchInsert","actor","db","sleep","event","actor","db","AsyncMutex","MESSAGE_COUNT","MESSAGE_TOOL_REF_COUNT","TOOL_CALL_COUNT","EXECUTOR_TOOL_COUNT","THREAD_EVENT_COUNT","MESSAGE_CONTENT_BYTES","THREAD_EVENT_PAYLOAD_BYTES","TOOL_CALL_RESULT_BYTES","EXECUTOR_TOOL_SCHEMA_BYTES","event","result","runCatchupSnapshot","numberField","stringField","runBuildToolPlanContext","runRecoverToolCalls","delay","timedQuery","hasPendingLaunch","text","batchInsert","messageId","toolUseID","actor","event","actor","event","text","numberFromEnv","registry","setup","sleep","Hono","registry"]} \ No newline at end of file diff --git a/examples/kitchen-sink/package.json b/examples/kitchen-sink/package.json index acb2ac47df..3bc7c0e83a 100644 --- a/examples/kitchen-sink/package.json +++ b/examples/kitchen-sink/package.json @@ -15,8 +15,11 @@ "memory-soak": "tsx scripts/sqlite-memory-soak.ts", "proc-metrics": "tsx scripts/proc-metrics-report.ts", "smoke:raw-websocket-serverless": "tsx scripts/raw-websocket-serverless-smoke.ts", + "smoke:on-sleep-sigterm": "tsc --noEmit --pretty false --allowImportingTsExtensions --moduleResolution bundler --module esnext --target esnext --types node --lib esnext,dom --skipLibCheck --allowJs --checkJs false scripts/on-sleep-sigterm-smoke.ts src/actors/testing/sigterm-sleep-probe.ts src/actors/state/sqlite-drizzle/drizzle/migrations.js && node --experimental-strip-types --experimental-transform-types --import @rivetkit/sql-loader scripts/on-sleep-sigterm-smoke.ts", "fuzz:sleep-close": "tsx scripts/sleep-close-fuzz.ts", "mock-agentic-loop": "tsx scripts/mock-agentic-loop.ts", + "slow-reconnect": "tsx src/actors/testing/bench-slow-reconnect.ts", + "counter-latency": "cargo run --quiet --release --manifest-path scripts/counter-latency/Cargo.toml --", "benchmark": "tsx scripts/benchmark.ts", "db:generate": "find src/actors -name drizzle.config.ts -exec drizzle-kit generate --config {} \\;" }, diff --git a/examples/kitchen-sink/scripts/bench.ts b/examples/kitchen-sink/scripts/bench.ts index 930d973da8..560ad6e5e6 100644 --- a/examples/kitchen-sink/scripts/bench.ts +++ b/examples/kitchen-sink/scripts/bench.ts @@ -29,6 +29,7 @@ const url = new URL(RAW_ENDPOINT); const NAMESPACE = url.username; const TOKEN = url.password; const HOST = `${url.protocol}//${url.host}`; +const RVT_RUNNER = process.env.RIVET_POOL ?? "k8s"; async function callAction( actorName: string, @@ -42,7 +43,7 @@ async function callAction( "rvt-key": key.join(","), "rvt-token": TOKEN, "rvt-namespace": NAMESPACE, - "rvt-runner": "default", + "rvt-runner": RVT_RUNNER, }); const actionUrl = `${HOST}/gateway/${actorName}/action/${action}?${params}`; const res = await fetch(actionUrl, { diff --git a/examples/kitchen-sink/scripts/counter-latency/Cargo.lock b/examples/kitchen-sink/scripts/counter-latency/Cargo.lock new file mode 100644 index 0000000000..7954e6a97c --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/Cargo.lock @@ -0,0 +1,1607 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "counter-latency" +version = "0.0.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "futures-util", + "http", + "serde", + "serde_json", + "tokio", + "tokio-tungstenite", + "url", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl" +version = "0.10.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "native-tls", + "tokio", + "tokio-native-tls", + "tungstenite", +] + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "native-tls", + "rand", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/examples/kitchen-sink/scripts/counter-latency/Cargo.toml b/examples/kitchen-sink/scripts/counter-latency/Cargo.toml new file mode 100644 index 0000000000..df75380fdd --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "counter-latency" +version = "0.0.0" +edition = "2021" +publish = false + +[[bin]] +name = "counter-latency" +path = "src/main.rs" + +[dependencies] +anyhow = "1" +chrono = "0.4" +clap = { version = "4", features = ["derive"] } +futures-util = "0.3" +http = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tokio = { version = "1", features = ["full"] } +tokio-tungstenite = { version = "0.24", features = ["native-tls"] } +url = "2" + +[workspace] diff --git a/examples/kitchen-sink/scripts/counter-latency/src/args.rs b/examples/kitchen-sink/scripts/counter-latency/src/args.rs new file mode 100644 index 0000000000..9e77bd0baf --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/src/args.rs @@ -0,0 +1,195 @@ +// CLI + env parsing. Uses clap derive with short flags for every option. + +use std::env; +use std::process; + +use clap::{Parser, Subcommand, ValueEnum}; + +pub const DEFAULT_CONCURRENCY: u32 = 1_000; +pub const DEFAULT_CONCURRENT_INTERVAL_MS: u64 = 300; +pub const DEFAULT_MESSAGE_INTERVAL_MS: u64 = 1_000; +pub const DEFAULT_AGENT_MESSAGE_INTERVAL_MS: u64 = 30_000; +pub const DEFAULT_AGENT_CONCURRENT_2_SLEEP_MS: u64 = 5_000; +pub const DEFAULT_AGENT_CONCURRENT_2_TIMEOUT_MS: u64 = 120_000; +pub const DEFAULT_AGENT_CONCURRENT_2_QUERY_MULTIPLIER: u32 = 1; +pub const DEFAULT_TOKENS_PER_SECOND: f64 = 20.0; +pub const DEFAULT_DURATION_MS: u64 = 5_000; +pub const MESSAGE_GAP_WARN_MS: f64 = 3_000.0; +pub const ACTOR_STOPPED_CLOSE_CODE: u16 = 1000; +pub const ACTOR_STOPPED_CLOSE_REASON: &str = "hack_force_close"; + +#[derive(Parser)] +#[command( + name = "counter-latency", + about = "Mini load-test client for Rivet kitchen-sink actors", + long_about = "Subcommands:\n \ + concurrent ramp raw WebSocket tunnel-stress actors (steady or rolling)\n \ + agent-concurrent ramp SQLite-backed agent actors (steady or rolling)\n \ + agent-concurrent-2 cycle SQLite-backed agent actors through work and sleep\n\nEnv:\n \ + RIVET_ENDPOINT required, proto://:@host\n \ + RIVET_POOL runner pool name (default k8s)\n \ + RUN_FOR_MS stop after this many ms" +)] +struct Cli { + #[command(subcommand)] + command: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Ramp raw WebSocket tunnel-stress actors. Set `-c 1 --mode rolling` for an rtt-style workload. + Concurrent(ConcurrentCli), + /// Ramp SQLite-backed agent actors. Set `-c 1 --mode rolling` for an rtt-style workload. + #[command(name = "agent-concurrent")] + AgentConcurrent(ConcurrentCli), + /// Cycle SQLite-backed agent actors through one workload pass and forced sleep. + #[command(name = "agent-concurrent-2")] + AgentConcurrent2(ConcurrentCli), +} + +#[derive(clap::Args, Clone)] +struct ConcurrentCli { + /// Worker lifecycle: `steady` keeps each connection alive; `rolling` closes the connection + /// after the second inbound message (rtt-style) and a fresh worker replaces it, maintaining N + /// in-flight. + #[arg(short = 'M', long = "mode", value_enum, default_value_t = WorkerMode::Steady)] + worker_mode: WorkerMode, + /// Ramp-up gap in ms between connection starts. + #[arg(short = 'i', long, default_value_t = DEFAULT_CONCURRENT_INTERVAL_MS)] + interval: u64, + /// Number of in-flight connections. + #[arg(short = 'c', long, default_value_t = DEFAULT_CONCURRENCY)] + concurrency: u32, + /// Gap between client messages in ms (default: 1000 concurrent / 30000 agent-concurrent). + #[arg(short = 'm', long = "message-interval-ms")] + message_interval_ms: Option, + /// SQLite token inserts per second (agent-concurrent only). + #[arg(short = 't', long, default_value_t = DEFAULT_TOKENS_PER_SECOND)] + tokens_per_second: f64, + /// Inference stream duration in ms (agent-concurrent only). + #[arg(short = 'd', long, default_value_t = DEFAULT_DURATION_MS)] + duration_ms: u64, + /// Log all received WebSocket messages. + #[arg(short = 's', long)] + show_messages: bool, + /// Wait for actor ready before connecting (default: skip). + #[arg(short = 'w', long)] + wait_ready: bool, + /// Delay after forcing actor sleep before reconnecting (agent-concurrent-2 only). + #[arg(long, default_value_t = DEFAULT_AGENT_CONCURRENT_2_SLEEP_MS)] + sleep_ms: u64, + /// Per-workload timeout in ms (agent-concurrent-2 only). + #[arg(long, default_value_t = DEFAULT_AGENT_CONCURRENT_2_TIMEOUT_MS)] + timeout_ms: u64, + /// Delay before the server-side write transaction starts (agent-concurrent-2 only). + #[arg(long, default_value_t = 0)] + stagger_handle_ms: u64, + /// Number of times to repeat the SQL workload per cycle (agent-concurrent-2 only). + #[arg(long, default_value_t = DEFAULT_AGENT_CONCURRENT_2_QUERY_MULTIPLIER)] + query_multiplier: u32, +} + +#[derive(Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum WorkerMode { + /// Each worker keeps its connection alive forever and reconnects on actor-stopped close. + Steady, + /// Each worker closes after its second inbound message. A fresh worker immediately replaces it. + Rolling, +} + +#[derive(Clone)] +pub struct ConcurrentArgs { + pub mode: ConcurrentMode, + pub worker_mode: WorkerMode, + pub interval: u64, + pub concurrency: u32, + pub message_interval: u64, + pub show_messages: bool, + pub skip_ready_wait: bool, + pub tokens_per_second: f64, + pub duration_ms: u64, + pub sleep_ms: u64, + pub timeout_ms: u64, + pub stagger_handle_ms: u64, + pub query_multiplier: u32, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ConcurrentMode { + Concurrent, + AgentConcurrent, + AgentConcurrent2, +} + +#[derive(Clone)] +pub enum Args { + Concurrent(ConcurrentArgs), +} + +impl Args { + pub fn interval(&self) -> u64 { + match self { + Args::Concurrent(a) => a.interval, + } + } +} + +pub struct EnvConfig { + pub run_for_ms: u64, + pub rivet_pool: String, + pub endpoint: String, +} + +impl EnvConfig { + pub fn from_env() -> Self { + let run_for_ms = env::var("RUN_FOR_MS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + let rivet_pool = env::var("RIVET_POOL").unwrap_or_else(|_| "k8s".to_string()); + let endpoint = match env::var("RIVET_ENDPOINT") { + Ok(v) if !v.is_empty() => v, + _ => { + eprintln!("RIVET_ENDPOINT is required (proto://:@host)"); + process::exit(1); + } + }; + Self { run_for_ms, rivet_pool, endpoint } + } +} + +pub fn parse_cli() -> Args { + let cli = Cli::parse(); + match cli.command { + Cmd::Concurrent(c) => Args::Concurrent(build_concurrent(ConcurrentMode::Concurrent, c)), + Cmd::AgentConcurrent(c) => { + Args::Concurrent(build_concurrent(ConcurrentMode::AgentConcurrent, c)) + } + Cmd::AgentConcurrent2(c) => { + Args::Concurrent(build_concurrent(ConcurrentMode::AgentConcurrent2, c)) + } + } +} + +fn build_concurrent(mode: ConcurrentMode, cli: ConcurrentCli) -> ConcurrentArgs { + let default_message_interval = match mode { + ConcurrentMode::AgentConcurrent => DEFAULT_AGENT_MESSAGE_INTERVAL_MS, + ConcurrentMode::AgentConcurrent2 => DEFAULT_MESSAGE_INTERVAL_MS, + ConcurrentMode::Concurrent => DEFAULT_MESSAGE_INTERVAL_MS, + }; + ConcurrentArgs { + mode, + worker_mode: cli.worker_mode, + interval: cli.interval, + concurrency: cli.concurrency, + message_interval: cli.message_interval_ms.unwrap_or(default_message_interval), + show_messages: cli.show_messages, + skip_ready_wait: !cli.wait_ready, + tokens_per_second: cli.tokens_per_second, + duration_ms: cli.duration_ms, + sleep_ms: cli.sleep_ms, + timeout_ms: cli.timeout_ms, + stagger_handle_ms: cli.stagger_handle_ms, + query_multiplier: cli.query_multiplier.max(1), + } +} diff --git a/examples/kitchen-sink/scripts/counter-latency/src/concurrent.rs b/examples/kitchen-sink/scripts/counter-latency/src/concurrent.rs new file mode 100644 index 0000000000..d0602d698c --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/src/concurrent.rs @@ -0,0 +1,1710 @@ +// Concurrent + agent-concurrent mode. Owns the per-worker WS loop with +// reconnect logic, the workload trait, and the live logging that mirrors +// scripts/counter-latency.ts. + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Instant; + +use futures_util::{SinkExt, StreamExt}; +use tokio::sync::Mutex; +use tokio::sync::mpsc; +use tokio::time::{sleep, timeout}; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::tungstenite::protocol::CloseFrame; +use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; + +use crate::args::{ + ACTOR_STOPPED_CLOSE_CODE, ACTOR_STOPPED_CLOSE_REASON, ConcurrentArgs, ConcurrentMode, + EnvConfig, MESSAGE_GAP_WARN_MS, WorkerMode, +}; +use crate::endpoint::Endpoint; +use crate::log::{ + BLUE, BOLD, CYAN, DIM, GREEN, RED, RESET, YELLOW, color_ms, format_actor, iso_now, pad, +}; +use crate::stats::{State, WorkerHealth}; +use crate::ws::open_raw_ws; + +pub fn make_key(worker: u32, prefix: &str) -> String { + let now_ms = chrono::Utc::now().timestamp_millis(); + format!("{}-{}-{}", prefix, worker, base36(now_ms as u64)) +} + +fn base36(mut n: u64) -> String { + if n == 0 { + return "0".to_string(); + } + let mut s = String::new(); + const ALPHA: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + while n > 0 { + s.push(ALPHA[(n % 36) as usize] as char); + n /= 36; + } + s.chars().rev().collect() +} + +#[derive(Clone)] +pub struct ConcurrentWorkerOptions { + pub message_interval: u64, + pub show_messages: bool, + pub skip_ready_wait: bool, + pub tokens_per_second: f64, + pub duration_ms: u64, +} + +pub struct WorkloadCtx { + pub endpoint: Arc, + pub args: Arc, + pub env: Arc, + pub state: Arc, +} + +impl WorkloadCtx { + /// Prefix format: `TIMESTAMP [connecting/pinging/connected/slow/failed]`. + /// + /// Order matches the worker lifecycle: handshake → pings → steady → slow → failed. + /// Each cell is colored independently so the dominant state is visually obvious. + pub fn log_prefix(&self) -> String { + let ts = iso_now(); + let counts = self.state.count_worker_health(); + let width = format!("{}", self.args.concurrency).len(); + let pad_num = |n: i64| format!("{:>width$}", n, width = width); + let status_part = format!( + "[{}{}{}/{}{}{}/{}{}{}/{}{}{}/{}{}{}]", + BLUE, + pad_num(counts.connecting), + RESET, + CYAN, + pad_num(counts.pinging), + RESET, + GREEN, + pad_num(counts.connected), + RESET, + YELLOW, + pad_num(counts.connected_slow), + RESET, + RED, + pad_num(counts.failed), + RESET, + ); + format!("{}{}{} {}", DIM, ts, RESET, status_part) + } +} + +pub trait Workload: Send + Sync { + fn key_prefix(&self) -> &'static str; + fn suppress_generic_gap(&self) -> bool { + false + } + fn actor_name(&self) -> &'static str; + fn on_open( + &self, + _ctx: Arc, + _worker: u32, + _key: String, + _options: ConcurrentWorkerOptions, + _send_tx: mpsc::Sender, + ) -> WorkloadHooks { + WorkloadHooks::default() + } +} + +#[derive(Default)] +pub struct WorkloadHooks { + pub on_message: Option>, +} + +pub struct TunnelStressWorkload; + +impl Workload for TunnelStressWorkload { + fn key_prefix(&self) -> &'static str { + "cl-t" + } + fn actor_name(&self) -> &'static str { + "tunnelStress" + } + fn on_open( + &self, + _ctx: Arc, + _worker: u32, + _key: String, + options: ConcurrentWorkerOptions, + send_tx: mpsc::Sender, + ) -> WorkloadHooks { + tokio::spawn(async move { + let mut sequence: u64 = 0; + loop { + sleep(std::time::Duration::from_millis(options.message_interval)).await; + sequence += 1; + let payload = serde_json::json!({ + "sequence": sequence, + "timestamp": chrono::Utc::now().timestamp_millis(), + }) + .to_string(); + if send_tx.send(payload).await.is_err() { + break; + } + } + }); + WorkloadHooks::default() + } +} + +pub struct AgentWorkload; + +impl Workload for AgentWorkload { + fn key_prefix(&self) -> &'static str { + "cl-a" + } + fn suppress_generic_gap(&self) -> bool { + true + } + fn actor_name(&self) -> &'static str { + "loadTestAgent" + } + fn on_open( + &self, + ctx: Arc, + worker: u32, + key: String, + options: ConcurrentWorkerOptions, + send_tx: mpsc::Sender, + ) -> WorkloadHooks { + let pending_inference_sends: Arc>> = + Arc::new(Mutex::new(HashMap::new())); + + // Periodic inference sender. + let pending_sends_clone = pending_inference_sends.clone(); + let key_for_send = key.clone(); + let tokens_per_second = options.tokens_per_second; + let duration_ms = options.duration_ms; + let message_interval = options.message_interval; + tokio::spawn(async move { + let mut sequence: u64 = 0; + let mut first = true; + loop { + if !first { + sleep(std::time::Duration::from_millis(message_interval)).await; + } + first = false; + sequence += 1; + let now_ms = chrono::Utc::now().timestamp_millis() as u64; + let request_id = format!( + "agent-{}-{}-{}", + worker, + to_base36(now_ms), + sequence + ); + pending_sends_clone + .lock() + .await + .insert(request_id.clone(), Instant::now()); + let payload = serde_json::json!({ + "type": "inference", + "requestId": request_id, + "tokensPerSecond": tokens_per_second, + "durationMs": duration_ms, + }) + .to_string(); + if send_tx.send(payload).await.is_err() { + break; + } + } + let _ = key_for_send; + }); + + let ctx_for_hook = ctx.clone(); + let key_for_hook = key.clone(); + let on_message: Box = Box::new(move |data: &str| { + let Ok(message) = serde_json::from_str::(data) else { + return; + }; + let ty = message.get("type").and_then(|v| v.as_str()).unwrap_or(""); + if ty == "inference-start" { + if let Some(request_id) = message.get("requestId").and_then(|v| v.as_str()) { + let pending = pending_inference_sends.clone(); + let request_id = request_id.to_string(); + let ctx_inner = ctx_for_hook.clone(); + let key_inner = key_for_hook.clone(); + tokio::spawn(async move { + let mut map = pending.lock().await; + if let Some(sent_at) = map.remove(&request_id) { + let elapsed_ms = + sent_at.elapsed().as_secs_f64() * 1000.0; + if elapsed_ms > MESSAGE_GAP_WARN_MS { + log_message_gap( + &ctx_inner, + worker, + &key_inner, + None, + elapsed_ms, + ); + } + } + }); + } + } else if ty == "slow-sql" { + let elapsed_ms = message.get("elapsedMs").and_then(|v| v.as_f64()); + let request_id = + message.get("requestId").and_then(|v| v.as_str()).unwrap_or("?"); + let token_index = message + .get("tokenIndex") + .and_then(|v| v.as_i64()) + .map(|n| n.to_string()) + .unwrap_or_else(|| "?".to_string()); + if let Some(ms) = elapsed_ms { + let detail = format!("req={} token={}", request_id, token_index); + log_slow_sql(&ctx_for_hook, worker, &key_for_hook, None, ms, &detail); + } + } + }); + WorkloadHooks { + on_message: Some(on_message), + } + } +} + +fn to_base36(mut n: u64) -> String { + if n == 0 { + return "0".to_string(); + } + let mut s = String::new(); + const ALPHA: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + while n > 0 { + s.push(ALPHA[(n % 36) as usize] as char); + n /= 36; + } + s.chars().rev().collect() +} + +pub async fn run_concurrent_mode( + args: ConcurrentArgs, + env: Arc, + endpoint: Arc, + state: Arc, +) { + let args = Arc::new(args); + if matches!(args.mode, ConcurrentMode::AgentConcurrent2) { + let ctx = Arc::new(WorkloadCtx { + endpoint: endpoint.clone(), + args: args.clone(), + env: env.clone(), + state: state.clone(), + }); + run_agent_concurrent_2_mode(args.clone(), env.clone(), ctx.clone(), state.clone()).await; + print_concurrent_summary(&ctx, "complete"); + return; + } + + let workload: Arc = match args.mode { + ConcurrentMode::AgentConcurrent => Arc::new(AgentWorkload), + ConcurrentMode::Concurrent => Arc::new(TunnelStressWorkload), + ConcurrentMode::AgentConcurrent2 => unreachable!(), + }; + let ctx = Arc::new(WorkloadCtx { + endpoint: endpoint.clone(), + args: args.clone(), + env: env.clone(), + state: state.clone(), + }); + + // Run-for-ms guard: close workers after the deadline. + if env.run_for_ms > 0 { + let state_clone = state.clone(); + let dur = std::time::Duration::from_millis(env.run_for_ms); + tokio::spawn(async move { + sleep(dur).await; + state_clone.set_stopping(); + }); + } + + let options = ConcurrentWorkerOptions { + message_interval: args.message_interval, + show_messages: args.show_messages, + skip_ready_wait: args.skip_ready_wait, + tokens_per_second: args.tokens_per_second, + duration_ms: args.duration_ms, + }; + + match args.worker_mode { + WorkerMode::Steady => { + run_steady(args.clone(), ctx.clone(), workload, state.clone(), options).await; + } + WorkerMode::Rolling => { + run_rolling(args.clone(), ctx.clone(), workload, state.clone(), options).await; + } + } + + print_concurrent_summary(&ctx, "complete"); +} + +async fn run_agent_concurrent_2_mode( + args: Arc, + env: Arc, + ctx: Arc, + state: Arc, +) { + if env.run_for_ms > 0 { + let state_clone = state.clone(); + let dur = std::time::Duration::from_millis(env.run_for_ms); + tokio::spawn(async move { + sleep(dur).await; + state_clone.set_stopping(); + }); + } + + let mut handles: Vec> = Vec::new(); + for i in 0..args.concurrency { + let id = i + 1; + state.set_workers_started(id as i64); + state.set_worker_health(id, WorkerHealth::Connecting); + let ctx_clone = ctx.clone(); + let handle = tokio::spawn(async move { + run_agent_concurrent_2_worker(id, ctx_clone).await; + }); + handles.push(handle); + if i < args.concurrency - 1 { + sleep(std::time::Duration::from_millis(args.interval)).await; + } + } + for handle in handles { + let _ = handle.await; + } +} + +async fn run_agent_concurrent_2_worker(worker: u32, ctx: Arc) { + let key = make_key(worker, "cl-a2"); + let mut sequence: u64 = 0; + let actor_id: Option = None; + + 'worker_loop: while !ctx.state.stopping() { + sequence += 1; + ctx.state.set_worker_health(worker, WorkerHealth::Connecting); + let t0 = Instant::now(); + let url = + ctx.endpoint + .build_raw_ws_url("loadTestAgent2", &key, ctx.args.skip_ready_wait); + + let ws = match open_raw_ws_with_stall_log(&url, &ctx, &key, actor_id.as_deref()).await { + Ok(ws) => ws, + Err(err) => { + let elapsed = t0.elapsed().as_secs_f64() * 1000.0; + log_connect_error( + &ctx, + worker, + &key, + actor_id.as_deref(), + elapsed, + &err.to_string(), + ); + return; + } + }; + let t_connect = Instant::now(); + let connect_ms = t_connect.duration_since(t0).as_secs_f64() * 1000.0; + record_connect(&ctx, worker, sequence > 1); + + let (mut sink, mut stream) = ws.split(); + let phase1 = run_ping_phase( + &mut sink, + &mut stream, + t0, + t_connect, + connect_ms, + &ctx, + &key, + actor_id.as_deref(), + ) + .await; + + match &phase1 { + PingOutcome::Completed { + first_ms, + second_ms, + total_ms, + } => { + log_rtt_line( + &ctx, + worker, + &key, + actor_id.as_deref(), + connect_ms, + *first_ms, + *second_ms, + *total_ms, + sequence > 1, + ); + } + PingOutcome::Failed { first_ms, close } => { + log_partial_rtt( + &ctx, + worker, + &key, + actor_id.as_deref(), + connect_ms, + *first_ms, + sequence > 1, + ); + if let Some((code, reason)) = close { + let detail = format!("code={} reason={}", code, reason); + log_disconnect(&ctx, worker, &key, actor_id.as_deref(), &detail, true); + } + // Per-worker phase-1 failure: continue outer loop IMMEDIATELY to reproduce + // thundering-herd reconnect on storm. Do not call set_stopping (would kill + // the whole test) and do not sleep (would stagger reconnects). + continue 'worker_loop; + } + } + + for repeat_index in 0..ctx.args.query_multiplier { + let request_id = format!( + "agent2-{}-{}-{}-{}", + worker, + to_base36(sequence), + repeat_index + 1, + to_base36(chrono::Utc::now().timestamp_millis() as u64), + ); + let payload = serde_json::json!({ + "type": "agent2_connect", + "clientId": request_id, + "staggerHandleMs": ctx.args.stagger_handle_ms, + }) + .to_string(); + + if let Err(err) = sink.send(Message::Text(payload.into())).await { + let _ = err; + log_websocket_error(&ctx, worker, &key, actor_id.as_deref()); + // Per-worker send error: drop this connection, continue immediately. + continue 'worker_loop; + } + + let result = timeout( + std::time::Duration::from_millis(ctx.args.timeout_ms), + wait_agent_concurrent_2_result( + &mut stream, + &ctx, + worker, + &key, + actor_id.as_deref(), + ), + ) + .await; + + match result { + Ok(AgentConcurrent2Cycle::Result { total_ms, summary }) => { + log_agent_concurrent_2_result( + &ctx, + worker, + &key, + actor_id.as_deref(), + total_ms, + &summary, + ); + } + Ok(AgentConcurrent2Cycle::ServerError { error }) => { + log_agent_concurrent_2_error(&ctx, worker, &key, actor_id.as_deref(), &error); + continue 'worker_loop; + } + Ok(AgentConcurrent2Cycle::Closed { detail }) => { + log_disconnect(&ctx, worker, &key, actor_id.as_deref(), &detail, true); + continue 'worker_loop; + } + Err(_) => { + let detail = format!( + "timeout waiting for agent-concurrent-2 result after {}ms", + ctx.args.timeout_ms, + ); + log_disconnect(&ctx, worker, &key, actor_id.as_deref(), &detail, true); + continue 'worker_loop; + } + } + } + + let _ = sink + .send(Message::Text( + serde_json::json!({ "type": "force_sleep" }).to_string().into(), + )) + .await; + let _ = timeout( + std::time::Duration::from_millis(5_000), + wait_agent_concurrent_2_sleeping(&mut stream), + ) + .await; + let _ = sink + .send(Message::Close(Some(CloseFrame { + code: CloseCode::Normal, + reason: "counter-latency complete".into(), + }))) + .await; + + sleep(std::time::Duration::from_millis(ctx.args.sleep_ms)).await; + } +} + +enum AgentConcurrent2Cycle { + Result { total_ms: f64, summary: String }, + ServerError { error: String }, + Closed { detail: String }, +} + +async fn wait_agent_concurrent_2_result( + stream: &mut R, + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, +) -> AgentConcurrent2Cycle +where + R: futures_util::stream::Stream< + Item = Result, + > + Unpin, +{ + while let Some(incoming) = stream.next().await { + match incoming { + Ok(Message::Text(text)) => { + let data = text.as_str(); + if ctx.args.show_messages { + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} message={}", + prefix, + pad(key, 32), + format_actor(actor_id), + data, + ); + } + let Ok(message) = serde_json::from_str::(data) else { + continue; + }; + let ty = message.get("type").and_then(|v| v.as_str()).unwrap_or(""); + if ty == "agent2_result" { + let total_ms = message + .get("totalMs") + .and_then(|v| v.as_f64()) + .unwrap_or_default(); + let summary = summarize_agent_concurrent_2_result(&message, ctx); + return AgentConcurrent2Cycle::Result { total_ms, summary }; + } + if ty == "agent2_error" { + let mut error = message + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("unknown server error") + .to_string(); + if let Some(stats) = summarize_agent_concurrent_2_query_stats(&message, ctx) { + error.push(' '); + error.push_str(&stats); + } + return AgentConcurrent2Cycle::ServerError { error }; + } + } + Ok(Message::Binary(_) | Message::Ping(_) | Message::Pong(_) | Message::Frame(_)) => {} + Ok(Message::Close(frame)) => { + let detail = match frame { + Some(f) => format!("code={} reason={}", u16::from(f.code), f.reason), + None => "code=0 reason=".to_string(), + }; + return AgentConcurrent2Cycle::Closed { detail }; + } + Err(_) => { + log_websocket_error(ctx, worker, key, actor_id); + return AgentConcurrent2Cycle::Closed { + detail: "websocket error".to_string(), + }; + } + } + } + AgentConcurrent2Cycle::Closed { + detail: "stream ended".to_string(), + } +} + +async fn wait_agent_concurrent_2_sleeping(stream: &mut R) +where + R: futures_util::stream::Stream< + Item = Result, + > + Unpin, +{ + while let Some(incoming) = stream.next().await { + let Ok(Message::Text(text)) = incoming else { + continue; + }; + let Ok(message) = serde_json::from_str::(text.as_str()) else { + continue; + }; + if message.get("type").and_then(|v| v.as_str()) == Some("sleeping") { + return; + } + } +} + +fn summarize_agent_concurrent_2_result( + message: &serde_json::Value, + ctx: &Arc, +) -> String { + let Some(results) = message.get("results").and_then(|v| v.as_array()) else { + return "results=none".to_string(); + }; + let mut parts = results + .iter() + .map(|result| { + let name = result.get("name").and_then(|v| v.as_str()).unwrap_or("?"); + let total = result + .get("totalMs") + .and_then(|v| v.as_f64()) + .unwrap_or_default() + .round() as i64; + format!("{}={}ms", name, total) + }) + .collect::>(); + if let Some(stats) = summarize_agent_concurrent_2_query_stats(message, ctx) { + parts.push(stats); + } + parts.join(" ") +} + +fn summarize_agent_concurrent_2_query_stats( + message: &serde_json::Value, + ctx: &Arc, +) -> Option { + let stats = message.get("stats")?; + let cycle = stats.get("cycle")?; + let wake = stats.get("wake")?; + let actor = stats.get("actor")?; + + let cycle_total = stat_i64(cycle, "total"); + ctx.state + .stats + .agent2_queries + .fetch_add(cycle_total, Ordering::Relaxed); + ctx.state + .stats + .agent2_reads + .fetch_add(stat_i64(cycle, "reads"), Ordering::Relaxed); + ctx.state + .stats + .agent2_mutations + .fetch_add(stat_i64(cycle, "mutations"), Ordering::Relaxed); + ctx.state + .stats + .agent2_tx + .fetch_add(stat_i64(cycle, "tx"), Ordering::Relaxed); + ctx.state + .stats + .agent2_other + .fetch_add(stat_i64(cycle, "other"), Ordering::Relaxed); + ctx.state + .stats + .agent2_rows + .fetch_add(stat_i64(cycle, "rows"), Ordering::Relaxed); + ctx.state + .stats + .agent2_query_errors + .fetch_add(stat_i64(cycle, "errors"), Ordering::Relaxed); + ctx.state + .stats + .agent2_slow_queries + .fetch_add(stat_i64(cycle, "slow"), Ordering::Relaxed); + + let wake_index = stat_i64(stats, "wakeIndex"); + let actor_iteration = stat_i64(stats, "actorIteration"); + let wake_iteration = stat_i64(stats, "wakeIteration"); + let max_step = cycle + .get("maxStep") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let max_ms = stat_i64(cycle, "maxMs"); + let top_tables = top_counter_entries(cycle.get("byTable"), 3); + Some(format!( + "wake={} iter={}/{} cycleQ={} r/m/tx/o={}/{}/{}/{} wakeQ={} actorQ={} rows={} qerr={} qslow={} maxQ={}:{}ms tables={}", + wake_index, + wake_iteration, + actor_iteration, + cycle_total, + stat_i64(cycle, "reads"), + stat_i64(cycle, "mutations"), + stat_i64(cycle, "tx"), + stat_i64(cycle, "other"), + stat_i64(wake, "total"), + stat_i64(actor, "total"), + stat_i64(cycle, "rows"), + stat_i64(cycle, "errors"), + stat_i64(cycle, "slow"), + max_step, + max_ms, + top_tables, + )) +} + +fn stat_i64(value: &serde_json::Value, key: &str) -> i64 { + value.get(key).and_then(|v| v.as_i64()).unwrap_or_default() +} + +fn top_counter_entries(value: Option<&serde_json::Value>, limit: usize) -> String { + let Some(map) = value.and_then(|v| v.as_object()) else { + return "-".to_string(); + }; + let mut entries = map + .iter() + .filter_map(|(key, value)| value.as_i64().map(|count| (key.as_str(), count))) + .collect::>(); + entries.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); + let summary = entries + .into_iter() + .take(limit) + .map(|(key, count)| format!("{}:{}", key, count)) + .collect::>() + .join(","); + if summary.is_empty() { + "-".to_string() + } else { + summary + } +} + +/// Steady scheduler. Spawn N workers up front (with --interval gaps for ramp). Each worker holds +/// its connection forever, reconnects on actor-stopped close. +async fn run_steady( + args: Arc, + ctx: Arc, + workload: Arc, + state: Arc, + options: ConcurrentWorkerOptions, +) { + let mut handles: Vec> = Vec::new(); + for i in 0..args.concurrency { + let id = i + 1; + state.set_workers_started(id as i64); + state.set_worker_health(id, WorkerHealth::Connecting); + let ctx_clone = ctx.clone(); + let workload_clone = workload.clone(); + let options_clone = options.clone(); + let handle = tokio::spawn(async move { + run_concurrent_worker(id, workload_clone, ctx_clone, options_clone, WorkerMode::Steady) + .await; + }); + handles.push(handle); + if i < args.concurrency - 1 { + sleep(std::time::Duration::from_millis(args.interval)).await; + } + } + for h in handles { + let _ = h.await; + } +} + +/// Rolling scheduler. Spawn rate is `--interval` (one worker per `interval` ms) throughout — +/// during ramp AND in steady state. `--concurrency` is a hard cap enforced via a semaphore: if N +/// workers are already in flight when the next spawn tick fires, the spawn blocks until one +/// finishes. Each worker runs one cycle (open → ping1 → ping2 → close) and exits, so the +/// scheduler launches a replacement on the next tick. +/// +/// This intentionally keeps the spawn cadence steady even when workers are short-lived. With +/// `-c 1 -i 1000` it behaves like the old rtt mode (1 spawn/sec, 1 in flight). With +/// `-c 10 -i 100` it's 10 spawns/sec capped at 10 in flight. +async fn run_rolling( + args: Arc, + ctx: Arc, + workload: Arc, + state: Arc, + options: ConcurrentWorkerOptions, +) { + let permits = Arc::new(tokio::sync::Semaphore::new(args.concurrency as usize)); + let mut next_id: u32 = 0; + let mut handles: Vec> = Vec::new(); + let mut next_spawn = Instant::now(); + let interval = std::time::Duration::from_millis(args.interval); + + while !state.stopping() { + // Steady spawn cadence: wait until the next scheduled tick before attempting to acquire + // a permit. We use a fixed wall-clock cadence (`next_spawn += interval`) instead of + // `sleep(interval)` after each spawn so that brief permit-wait stalls don't propagate + // into permanent skew. + let now = Instant::now(); + if next_spawn > now { + tokio::time::sleep_until(next_spawn.into()).await; + } + next_spawn += interval; + + let Ok(permit) = Arc::clone(&permits).acquire_owned().await else { + break; + }; + if state.stopping() { + drop(permit); + break; + } + + next_id += 1; + let id = next_id; + state.set_workers_started(id as i64); + state.set_worker_health(id, WorkerHealth::Connecting); + + let ctx_clone = ctx.clone(); + let workload_clone = workload.clone(); + let options_clone = options.clone(); + let state_clone = state.clone(); + let handle = tokio::spawn(async move { + run_concurrent_worker(id, workload_clone, ctx_clone, options_clone, WorkerMode::Rolling) + .await; + // Drop tracked health entry so the scoreboard counts the next replacement, not a + // growing ledger of completed workers. + state_clone.drop_worker_health(id); + drop(permit); + }); + handles.push(handle); + } + + for h in handles { + let _ = h.await; + } +} + +async fn run_concurrent_worker( + worker: u32, + workload: Arc, + ctx: Arc, + options: ConcurrentWorkerOptions, + worker_mode: WorkerMode, +) { + let key = make_key(worker, workload.key_prefix()); + let mut reconnect = false; + let actor_id: Option = None; + + while !ctx.state.stopping() { + let t0 = Instant::now(); + let url = ctx.endpoint.build_raw_ws_url( + workload.actor_name(), + &key, + options.skip_ready_wait, + ); + + let ws = match open_raw_ws_with_stall_log(&url, &ctx, &key, actor_id.as_deref()).await { + Ok(ws) => ws, + Err(err) => { + let elapsed = t0.elapsed().as_secs_f64() * 1000.0; + log_connect_error( + &ctx, + worker, + &key, + actor_id.as_deref(), + elapsed, + &err.to_string(), + ); + return; + } + }; + let t_connect = Instant::now(); + let connect_ms = t_connect.duration_since(t0).as_secs_f64() * 1000.0; + record_connect(&ctx, worker, reconnect); + let was_reconnect = reconnect; + reconnect = false; + + let (mut sink, mut stream) = ws.split(); + + // Phase 1: ping probe RTT. Send {type:"ping", id:1} → wait for {type:"pong", id:1} → + // send id:2 → wait for id:2. Workload-generated messages that arrive in between are + // dropped here (they have no useful semantics yet — `on_open` hasn't run). + let phase1 = run_ping_phase( + &mut sink, + &mut stream, + t0, + t_connect, + connect_ms, + &ctx, + &key, + actor_id.as_deref(), + ) + .await; + + match &phase1 { + PingOutcome::Completed { + first_ms, + second_ms, + total_ms, + } => { + log_rtt_line( + &ctx, + worker, + &key, + actor_id.as_deref(), + connect_ms, + *first_ms, + *second_ms, + *total_ms, + was_reconnect, + ); + } + PingOutcome::Failed { first_ms, close } => { + log_partial_rtt(&ctx, worker, &key, actor_id.as_deref(), connect_ms, *first_ms, was_reconnect); + if let Some((code, reason)) = close { + let detail = format!("code={} reason={}", code, reason); + log_disconnect(&ctx, worker, &key, actor_id.as_deref(), &detail, true); + } + } + } + + // Rolling mode: we're done after one rtt line. Close and exit so the scheduler can + // launch a replacement. + if matches!(worker_mode, WorkerMode::Rolling) { + let _ = sink + .send(Message::Close(Some(CloseFrame { + code: CloseCode::Normal, + reason: "counter-latency complete".into(), + }))) + .await; + ctx.state.set_worker_health(worker, WorkerHealth::Failed); + break; + } + + // If phase 1 failed: maybe reconnect on actor-stopped close, otherwise exit. + if let PingOutcome::Failed { close, .. } = &phase1 { + if !ctx.state.stopping() + && close + .as_ref() + .map(|(code, reason)| { + *code == ACTOR_STOPPED_CLOSE_CODE + && reason == ACTOR_STOPPED_CLOSE_REASON + }) + .unwrap_or(false) + { + let (code, reason) = close.clone().unwrap(); + log_reconnect(&ctx, worker, &key, actor_id.as_deref(), code, &reason); + reconnect = true; + continue; + } + ctx.state.set_worker_health(worker, WorkerHealth::Failed); + break; + } + + // Phase 2 (steady): start the workload's periodic sender, then run the normal message + // loop. Per-message MESSAGE-GAP / SLOW-SQL detection fires as before; no more rtt + // timing in this phase. + let (send_tx, mut send_rx) = mpsc::channel::(64); + let hooks = + workload.on_open(ctx.clone(), worker, key.clone(), options.clone(), send_tx); + + let mut last_message_at: Option = None; + let mut saw_websocket_error = false; + let mut close_info: Option<(u16, String)> = None; + let mut steady_silence_since = Instant::now(); + // Log a stall warning at most once per silence window for this connection. Resets when + // any inbound message arrives (silence window restarts). + let mut steady_stall_logged = false; + let mut steady_stall_tick = tokio::time::interval(STALL_TICK_INTERVAL); + steady_stall_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + steady_stall_tick.tick().await; // drain immediate tick + + loop { + tokio::select! { + biased; + _ = steady_stall_tick.tick() => { + if !steady_stall_logged { + let elapsed_ms = steady_silence_since.elapsed().as_millis() as u64; + if elapsed_ms >= STALL_WARN_THRESHOLD_MS { + ctx.state.stats.stalls.fetch_add(1, Ordering::Relaxed); + ctx.state.flag_worker_slow(worker); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} {}STALL{} stage=no inbound message in steady phase elapsed_ms={}", + prefix, + pad(&key, 32), + format_actor(actor_id.as_deref()), + YELLOW, + RESET, + elapsed_ms, + ); + steady_stall_logged = true; + } + } + } + maybe = send_rx.recv() => { + match maybe { + Some(payload) => { + if let Err(err) = sink.send(Message::Text(payload.into())).await { + saw_websocket_error = true; + log_websocket_error(&ctx, worker, &key, actor_id.as_deref()); + let _ = err; + break; + } + } + None => {} + } + } + incoming = stream.next() => { + steady_silence_since = Instant::now(); + steady_stall_logged = false; + match incoming { + Some(Ok(Message::Text(text))) => { + let now = Instant::now(); + let data = text.as_str(); + handle_steady_message( + &ctx, + worker, + &key, + actor_id.as_deref(), + &workload, + data, + &mut last_message_at, + now, + options.show_messages, + &hooks, + ); + } + Some(Ok(Message::Binary(bin))) => { + let now = Instant::now(); + handle_steady_binary( + &ctx, + worker, + &key, + actor_id.as_deref(), + &workload, + bin.len(), + &mut last_message_at, + now, + options.show_messages, + ); + } + Some(Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_))) => { + continue; + } + Some(Ok(Message::Close(frame))) => { + let (code, reason) = match frame { + Some(f) => (u16::from(f.code), f.reason.to_string()), + None => (0, String::new()), + }; + close_info = Some((code, reason)); + break; + } + Some(Err(_)) => { + saw_websocket_error = true; + log_websocket_error(&ctx, worker, &key, actor_id.as_deref()); + break; + } + None => break, + } + } + } + } + + // Clean shutdown: try to send a polite close. + let _ = sink + .send(Message::Close(Some(CloseFrame { + code: CloseCode::Normal, + reason: "counter-latency complete".into(), + }))) + .await; + + let (code, reason) = close_info.unwrap_or((0, String::new())); + if !ctx.state.stopping() + && !saw_websocket_error + && code == ACTOR_STOPPED_CLOSE_CODE + && reason == ACTOR_STOPPED_CLOSE_REASON + { + log_reconnect(&ctx, worker, &key, actor_id.as_deref(), code, &reason); + reconnect = true; + } else { + let unclean = !ctx.state.stopping(); + let detail = format!("code={} reason={}", code, reason); + log_disconnect(&ctx, worker, &key, actor_id.as_deref(), &detail, unclean); + } + if saw_websocket_error { + ctx.state.set_worker_health(worker, WorkerHealth::Failed); + } + if !reconnect { + break; + } + } +} + +enum PingOutcome { + Completed { + first_ms: f64, + second_ms: f64, + total_ms: f64, + }, + Failed { + first_ms: Option, + close: Option<(u16, String)>, + }, +} + +/// Phase 1 of every connection: send two pings, wait for pongs, measure round-trip latency. +/// Pong messages are `{"type":"pong","id":N,...}`; anything else inbound is dropped so the rtt +/// numbers are clean. No client-side timeout — the cycle ends only when both pongs arrive or +/// the WS closes / errors. The engine + actor decide when to give up; we just measure what they +/// produce. While waiting on a stage > `STALL_WARN_THRESHOLD`, emit a periodic warning so the +/// operator can see what's blocked without waiting for the eventual close. +async fn run_ping_phase( + sink: &mut S, + stream: &mut R, + t0: Instant, + t_connect: Instant, + _connect_ms: f64, + ctx: &Arc, + key: &str, + actor_id: Option<&str>, +) -> PingOutcome +where + S: SinkExt + Unpin, + R: futures_util::stream::Stream< + Item = Result, + > + Unpin, +{ + const PING1: &str = r#"{"type":"ping","id":1}"#; + const PING2: &str = r#"{"type":"ping","id":2}"#; + + let ping1_send_ts = Instant::now(); + if sink.send(Message::Text(PING1.into())).await.is_err() { + return PingOutcome::Failed { + first_ms: None, + close: None, + }; + } + + let mut first_ms: Option = None; + let mut ping2_send_ts: Option = None; + let mut stage = PingStage::WaitingForPong1; + let mut stage_start = ping1_send_ts; + let mut stage_stall_logged = false; + let mut stall_tick = tokio::time::interval(STALL_TICK_INTERVAL); + stall_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // Tick fires immediately on first poll; drain the immediate one so the first real tick + // fires after STALL_TICK_INTERVAL. + stall_tick.tick().await; + + loop { + tokio::select! { + _ = stall_tick.tick() => { + // Log a stall warning at most once per stage per connection. Resets when the + // stage advances (pong 1 → waiting for pong 2). + if !stage_stall_logged { + let elapsed_ms = stage_start.elapsed().as_millis() as u64; + if elapsed_ms >= STALL_WARN_THRESHOLD_MS { + log_phase1_stall(ctx, key, actor_id, stage, elapsed_ms); + stage_stall_logged = true; + } + } + continue; + } + incoming = stream.next() => { + match incoming { + Some(Ok(Message::Text(text))) => { + match parse_pong_id(text.as_str()) { + Some(1) => { + let now = Instant::now(); + first_ms = Some(now.duration_since(ping1_send_ts).as_secs_f64() * 1000.0); + let ts = Instant::now(); + if sink.send(Message::Text(PING2.into())).await.is_err() { + return PingOutcome::Failed { first_ms, close: None }; + } + ping2_send_ts = Some(ts); + stage = PingStage::WaitingForPong2; + stage_start = ts; + stage_stall_logged = false; + } + Some(2) => { + let Some(send_ts) = ping2_send_ts else { + // We got pong id=2 before we sent ping id=2 — protocol bug; treat as failed. + return PingOutcome::Failed { first_ms, close: None }; + }; + let now = Instant::now(); + let second_ms = now.duration_since(send_ts).as_secs_f64() * 1000.0; + let total_ms = now.duration_since(t0).as_secs_f64() * 1000.0; + let _ = t_connect; + return PingOutcome::Completed { + first_ms: first_ms.unwrap_or(0.0), + second_ms, + total_ms, + }; + } + _ => { + // Some other workload-generated text frame. Drop it during phase 1. + } + } + } + Some(Ok(Message::Binary(_) | Message::Ping(_) | Message::Pong(_) | Message::Frame(_))) => { + // Drop binary / control frames during phase 1. + } + Some(Ok(Message::Close(frame))) => { + let close = frame.map(|f| (u16::from(f.code), f.reason.to_string())); + return PingOutcome::Failed { first_ms, close }; + } + Some(Err(_)) | None => { + return PingOutcome::Failed { first_ms, close: None }; + } + } + } + } + } +} + +/// Stall warning threshold. Fires at most once per stage per connection (ws handshake, ping +/// stage, steady-mode silence window). After it fires, no further stall logs for that same +/// stage on the same connection until the stage advances or the connection cycles. +const STALL_WARN_THRESHOLD_MS: u64 = 2_000; +const STALL_TICK_INTERVAL: std::time::Duration = std::time::Duration::from_millis(500); + +#[derive(Copy, Clone)] +enum PingStage { + WaitingForPong1, + WaitingForPong2, +} + +impl PingStage { + fn label(self) -> &'static str { + match self { + PingStage::WaitingForPong1 => "waiting for pong id=1", + PingStage::WaitingForPong2 => "waiting for pong id=2", + } + } +} + +fn log_phase1_stall( + ctx: &Arc, + key: &str, + actor_id: Option<&str>, + stage: PingStage, + elapsed_ms: u64, +) { + ctx.state + .stats + .stalls + .fetch_add(1, Ordering::Relaxed); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} {}STALL{} stage={} elapsed_ms={}", + prefix, + pad(key, 32), + format_actor(actor_id), + YELLOW, + RESET, + stage.label(), + elapsed_ms, + ); +} + +/// Parse `{"type":"pong","id":N,...}`. Returns Some(N) if the frame is a pong, None otherwise. +/// Tolerant of unrelated fields and surrounding workload messages. +fn parse_pong_id(text: &str) -> Option { + let parsed: serde_json::Value = serde_json::from_str(text).ok()?; + if parsed.get("type")?.as_str()? != "pong" { + return None; + } + parsed.get("id")?.as_u64() +} + +/// Wraps `open_raw_ws` with a periodic stall warning while the WS handshake is still pending. +/// The connect future itself has no client-side timeout — we just observe and log how long it's +/// taking. Returns the same Result the underlying open returns. +async fn open_raw_ws_with_stall_log( + url: &str, + ctx: &Arc, + key: &str, + actor_id: Option<&str>, +) -> anyhow::Result { + let connect_started = Instant::now(); + // At most one stall log per handshake attempt. + let mut stall_logged = false; + let mut stall_tick = tokio::time::interval(STALL_TICK_INTERVAL); + stall_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + stall_tick.tick().await; // drain immediate tick + + let connect_future = open_raw_ws(url); + tokio::pin!(connect_future); + + loop { + tokio::select! { + res = &mut connect_future => { + return res; + } + _ = stall_tick.tick() => { + if !stall_logged { + let elapsed_ms = connect_started.elapsed().as_millis() as u64; + if elapsed_ms >= STALL_WARN_THRESHOLD_MS { + ctx.state.stats.stalls.fetch_add(1, Ordering::Relaxed); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} {}STALL{} stage=waiting for ws handshake elapsed_ms={}", + prefix, + pad(key, 32), + format_actor(actor_id), + YELLOW, + RESET, + elapsed_ms, + ); + stall_logged = true; + } + } + } + } + } +} + +#[allow(clippy::too_many_arguments)] +fn handle_steady_message( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + workload: &Arc, + data: &str, + last_message_at: &mut Option, + now: Instant, + show_messages: bool, + hooks: &WorkloadHooks, +) { + if !workload.suppress_generic_gap() { + if let Some(prev) = *last_message_at { + let gap_ms = now.duration_since(prev).as_secs_f64() * 1000.0; + if gap_ms > MESSAGE_GAP_WARN_MS { + log_message_gap(ctx, worker, key, actor_id, gap_ms); + } + } + } + *last_message_at = Some(now); + + if show_messages { + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} message={}", + prefix, + pad(key, 32), + format_actor(actor_id), + data, + ); + } + + if let Some(handler) = &hooks.on_message { + handler(data); + } +} + +#[allow(clippy::too_many_arguments)] +fn handle_steady_binary( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + workload: &Arc, + bytes_len: usize, + last_message_at: &mut Option, + now: Instant, + show_messages: bool, +) { + if !workload.suppress_generic_gap() { + if let Some(prev) = *last_message_at { + let gap_ms = now.duration_since(prev).as_secs_f64() * 1000.0; + if gap_ms > MESSAGE_GAP_WARN_MS { + log_message_gap(ctx, worker, key, actor_id, gap_ms); + } + } + } + *last_message_at = Some(now); + + if show_messages { + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} message=", + prefix, + pad(key, 32), + format_actor(actor_id), + bytes_len, + ); + } +} + +/// Bump the connect/reconnect counters and mark the worker healthy. The rtt line itself is +/// deferred until the second inbound message arrives (combined `connect=… first=… second=… +/// total=…`). If the connection closes before that, `log_partial_rtt` emits whatever fragments +/// we have. +fn record_connect(ctx: &Arc, worker: u32, reconnect: bool) { + ctx.state.stats.connects.fetch_add(1, Ordering::Relaxed); + if reconnect { + ctx.state.stats.reconnects.fetch_add(1, Ordering::Relaxed); + } + ctx.state.set_worker_health(worker, WorkerHealth::Pinging); +} + +/// rtt-shape line. Same format as the old `rtt` subcommand for parity: +/// `connect=… first=… second=… total=…`. Also promotes the worker from `Pinging` → `Connected` +/// since phase 1 is complete by definition when this line emits. +#[allow(clippy::too_many_arguments)] +fn log_rtt_line( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + connect_ms: f64, + first_ms: f64, + second_ms: f64, + total_ms: f64, + reconnect: bool, +) { + ctx.state.stats.first_messages.fetch_add(1, Ordering::Relaxed); + ctx.state.set_worker_health(worker, WorkerHealth::Connected); + let prefix = ctx.log_prefix(); + let label = if reconnect { "reconnect" } else { "connect" }; + crate::out!( + "{} {}{} {}={} first={} second={} total={}", + prefix, + pad(key, 32), + format_actor(actor_id), + label, + color_ms(connect_ms), + color_ms(first_ms), + color_ms(second_ms), + color_ms(total_ms), + ); +} + +/// Fallback for connections that close before delivering enough messages to compute the full +/// rtt line. Emits whatever fragments are available so the line is still visible. +/// +/// Also downgrades the worker's health from Healthy → Ended at this point so the scoreboard +/// reflects the failure immediately instead of waiting for the subsequent DISCONNECT log to +/// flip the count. +fn log_partial_rtt( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + connect_ms: f64, + first_ms: Option, + reconnect: bool, +) { + ctx.state.set_worker_health(worker, WorkerHealth::Failed); + let prefix = ctx.log_prefix(); + let label = if reconnect { "reconnect" } else { "connect" }; + let first_part = first_ms + .map(|ms| format!(" first={}", color_ms(ms))) + .unwrap_or_else(|| " first=none".to_string()); + crate::out!( + "{} {}{} {}FAILED{} {}={}{} second=none total=none", + prefix, + pad(key, 32), + format_actor(actor_id), + RED, + RESET, + label, + color_ms(connect_ms), + first_part, + ); +} + +fn log_disconnect( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + reason: &str, + unclean: bool, +) { + ctx.state.stats.disconnects.fetch_add(1, Ordering::Relaxed); + if unclean { + ctx.state.stats.unclean_failures_or_disconnects.fetch_add(1, Ordering::Relaxed); + } + ctx.state.set_worker_health(worker, WorkerHealth::Failed); + let prefix = ctx.log_prefix(); + let (label_prefix, label) = if unclean { + (RED, "DISCONNECT") + } else { + (DIM, "disconnect") + }; + crate::out!( + "{} {}{} {}{} {}{}", + prefix, + pad(key, 32), + format_actor(actor_id), + label_prefix, + label, + reason, + RESET, + ); +} + +fn log_reconnect( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + code: u16, + reason: &str, +) { + ctx.state.set_worker_health(worker, WorkerHealth::Connecting); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} actor-stopped reconnect code={} reason={}", + prefix, + pad(key, 32), + format_actor(actor_id), + code, + reason, + ); +} + +fn log_message_gap( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + gap_ms: f64, +) { + ctx.state.stats.message_gaps.fetch_add(1, Ordering::Relaxed); + ctx.state.stats.unclean_failures_or_disconnects.fetch_add(1, Ordering::Relaxed); + ctx.state.flag_worker_slow(worker); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} {}MESSAGE-GAP {}{}", + prefix, + pad(key, 32), + format_actor(actor_id), + RED, + color_ms(gap_ms), + RESET, + ); +} + +fn log_slow_sql( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + elapsed_ms: f64, + detail: &str, +) { + ctx.state.stats.slow_sql.fetch_add(1, Ordering::Relaxed); + ctx.state.flag_worker_slow(worker); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} {}SLOW-SQL {} {}{}", + prefix, + pad(key, 32), + format_actor(actor_id), + YELLOW, + color_ms(elapsed_ms), + detail, + RESET, + ); +} + +fn log_connect_error( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + elapsed_ms: f64, + reason: &str, +) { + ctx.state.stats.connect_errors.fetch_add(1, Ordering::Relaxed); + ctx.state.stats.unclean_failures_or_disconnects.fetch_add(1, Ordering::Relaxed); + ctx.state.set_worker_health(worker, WorkerHealth::Failed); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} {}CONNECT-ERROR {}{} ({})", + prefix, + pad(key, 32), + format_actor(actor_id), + RED, + reason, + RESET, + color_ms(elapsed_ms), + ); +} + +fn log_websocket_error( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, +) { + ctx.state.stats.websocket_errors.fetch_add(1, Ordering::Relaxed); + ctx.state.stats.unclean_failures_or_disconnects.fetch_add(1, Ordering::Relaxed); + ctx.state.flag_worker_slow(worker); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} {}WEBSOCKET-ERROR{}", + prefix, + pad(key, 32), + format_actor(actor_id), + RED, + RESET, + ); +} + +fn log_agent_concurrent_2_result( + ctx: &Arc, + _worker: u32, + key: &str, + actor_id: Option<&str>, + total_ms: f64, + summary: &str, +) { + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} agent-concurrent-2 total={} {}", + prefix, + pad(key, 32), + format_actor(actor_id), + color_ms(total_ms), + summary, + ); +} + +fn log_agent_concurrent_2_error( + ctx: &Arc, + worker: u32, + key: &str, + actor_id: Option<&str>, + error: &str, +) { + ctx.state.stats.slow_sql.fetch_add(1, Ordering::Relaxed); + ctx.state.stats.unclean_failures_or_disconnects.fetch_add(1, Ordering::Relaxed); + ctx.state.set_worker_health(worker, WorkerHealth::Failed); + let prefix = ctx.log_prefix(); + crate::out!( + "{} {}{} {}AGENT-CONCURRENT-2-ERROR {}{}", + prefix, + pad(key, 32), + format_actor(actor_id), + RED, + error, + RESET, + ); +} + +pub fn print_concurrent_summary(ctx: &Arc, reason: &str) { + let counts = ctx.state.count_worker_health(); + crate::out!( + "{}counter-latency summary{} reason={} workers={} [{}{}{}/{}{}{}/{}{}{}/{}{}{}/{}{}{}] disconnects={} connect-errors={} websocket-errors={} message-gaps={} slow-sql={} stalls={} connects={} reconnects={} first-messages={} agent2-q={} agent2-r/m/tx/o={}/{}/{}/{} agent2-rows={} agent2-qerr={} agent2-qslow={}", + BOLD, + RESET, + reason, + ctx.state.workers_started(), + BLUE, counts.connecting, RESET, + CYAN, counts.pinging, RESET, + GREEN, counts.connected, RESET, + YELLOW, counts.connected_slow, RESET, + RED, counts.failed, RESET, + ctx.state.stats.disconnects.load(Ordering::Relaxed), + ctx.state.stats.connect_errors.load(Ordering::Relaxed), + ctx.state.stats.websocket_errors.load(Ordering::Relaxed), + ctx.state.stats.message_gaps.load(Ordering::Relaxed), + ctx.state.stats.slow_sql.load(Ordering::Relaxed), + ctx.state.stats.stalls.load(Ordering::Relaxed), + ctx.state.stats.connects.load(Ordering::Relaxed), + ctx.state.stats.reconnects.load(Ordering::Relaxed), + ctx.state.stats.first_messages.load(Ordering::Relaxed), + ctx.state.stats.agent2_queries.load(Ordering::Relaxed), + ctx.state.stats.agent2_reads.load(Ordering::Relaxed), + ctx.state.stats.agent2_mutations.load(Ordering::Relaxed), + ctx.state.stats.agent2_tx.load(Ordering::Relaxed), + ctx.state.stats.agent2_other.load(Ordering::Relaxed), + ctx.state.stats.agent2_rows.load(Ordering::Relaxed), + ctx.state.stats.agent2_query_errors.load(Ordering::Relaxed), + ctx.state.stats.agent2_slow_queries.load(Ordering::Relaxed), + ); + if let Some(path) = crate::tee::log_file_path() { + crate::out!("counter-latency log: {}", path); + } +} diff --git a/examples/kitchen-sink/scripts/counter-latency/src/endpoint.rs b/examples/kitchen-sink/scripts/counter-latency/src/endpoint.rs new file mode 100644 index 0000000000..0184be1faf --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/src/endpoint.rs @@ -0,0 +1,122 @@ +// Endpoint URL parsing + raw WebSocket URL builder. Mirrors the top-level +// constants `NAMESPACE`, `TOKEN`, `WS_ORIGIN`, `RVT_RUNNER` and +// `buildRawWebSocketUrl` in scripts/counter-latency.ts. + +use anyhow::{Context, Result, anyhow}; +use url::Url; + +pub struct Endpoint { + pub namespace: String, + pub token: String, + pub ws_origin: String, + pub display_origin: String, + pub rvt_runner: String, +} + +impl Endpoint { + pub fn parse(raw_endpoint: &str, rivet_pool: String) -> Result { + let url = Url::parse(raw_endpoint).context("invalid endpoint URL")?; + let namespace = percent_decode(url.username())?; + let token = match url.password() { + Some(p) => percent_decode(p)?, + None => String::new(), + }; + let ws_proto = if url.scheme() == "https" { "wss" } else { "ws" }; + let host = url + .host_str() + .ok_or_else(|| anyhow!("endpoint missing host"))?; + let port = url.port(); + let host_with_port = match port { + Some(p) => format!("{}:{}", host, p), + None => host.to_string(), + }; + let ws_origin = format!("{}://{}", ws_proto, host_with_port); + let display_origin = format!("{}://{}", url.scheme(), host_with_port); + Ok(Self { + namespace, + token, + ws_origin, + display_origin, + rvt_runner: rivet_pool, + }) + } + + pub fn build_raw_ws_url(&self, actor_name: &str, key: &str, skip_ready_wait: bool) -> String { + let mut params = Vec::<(String, String)>::new(); + params.push(("rvt-namespace".into(), self.namespace.clone())); + params.push(("rvt-method".into(), "getOrCreate".into())); + params.push(("rvt-runner".into(), self.rvt_runner.clone())); + params.push(("rvt-key".into(), key.into())); + params.push(("rvt-crash-policy".into(), "sleep".into())); + if !self.token.is_empty() { + params.push(("rvt-token".into(), self.token.clone())); + } + if skip_ready_wait { + params.push(("rvt-skip-ready-wait".into(), "true".into())); + } + let qs = params + .iter() + .map(|(k, v)| format!("{}={}", encode_query(k), encode_query(v))) + .collect::>() + .join("&"); + format!( + "{}/gateway/{}/websocket?{}", + self.ws_origin, + encode_path(actor_name), + qs, + ) + } +} + +fn percent_decode(s: &str) -> Result { + let decoded = urlencoding_decode(s)?; + Ok(decoded) +} + +// Minimal URL-encoding helpers using percent-encoding semantics compatible +// with `encodeURIComponent` for the characters we care about +// (alphanumerics + `-._~` left as-is; everything else percent-encoded). +fn encode_query(s: &str) -> String { + encode_uri_component(s) +} + +fn encode_path(s: &str) -> String { + encode_uri_component(s) +} + +fn encode_uri_component(s: &str) -> String { + let mut buf = String::with_capacity(s.len()); + for b in s.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~') { + buf.push(b as char); + } else { + buf.push_str(&format!("%{:02X}", b)); + } + } + buf +} + +fn urlencoding_decode(s: &str) -> Result { + let bytes = s.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'%' if i + 2 < bytes.len() => { + let hex = std::str::from_utf8(&bytes[i + 1..i + 3])?; + let v = u8::from_str_radix(hex, 16).context("invalid percent-encoding")?; + out.push(v); + i += 3; + } + b'+' => { + out.push(b' '); + i += 1; + } + b => { + out.push(b); + i += 1; + } + } + } + Ok(String::from_utf8(out).context("invalid UTF-8 after decode")?) +} diff --git a/examples/kitchen-sink/scripts/counter-latency/src/log.rs b/examples/kitchen-sink/scripts/counter-latency/src/log.rs new file mode 100644 index 0000000000..2a12152008 --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/src/log.rs @@ -0,0 +1,60 @@ +// ANSI helpers, gradient color, log formatting helpers. 1:1 port of the +// console.log layer in scripts/counter-latency.ts. + +use chrono::Utc; + +pub const RESET: &str = "\x1b[0m"; +pub const GREEN: &str = "\x1b[38;2;0;255;0m"; +pub const RED: &str = "\x1b[38;2;255;0;0m"; +pub const YELLOW: &str = "\x1b[38;2;255;200;0m"; +pub const BLUE: &str = "\x1b[38;2;80;160;255m"; +pub const CYAN: &str = "\x1b[38;2;0;200;220m"; +pub const DIM: &str = "\x1b[2m"; +pub const BOLD: &str = "\x1b[1m"; + +pub const COLOR_MIN_MS: f64 = 800.0; +pub const COLOR_MAX_MS: f64 = 2_000.0; + +pub fn gradient_color(ms: f64) -> String { + let clamped = ms.clamp(COLOR_MIN_MS, COLOR_MAX_MS); + let t = (clamped - COLOR_MIN_MS) / (COLOR_MAX_MS - COLOR_MIN_MS); + let r; + let g; + if t <= 0.5 { + r = (t * 2.0 * 255.0).round() as u32; + g = 255u32; + } else { + r = 255u32; + g = ((1.0 - (t - 0.5) * 2.0) * 255.0).round() as u32; + } + format!("\x1b[38;2;{};{};0m", r, g) +} + +pub fn color_ms(ms: f64) -> String { + let fixed = format!("{:>5}", ms.round() as i64); + format!("{}{}ms{}", gradient_color(ms), fixed, RESET) +} + +pub fn pad(s: &str, n: usize) -> String { + if s.len() >= n { + s.to_string() + } else { + let mut buf = String::with_capacity(n); + buf.push_str(s); + for _ in s.len()..n { + buf.push(' '); + } + buf + } +} + +pub fn format_actor(actor_id: Option<&str>) -> String { + match actor_id { + Some(id) if !id.is_empty() => format!(" actor={}", id), + _ => String::new(), + } +} + +pub fn iso_now() -> String { + Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string() +} diff --git a/examples/kitchen-sink/scripts/counter-latency/src/main.rs b/examples/kitchen-sink/scripts/counter-latency/src/main.rs new file mode 100644 index 0000000000..9893c15426 --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/src/main.rs @@ -0,0 +1,179 @@ +// counter-latency: Rust port of scripts/counter-latency.ts. +// Subcommands: +// concurrent ramp raw WS tunnel-stress actors (steady or rolling). +// agent-concurrent ramp SQLite-backed agent actors (steady or rolling). +// agent-concurrent-2 cycle SQLite-backed agent actors through work and sleep. +// +// Env: +// RUN_FOR_MS optional run cap for both modes. +// RIVET_POOL runner pool name (default "k8s"). + +mod args; +mod concurrent; +mod endpoint; +mod log; +mod stats; +mod tee; +mod ws; + +use std::sync::Arc; + +use crate::args::{Args, EnvConfig}; +use crate::concurrent::{WorkloadCtx, print_concurrent_summary}; +use crate::endpoint::Endpoint; +use crate::log::{BOLD, COLOR_MIN_MS, COLOR_MAX_MS, DIM, RESET, gradient_color}; +use crate::stats::State; + +fn main() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("tokio runtime"); + runtime.block_on(run()); +} + +async fn run() { + let parsed = args::parse_cli(); + let env_cfg = Arc::new(EnvConfig::from_env()); + + let run_id = format!( + "{}-{}", + chrono::Utc::now().format("%Y%m%dT%H%M%S"), + std::process::id(), + ); + match tee::init(&run_id) { + Ok(path) => eprintln!("counter-latency log: {}", path), + Err(err) => { + eprintln!("fatal: cannot open log file: {}", err); + std::process::exit(1); + } + } + + let endpoint = match Endpoint::parse(&env_cfg.endpoint, env_cfg.rivet_pool.clone()) { + Ok(e) => Arc::new(e), + Err(err) => { + eprintln!("fatal: {}", err); + std::process::exit(1); + } + }; + print_header(&parsed, &env_cfg, &endpoint); + + match parsed { + Args::Concurrent(concurrent_args) => { + let state = Arc::new(State::new()); + let ctx = Arc::new(WorkloadCtx { + endpoint: endpoint.clone(), + args: Arc::new(concurrent_args.clone()), + env: env_cfg.clone(), + state: state.clone(), + }); + install_signal_handlers(ctx.clone()); + concurrent::run_concurrent_mode( + concurrent_args, + env_cfg.clone(), + endpoint.clone(), + state.clone(), + ) + .await; + } + } +} + +fn install_signal_handlers(ctx: Arc) { + let ctx_int = ctx.clone(); + tokio::spawn(async move { + if tokio::signal::ctrl_c().await.is_ok() { + ctx_int.state.set_stopping(); + print_concurrent_summary(&ctx_int, "sigint"); + std::process::exit(130); + } + }); + + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let ctx_term = ctx.clone(); + tokio::spawn(async move { + let Ok(mut sig) = signal(SignalKind::terminate()) else { + return; + }; + if sig.recv().await.is_some() { + ctx_term.state.set_stopping(); + print_concurrent_summary(&ctx_term, "sigterm"); + std::process::exit(143); + } + }); + } +} + +fn print_header(args: &Args, env: &EnvConfig, endpoint: &Endpoint) { + let mode = match args { + Args::Concurrent(a) => match a.mode { + args::ConcurrentMode::Concurrent => "concurrent", + args::ConcurrentMode::AgentConcurrent => "agent-concurrent", + args::ConcurrentMode::AgentConcurrent2 => "agent-concurrent-2", + }, + }; + let header = format!( + "{}counter-latency{} endpoint={} ns={} mode={} interval={}ms", + BOLD, + RESET, + endpoint.display_origin, + endpoint.namespace, + mode, + args.interval(), + ); + match args { + Args::Concurrent(a) => { + let worker_mode = match a.worker_mode { + args::WorkerMode::Steady => "steady", + args::WorkerMode::Rolling => "rolling", + }; + let agent_part = match a.mode { + args::ConcurrentMode::AgentConcurrent => format!( + " tokens-per-second={} duration-ms={}", + a.tokens_per_second, a.duration_ms, + ), + args::ConcurrentMode::AgentConcurrent2 => format!( + " sleep-ms={} timeout-ms={} stagger-handle-ms={} query-multiplier={}", + a.sleep_ms, a.timeout_ms, a.stagger_handle_ms, a.query_multiplier, + ), + args::ConcurrentMode::Concurrent => String::new(), + }; + let run_for_part = if env.run_for_ms > 0 { + format!(" run-for-ms={}", env.run_for_ms) + } else { + String::new() + }; + out!( + "{} worker-mode={} concurrency={} message-every={}ms show-messages={} skip-ready-wait={} rvt-runner={}{}{}", + header, + worker_mode, + a.concurrency, + a.message_interval, + a.show_messages, + a.skip_ready_wait, + env.rivet_pool, + agent_part, + run_for_part, + ); + } + } + let mid = (COLOR_MIN_MS + COLOR_MAX_MS) / 2.0; + out!( + "{}gradient: {}{}ms{}{} -> {}{}ms{}{} -> {}{}ms{}", + DIM, + gradient_color(COLOR_MIN_MS), + COLOR_MIN_MS as i64, + RESET, + DIM, + gradient_color(mid), + mid as i64, + RESET, + DIM, + gradient_color(COLOR_MAX_MS), + COLOR_MAX_MS as i64, + RESET, + ); + out!(); +} diff --git a/examples/kitchen-sink/scripts/counter-latency/src/stats.rs b/examples/kitchen-sink/scripts/counter-latency/src/stats.rs new file mode 100644 index 0000000000..9b2128f719 --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/src/stats.rs @@ -0,0 +1,151 @@ +// Worker health + concurrent stats counters. Mirrors the global state at +// the top of scripts/counter-latency.ts. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum WorkerHealth { + /// WS handshake in progress. + Connecting, + /// WS open; ping phase in progress (waiting for pong id=1 or id=2). + Pinging, + /// Ping phase completed cleanly; worker is in steady state or has emitted its rolling rtt + /// line. + Connected, + /// Worker was Connected but has been flagged slow — STALL fired, MESSAGE-GAP fired, or + /// SLOW-SQL fired. Stays in this state for the remainder of the cycle. + ConnectedSlow, + /// Phase 1 failed, WS errored, or disconnect logged. + Failed, +} + +pub struct ConcurrentStats { + pub connects: AtomicI64, + pub reconnects: AtomicI64, + pub first_messages: AtomicI64, + pub connect_errors: AtomicI64, + pub websocket_errors: AtomicI64, + pub disconnects: AtomicI64, + pub message_gaps: AtomicI64, + pub slow_sql: AtomicI64, + pub stalls: AtomicI64, + pub unclean_failures_or_disconnects: AtomicI64, + pub agent2_queries: AtomicI64, + pub agent2_reads: AtomicI64, + pub agent2_mutations: AtomicI64, + pub agent2_tx: AtomicI64, + pub agent2_other: AtomicI64, + pub agent2_rows: AtomicI64, + pub agent2_query_errors: AtomicI64, + pub agent2_slow_queries: AtomicI64, +} + +impl ConcurrentStats { + pub fn new() -> Self { + Self { + connects: AtomicI64::new(0), + reconnects: AtomicI64::new(0), + first_messages: AtomicI64::new(0), + connect_errors: AtomicI64::new(0), + websocket_errors: AtomicI64::new(0), + disconnects: AtomicI64::new(0), + message_gaps: AtomicI64::new(0), + slow_sql: AtomicI64::new(0), + stalls: AtomicI64::new(0), + unclean_failures_or_disconnects: AtomicI64::new(0), + agent2_queries: AtomicI64::new(0), + agent2_reads: AtomicI64::new(0), + agent2_mutations: AtomicI64::new(0), + agent2_tx: AtomicI64::new(0), + agent2_other: AtomicI64::new(0), + agent2_rows: AtomicI64::new(0), + agent2_query_errors: AtomicI64::new(0), + agent2_slow_queries: AtomicI64::new(0), + } + } +} + +pub struct State { + pub stats: ConcurrentStats, + pub workers_started: AtomicI64, + pub stopping: AtomicBool, + pub worker_health: Mutex>, +} + +/// Per-state counts returned by `count_worker_health`, in the same order as the scoreboard column +/// labels. +pub struct HealthCounts { + pub connecting: i64, + pub pinging: i64, + pub connected: i64, + pub connected_slow: i64, + pub failed: i64, +} + +impl State { + pub fn new() -> Self { + Self { + stats: ConcurrentStats::new(), + workers_started: AtomicI64::new(0), + stopping: AtomicBool::new(false), + worker_health: Mutex::new(HashMap::new()), + } + } + + pub fn set_worker_health(&self, worker: u32, state: WorkerHealth) { + self.worker_health.lock().unwrap().insert(worker, state); + } + + pub fn drop_worker_health(&self, worker: u32) { + self.worker_health.lock().unwrap().remove(&worker); + } + + /// Promote a worker from `Connected` to `ConnectedSlow`. No-op if the worker is in any other + /// state (we don't want to mark a failed worker as slow, nor downgrade a worker that's still + /// in the ping phase). + pub fn flag_worker_slow(&self, worker: u32) { + let mut map = self.worker_health.lock().unwrap(); + if let Some(WorkerHealth::Connected) = map.get(&worker) { + map.insert(worker, WorkerHealth::ConnectedSlow); + } + } + + pub fn count_worker_health(&self) -> HealthCounts { + let map = self.worker_health.lock().unwrap(); + let mut counts = HealthCounts { + connecting: 0, + pinging: 0, + connected: 0, + connected_slow: 0, + failed: 0, + }; + for s in map.values() { + match s { + WorkerHealth::Connecting => counts.connecting += 1, + WorkerHealth::Pinging => counts.pinging += 1, + WorkerHealth::Connected => counts.connected += 1, + WorkerHealth::ConnectedSlow => counts.connected_slow += 1, + WorkerHealth::Failed => counts.failed += 1, + } + } + counts + } + + pub fn workers_started(&self) -> i64 { + self.workers_started.load(Ordering::Relaxed) + } + + pub fn set_workers_started(&self, n: i64) { + self.workers_started.store(n, Ordering::Relaxed); + } + + pub fn stopping(&self) -> bool { + self.stopping.load(Ordering::Relaxed) + } + + pub fn set_stopping(&self) { + self.stopping.store(true, Ordering::Relaxed); + } +} diff --git a/examples/kitchen-sink/scripts/counter-latency/src/tee.rs b/examples/kitchen-sink/scripts/counter-latency/src/tee.rs new file mode 100644 index 0000000000..d12c47c274 --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/src/tee.rs @@ -0,0 +1,47 @@ +// Mirror every stdout line written via `out!` to a per-run /tmp/counter-latency-.txt +// transcript. Initialized once at startup; all log helpers route through `out!`. + +use std::fs::File; +use std::io::{Write, stdout}; +use std::sync::{Mutex, OnceLock}; + +static LOG_FILE: OnceLock> = OnceLock::new(); +static LOG_FILE_PATH: OnceLock = OnceLock::new(); + +pub fn init(id: &str) -> std::io::Result { + let path = format!("/tmp/counter-latency-{}.txt", id); + let file = File::create(&path)?; + LOG_FILE + .set(Mutex::new(file)) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::AlreadyExists, "log file already initialized"))?; + LOG_FILE_PATH + .set(path.clone()) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::AlreadyExists, "log path already set"))?; + Ok(path) +} + +pub fn log_file_path() -> Option<&'static str> { + LOG_FILE_PATH.get().map(|s| s.as_str()) +} + +pub fn emit(line: &str) { + { + let mut out = stdout().lock(); + let _ = writeln!(out, "{}", line); + } + if let Some(file_mu) = LOG_FILE.get() { + if let Ok(mut f) = file_mu.lock() { + let _ = writeln!(f, "{}", line); + } + } +} + +#[macro_export] +macro_rules! out { + () => { + $crate::tee::emit(""); + }; + ($($arg:tt)*) => { + $crate::tee::emit(&format!($($arg)*)); + }; +} diff --git a/examples/kitchen-sink/scripts/counter-latency/src/ws.rs b/examples/kitchen-sink/scripts/counter-latency/src/ws.rs new file mode 100644 index 0000000000..f6b90b481a --- /dev/null +++ b/examples/kitchen-sink/scripts/counter-latency/src/ws.rs @@ -0,0 +1,23 @@ +// WebSocket connect helper for raw rivet gateway routing. Wraps +// tokio-tungstenite and sets the protocol subheaders the gateway +// expects (`rivet`, `rivet_encoding.json`). + +use anyhow::{Context, Result}; +use http::HeaderValue; +use tokio::net::TcpStream; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; + +pub const RIVET_PROTOCOLS: &[&str] = &["rivet", "rivet_encoding.json"]; + +pub type Ws = WebSocketStream>; + +pub async fn open_raw_ws(url: &str) -> Result { + let mut req = url.into_client_request().context("invalid websocket URL")?; + req.headers_mut().insert( + "Sec-WebSocket-Protocol", + HeaderValue::from_static("rivet, rivet_encoding.json"), + ); + let (ws, _resp) = connect_async(req).await.context("websocket connect failed")?; + Ok(ws) +} diff --git a/examples/kitchen-sink/scripts/deploy-cloud-run.sh b/examples/kitchen-sink/scripts/deploy-cloud-run.sh new file mode 100755 index 0000000000..f4cd384ab9 --- /dev/null +++ b/examples/kitchen-sink/scripts/deploy-cloud-run.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Manually deploy the kitchen-sink (built from the local workspace) to the +# `kitchen-sink-staging` and `rivet-kitchen-sink` Cloud Run services in +# dev-projects-491221 / us-east4. +# +# This is the "current workspace" path. For deploying a *published* rivetkit +# preview build instead (temp-copy + pinned versions), see the second flow in +# examples/kitchen-sink/CLAUDE.md. +# +# Prereqs: +# - docker +# - gcloud (authenticated to nathan@rivet.gg or any account with +# run.developer on dev-projects-491221 and artifactregistry.writer on the +# us-east4 cloud-run-source-deploy repo) +# - jj (for tag derivation; falls back to `git rev-parse --short HEAD` if +# jj is not installed) +# - rivetkit-typescript/packages/rivetkit-napi/rivetkit-napi.linux-x64-gnu.node +# already built. Build it with: +# cd rivetkit-typescript/packages/rivetkit-napi && pnpm build:release +# +# Usage: +# examples/kitchen-sink/scripts/deploy-cloud-run.sh [--only staging|prod] +# +# Defaults to deploying to BOTH services. Pass --only staging to deploy only +# kitchen-sink-staging, or --only prod for only rivet-kitchen-sink. + +set -euo pipefail + +REPO_ROOT=$(cd "$(dirname "$0")/../../.." && pwd) +cd "$REPO_ROOT" + +ONLY="" +if [[ "${1:-}" == "--only" ]]; then + ONLY="${2:-}" + if [[ "$ONLY" != "staging" && "$ONLY" != "prod" ]]; then + echo "error: --only must be staging or prod" >&2 + exit 1 + fi +fi + +PROJECT=dev-projects-491221 +REGION=us-east4 +REGISTRY="${REGION}-docker.pkg.dev/${PROJECT}/cloud-run-source-deploy/rivet-dev-rivet/rivet-kitchen-sink" + +NAPI_SRC="rivetkit-typescript/packages/rivetkit-napi/rivetkit-napi.linux-x64-gnu.node" +NAPI_DST="examples/kitchen-sink/rivetkit-napi.linux-x64-gnu.node" + +if [[ ! -f "$NAPI_SRC" ]]; then + echo "error: $NAPI_SRC is missing." >&2 + echo "Build it first: cd rivetkit-typescript/packages/rivetkit-napi && pnpm build:release" >&2 + exit 1 +fi + +if command -v jj >/dev/null 2>&1; then + SHA=$(jj log -r @ --no-graph -T 'commit_id.short()') +else + SHA=$(git rev-parse --short HEAD) +fi +TAG="manual-${SHA}" +IMG="${REGISTRY}:${TAG}" + +echo "[deploy] staging napi binary" +cp "$NAPI_SRC" "$NAPI_DST" + +echo "[deploy] building $IMG" +docker build --progress=plain -t "$IMG" -f examples/kitchen-sink/Dockerfile . + +echo "[deploy] configuring docker auth for ${REGION}-docker.pkg.dev" +gcloud auth configure-docker "${REGION}-docker.pkg.dev" --quiet >/dev/null + +echo "[deploy] pushing $IMG" +docker push "$IMG" + +deploy_one() { + local svc="$1" + echo "[deploy] updating Cloud Run service $svc -> $IMG" + gcloud run services update "$svc" \ + --image="$IMG" \ + --project="$PROJECT" \ + --region="$REGION" \ + --quiet + local url + url=$(gcloud run services describe "$svc" \ + --project="$PROJECT" --region="$REGION" \ + --format='value(status.url)') + echo "[deploy] verifying $svc /api/rivet/health" + if ! curl --max-time 20 -fsS "$url/api/rivet/health"; then + echo + echo "[deploy] WARNING: health check failed for $svc" >&2 + return 1 + fi + echo +} + +if [[ -z "$ONLY" || "$ONLY" == "staging" ]]; then + deploy_one kitchen-sink-staging +fi +if [[ -z "$ONLY" || "$ONLY" == "prod" ]]; then + deploy_one rivet-kitchen-sink +fi + +echo "[deploy] done." diff --git a/examples/kitchen-sink/scripts/mock-agentic-loop.ts b/examples/kitchen-sink/scripts/mock-agentic-loop.ts index 789a8f9baa..2f0709bea2 100644 --- a/examples/kitchen-sink/scripts/mock-agentic-loop.ts +++ b/examples/kitchen-sink/scripts/mock-agentic-loop.ts @@ -38,7 +38,7 @@ const NAMESPACE = const TOKEN = process.env.MOCK_AGENTIC_TOKEN ?? process.env.RIVET_TOKEN ?? "dev"; const POOL_NAME = - process.env.MOCK_AGENTIC_POOL ?? process.env.RIVET_POOL ?? "default"; + process.env.MOCK_AGENTIC_POOL ?? process.env.RIVET_POOL ?? "k8s"; const KEY_PREFIX = process.env.MOCK_AGENTIC_KEY_PREFIX ?? "mock-agentic-loop"; const DURATION_MS = numberFromEnv("MOCK_AGENTIC_DURATION_MS", 180_000); const INFERENCE_MIN_SECONDS = numberFromEnv( diff --git a/examples/kitchen-sink/scripts/on-sleep-remote-watch.ts b/examples/kitchen-sink/scripts/on-sleep-remote-watch.ts new file mode 100644 index 0000000000..4985a41692 --- /dev/null +++ b/examples/kitchen-sink/scripts/on-sleep-remote-watch.ts @@ -0,0 +1,508 @@ +// Remote onSleep watcher for a kitchen-sink envoy pool. +// +// This does not start the engine or kitchen-sink. Point it at a remote Rivet +// endpoint, then manually roll the envoy pods after the WebSocket is open. +// +// Usage: +// RIVET_ENDPOINT=https://namespace:token@... \ +// pnpm smoke:on-sleep-remote -- \ +// --pool kitchen-sink \ +// --on-sleep-duration-ms 60000 \ +// --open-delay-ms 0 \ +// --reconnect-timeout-ms 5000 + +import { createClient } from "rivetkit/client"; + +installTimestampedConsole(); + +const CLI_ARGS = parseCliArgs(process.argv.slice(2)); +const RAW_ENDPOINT = requiredStringFromConfig("endpoint", ["RIVET_ENDPOINT"]); +const POOL_NAME = stringFromConfig("pool", ["RIVET_POOL"], "default"); +const ENDPOINT = parseEndpoint(RAW_ENDPOINT, POOL_NAME); +const KEY = stringFromConfig( + "key", + ["SIGTERM_SLEEP_KEY"], + `remote-sleep-probe-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, +); +const LABEL = stringFromConfig("label", ["SIGTERM_SLEEP_LABEL"], KEY); +const ON_SLEEP_DURATION_MS = numberFromConfig( + "on-sleep-duration-ms", + ["SIGTERM_SLEEP_ON_SLEEP_DURATION_MS"], + 60_000, +); +const ON_SLEEP_TICK_MS = numberFromConfig( + "on-sleep-tick-ms", + ["SIGTERM_SLEEP_ON_SLEEP_TICK_MS"], + 1_000, +); +const OPEN_TIMEOUT_MS = numberFromConfig("open-timeout-ms", [], 15_000); +const OPEN_DELAY_MS = numberFromConfig("open-delay-ms", [], 0); +const MESSAGE_TIMEOUT_MS = numberFromConfig("message-timeout-ms", [], 5_000); +const RECONNECT_TIMEOUT_MS = numberFromConfig("reconnect-timeout-ms", [], 5_000); +const WATCH_TIMEOUT_MS = numberFromConfig( + "watch-timeout-ms", + [], + Math.max(5 * 60_000, ON_SLEEP_DURATION_MS + 2 * 60_000), +); +const CLOSE_CODE = 1000; +const CLOSE_REASON = "actor stopped"; + +type JsonRecord = Record; + +interface CloseInfo { + code: number; + reason: string; + wasClean: boolean; + at: number; +} + +interface ProofRow { + id: number; + event: string; + sleep_count: number; + detail: string | null; + created_at: number; +} + +interface Proof { + state: { + label: string; + wakeCount: number; + sleepCount: number; + onSleepDurationMs: number; + onSleepTickMs: number; + connectionCount: number; + messageCount: number; + onSleepStartedAt: number | null; + onSleepAsyncFinishedAt: number | null; + onSleepFinishedAt: number | null; + onSleepLastError: string | null; + }; + rows: ProofRow[]; +} + +function installTimestampedConsole(): void { + const originalLog = console.log.bind(console); + const originalError = console.error.bind(console); + const originalWarn = console.warn.bind(console); + console.log = (...args: unknown[]) => originalLog(`[${new Date().toISOString()}]`, ...args); + console.error = (...args: unknown[]) => + originalError(`[${new Date().toISOString()}]`, ...args); + console.warn = (...args: unknown[]) => + originalWarn(`[${new Date().toISOString()}]`, ...args); +} + +function parseCliArgs(args: string[]): Map { + const parsed = new Map(); + for (let i = 0; i < args.length; i += 1) { + const arg = args[i]; + if (arg === "--") continue; + if (!arg.startsWith("--")) { + throw new Error(`unexpected argument "${arg}". Use --name value.`); + } + + const eqIndex = arg.indexOf("="); + if (eqIndex !== -1) { + const name = arg.slice(2, eqIndex); + const value = arg.slice(eqIndex + 1); + if (!name || value === "") { + throw new Error(`invalid argument "${arg}". Use --name=value.`); + } + parsed.set(name, value); + continue; + } + + const name = arg.slice(2); + const value = args[i + 1]; + if (!name || value === undefined || value.startsWith("--")) { + throw new Error(`missing value for --${name}`); + } + parsed.set(name, value); + i += 1; + } + return parsed; +} + +function stringFromConfig( + argName: string, + envNames: string[], + fallback: string, +): string { + const arg = CLI_ARGS.get(argName); + if (arg !== undefined) return arg; + for (const envName of envNames) { + const raw = process.env[envName]; + if (raw !== undefined && raw !== "") return raw; + } + return fallback; +} + +function requiredStringFromConfig(argName: string, envNames: string[]): string { + const arg = CLI_ARGS.get(argName); + if (arg !== undefined && arg !== "") return arg; + for (const envName of envNames) { + const raw = process.env[envName]; + if (raw !== undefined && raw !== "") return raw; + } + throw new Error( + `missing required --${argName}. Set ${envNames.join(" or ")} or pass --${argName}.`, + ); +} + +function numberFromConfig( + argName: string, + envNames: string[], + fallback: number, +): number { + const raw = + CLI_ARGS.get(argName) ?? + envNames.map((envName) => process.env[envName]).find((value) => value); + if (raw === undefined || raw === "") return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 0) { + throw new Error(`--${argName} must be a finite non-negative number`); + } + return parsed; +} + +function formatError(error: unknown): string { + if (error instanceof Error) return `${error.name}: ${error.message}`; + return String(error); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function parseEndpoint(raw: string, poolName: string) { + const url = new URL(raw); + const namespace = decodeURIComponent(url.username); + const token = url.password ? decodeURIComponent(url.password) : undefined; + if (!namespace) { + throw new Error("RIVET_ENDPOINT must include namespace auth, e.g. https://namespace:token@host"); + } + url.username = ""; + url.password = ""; + const endpoint = url.toString().replace(/\/$/, ""); + const wsProtocol = url.protocol === "https:" ? "wss:" : "ws:"; + const wsOrigin = `${wsProtocol}//${url.host}`; + return { endpoint, namespace, token, poolName, wsOrigin }; +} + +function buildRawWebSocketUrl(): string { + const params = new URLSearchParams(); + params.set("rvt-namespace", ENDPOINT.namespace); + params.set("rvt-method", "getOrCreate"); + params.set("rvt-runner", ENDPOINT.poolName); + params.set("rvt-key", KEY); + params.set("rvt-crash-policy", "sleep"); + params.set("rvt-skip-ready-wait", "true"); + if (ENDPOINT.token) { + params.set("rvt-token", ENDPOINT.token); + } + return `${ENDPOINT.wsOrigin}/gateway/sigtermSleepProbe/websocket?${params}`; +} + +function client() { + return createClient({ + endpoint: RAW_ENDPOINT, + poolName: ENDPOINT.poolName, + }); +} + +async function waitForOpen(ws: WebSocket, timeoutMs: number): Promise { + if (ws.readyState === WebSocket.OPEN) return; + await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error(`websocket open timeout after ${timeoutMs}ms`)), + timeoutMs, + ); + const cleanup = () => clearTimeout(timeout); + ws.addEventListener( + "open", + () => { + cleanup(); + resolve(); + }, + { once: true }, + ); + ws.addEventListener( + "error", + () => { + cleanup(); + reject(new Error("websocket error before open")); + }, + { once: true }, + ); + ws.addEventListener( + "close", + (event) => { + cleanup(); + reject( + new Error( + `websocket closed before open code=${event.code} reason=${event.reason}`, + ), + ); + }, + { once: true }, + ); + }); +} + +async function waitForMessage( + ws: WebSocket, + predicate: (message: JsonRecord) => boolean, + timeoutMs: number, + label: string, +): Promise { + return await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error(`${label} message timeout after ${timeoutMs}ms`)), + timeoutMs, + ); + const cleanup = () => { + clearTimeout(timeout); + ws.removeEventListener("message", onMessage); + ws.removeEventListener("close", onClose); + }; + const onMessage = (event: MessageEvent) => { + const data = typeof event.data === "string" ? event.data : String(event.data); + console.log(`[ws:message] ${data}`); + let parsed: JsonRecord; + try { + parsed = JSON.parse(data) as JsonRecord; + } catch { + return; + } + if (!predicate(parsed)) return; + cleanup(); + resolve(parsed); + }; + const onClose = (event: CloseEvent) => { + cleanup(); + reject( + new Error( + `${label} closed while waiting code=${event.code} reason=${event.reason}`, + ), + ); + }; + ws.addEventListener("message", onMessage); + ws.addEventListener("close", onClose, { once: true }); + }); +} + +function waitForClose( + ws: WebSocket, + timeoutMs: number, + state: { + sawOnSleepStarted: boolean; + sawOnSleepFinished: boolean; + onSleepTickCount: number; + }, +): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error(`websocket close timeout after ${timeoutMs}ms`)), + timeoutMs, + ); + ws.addEventListener("message", (event) => { + const data = typeof event.data === "string" ? event.data : String(event.data); + console.log(`[ws:message] ${data}`); + try { + const parsed = JSON.parse(data) as JsonRecord; + if (parsed.type === "onSleepStarted") { + state.sawOnSleepStarted = true; + } + if (parsed.type === "onSleepTick") { + state.onSleepTickCount += 1; + } + if (parsed.type === "onSleepFinished") { + state.sawOnSleepFinished = true; + } + } catch { + // Raw non-JSON messages are still logged above. + } + }); + ws.addEventListener( + "close", + (event) => { + clearTimeout(timeout); + resolve({ + code: event.code, + reason: event.reason, + wasClean: event.wasClean, + at: Date.now(), + }); + }, + { once: true }, + ); + }); +} + +async function connectAndPingPong( + label: string, + openTimeoutMs: number, + messageTimeoutMs: number, +): Promise { + const webSocketUrl = buildRawWebSocketUrl(); + console.log(`[ws] ${label} connecting url=${webSocketUrl}`); + const ws = new WebSocket(webSocketUrl, ["rivet", "rivet_encoding.json"]); + ws.addEventListener("error", () => { + console.error(`[ws:error] ${label}`); + }); + ws.addEventListener("close", (event) => { + console.log( + `[ws:close] ${label} code=${event.code} reason=${event.reason} wasClean=${event.wasClean}`, + ); + }); + + try { + await waitForOpen(ws, openTimeoutMs); + console.log(`[ws] ${label} open`); + await waitForMessage( + ws, + (message) => message.type === "welcome", + messageTimeoutMs, + `${label} welcome`, + ); + ws.send(JSON.stringify({ type: "ping", label, timestamp: Date.now() })); + await waitForMessage( + ws, + (message) => message.type === "pong", + messageTimeoutMs, + `${label} pong`, + ); + console.log(`[ws] ${label} ping pong ok`); + return ws; + } catch (error) { + if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) { + ws.close(1000, `${label} failed`); + } + throw error; + } +} + +async function reconnectAndPingPong(): Promise { + console.log(`[ws] reconnect immediate timeoutMs=${RECONNECT_TIMEOUT_MS}`); + return await connectAndPingPong( + "reconnect", + Math.min(OPEN_TIMEOUT_MS, RECONNECT_TIMEOUT_MS), + Math.min(MESSAGE_TIMEOUT_MS, RECONNECT_TIMEOUT_MS), + ); +} + +function assertClose(close: CloseInfo, startedAt: number): void { + const elapsedMs = close.at - startedAt; + if (close.code !== CLOSE_CODE || close.reason !== CLOSE_REASON) { + throw new Error( + `expected close code=${CLOSE_CODE} reason=${CLOSE_REASON}; got code=${close.code} reason=${close.reason}`, + ); + } + if (elapsedMs < ON_SLEEP_DURATION_MS) { + throw new Error( + `websocket closed too early: ${elapsedMs}ms < ${ON_SLEEP_DURATION_MS}ms`, + ); + } + if (elapsedMs > ON_SLEEP_DURATION_MS + 120_000) { + throw new Error( + `websocket closed too late: ${elapsedMs}ms > ${ON_SLEEP_DURATION_MS + 120_000}ms`, + ); + } +} + +function assertProof(proof: Proof): void { + const events = proof.rows.map((row) => row.event); + const start = proof.rows.find((row) => row.event === "on-sleep-start"); + const afterAwait = proof.rows.find( + (row) => row.event === "on-sleep-after-await", + ); + const finish = proof.rows.find((row) => row.event === "on-sleep-finish"); + const ticks = proof.rows.filter((row) => row.event === "on-sleep-tick"); + + if (proof.state.sleepCount < 1) { + throw new Error(`expected sleepCount >= 1, got ${proof.state.sleepCount}`); + } + if (proof.state.onSleepLastError !== null) { + throw new Error(`onSleep error: ${proof.state.onSleepLastError}`); + } + if (!start || !afterAwait || !finish) { + throw new Error(`missing onSleep proof rows. saw events=${events.join(",")}`); + } + + const elapsedMs = afterAwait.created_at - start.created_at; + if (elapsedMs < ON_SLEEP_DURATION_MS) { + throw new Error( + `onSleep proof delay too short: ${elapsedMs}ms < ${ON_SLEEP_DURATION_MS}ms`, + ); + } + if (finish.created_at < afterAwait.created_at) { + throw new Error("on-sleep-finish row was written before async row"); + } + + const expectedTicks = Math.ceil(ON_SLEEP_DURATION_MS / ON_SLEEP_TICK_MS); + if (ticks.length < expectedTicks) { + throw new Error( + `expected at least ${expectedTicks} on-sleep-tick rows, got ${ticks.length}`, + ); + } +} + +async function main(): Promise { + if (ON_SLEEP_DURATION_MS <= 0) { + throw new Error("--on-sleep-duration-ms must be positive"); + } + if (ON_SLEEP_TICK_MS <= 0) { + throw new Error("--on-sleep-tick-ms must be positive"); + } + + console.log( + `[config] endpoint=${ENDPOINT.endpoint} namespace=${ENDPOINT.namespace} pool=${ENDPOINT.poolName} key=${KEY} durationMs=${ON_SLEEP_DURATION_MS} tickMs=${ON_SLEEP_TICK_MS} openDelayMs=${OPEN_DELAY_MS} reconnectTimeoutMs=${RECONNECT_TIMEOUT_MS} watchTimeoutMs=${WATCH_TIMEOUT_MS}`, + ); + + const handle = client().sigtermSleepProbe.getOrCreate([KEY]); + const actorId = await handle.resolve(); + console.log(`[actor] actorId=${actorId}`); + + const prepared = await handle.prepare( + LABEL, + ON_SLEEP_DURATION_MS, + ON_SLEEP_TICK_MS, + ); + console.log(`[actor] prepared=${JSON.stringify(prepared)}`); + + if (OPEN_DELAY_MS > 0) { + console.log(`[ws] waiting before open delayMs=${OPEN_DELAY_MS}`); + await sleep(OPEN_DELAY_MS); + } + const ws = await connectAndPingPong("initial", OPEN_TIMEOUT_MS, MESSAGE_TIMEOUT_MS); + console.log("[manual] roll/restart the remote kitchen-sink envoy pods now"); + + const state = { + sawOnSleepStarted: false, + sawOnSleepFinished: false, + onSleepTickCount: 0, + }; + const startedAt = Date.now(); + const close = await waitForClose(ws, WATCH_TIMEOUT_MS, state); + const elapsedMs = close.at - startedAt; + console.log( + `[ws:close] code=${close.code} reason=${close.reason} wasClean=${close.wasClean} elapsedMs=${elapsedMs}`, + ); + console.log( + `[ws] observed sleep messages started=${state.sawOnSleepStarted} ticks=${state.onSleepTickCount} finished=${state.sawOnSleepFinished}`, + ); + + const reconnect = await reconnectAndPingPong(); + reconnect.close(1000, "remote smoke done"); + + const proof = (await handle.getProof()) as Proof; + assertClose(close, startedAt); + assertProof(proof); + console.log(`[proof] ${JSON.stringify(proof.state)}`); + console.log("[done] PASS observed shutdown close, reconnected, and verified onSleep proof"); +} + +main().catch((error) => { + console.error(`[fail] ${formatError(error)}`); + process.exitCode = 1; +}); diff --git a/examples/kitchen-sink/scripts/on-sleep-sigterm-smoke.ts b/examples/kitchen-sink/scripts/on-sleep-sigterm-smoke.ts new file mode 100644 index 0000000000..977f180178 --- /dev/null +++ b/examples/kitchen-sink/scripts/on-sleep-sigterm-smoke.ts @@ -0,0 +1,865 @@ +// SIGTERM sleep handoff smoke test. +// +// Requires an already-running engine, usually at http://127.0.0.1:6420. +// Starts two kitchen-sink serverful envoys with raw node, SIGTERMs the first, +// and verifies the actor completes onSleep before reconnecting on the second. +// +// Usage: +// pnpm --filter kitchen-sink smoke:on-sleep-sigterm -- \ +// --on-sleep-duration-ms 5000 \ +// --on-sleep-tick-ms 1000 +// +// Useful overrides: +// --endpoint http://127.0.0.1:6420 +// --namespace default +// --pool sigterm-sleep-test +// --reconnect-open-delay-ms 0 + +import { + spawn, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { createClient } from "rivetkit/client"; +import type { registry } from "../src/index.ts"; + +const KITCHEN_SINK_ROOT = fileURLToPath(new URL("..", import.meta.url)); +installTimestampedConsole(); +const CLI_ARGS = parseCliArgs(process.argv.slice(2)); +const ENDPOINT = stringFromConfig( + "endpoint", + ["SIGTERM_SLEEP_ENDPOINT", "RIVET_ENDPOINT"], + "http://127.0.0.1:6420", +); +const NAMESPACE = stringFromConfig( + "namespace", + ["SIGTERM_SLEEP_NAMESPACE", "RIVET_NAMESPACE"], + "default", +); +const TOKEN = stringFromConfig( + "token", + ["SIGTERM_SLEEP_TOKEN", "RIVET_TOKEN"], + "dev", +); +const POOL_NAME = stringFromConfig( + "pool", + ["SIGTERM_SLEEP_POOL", "RIVET_POOL"], + `sigterm-sleep-${Date.now()}`, +); +const KEY = stringFromConfig( + "key", + ["SIGTERM_SLEEP_KEY"], + `sigterm-sleep-probe-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, +); +const LABEL = stringFromConfig("label", ["SIGTERM_SLEEP_LABEL"], KEY); +const ON_SLEEP_DURATION_MS = numberFromConfig( + "on-sleep-duration-ms", + "SIGTERM_SLEEP_ON_SLEEP_DURATION_MS", + 5_000, +); +const ON_SLEEP_TICK_MS = numberFromConfig( + "on-sleep-tick-ms", + "SIGTERM_SLEEP_ON_SLEEP_TICK_MS", + 1_000, +); +const RUNNER_1_PORT = numberFromConfig( + "runner-1-port", + "SIGTERM_SLEEP_RUNNER_1_PORT", + 3101, +); +const RUNNER_2_PORT = numberFromConfig( + "runner-2-port", + "SIGTERM_SLEEP_RUNNER_2_PORT", + 3102, +); +const ENGINE_READY_TIMEOUT_MS = numberFromConfig( + "engine-ready-timeout-ms", + "SIGTERM_SLEEP_ENGINE_READY_TIMEOUT_MS", + 15_000, +); +const ENVOY_READY_TIMEOUT_MS = numberFromConfig( + "envoy-ready-timeout-ms", + "SIGTERM_SLEEP_ENVOY_READY_TIMEOUT_MS", + 60_000, +); +const RUNNER_EXIT_TIMEOUT_MS = numberFromConfig( + "runner-exit-timeout-ms", + "SIGTERM_SLEEP_RUNNER_EXIT_TIMEOUT_MS", + Math.max(45_000, ON_SLEEP_DURATION_MS + 45_000), +); +const WS_OPEN_TIMEOUT_MS = numberFromConfig( + "ws-open-timeout-ms", + "SIGTERM_SLEEP_WS_OPEN_TIMEOUT_MS", + 15_000, +); +const WS_MESSAGE_TIMEOUT_MS = numberFromConfig( + "ws-message-timeout-ms", + "SIGTERM_SLEEP_WS_MESSAGE_TIMEOUT_MS", + 10_000, +); +const RECONNECT_MESSAGE_TIMEOUT_MS = numberFromConfig( + "reconnect-message-timeout-ms", + "SIGTERM_SLEEP_RECONNECT_MESSAGE_TIMEOUT_MS", + 5_000, +); +const RECONNECT_TIMEOUT_MS = numberFromConfig( + "reconnect-timeout-ms", + "SIGTERM_SLEEP_RECONNECT_TIMEOUT_MS", + 5_000, +); +const RECONNECT_OPEN_DELAY_MS = numberFromConfig( + "reconnect-open-delay-ms", + "SIGTERM_SLEEP_RECONNECT_OPEN_DELAY_MS", + 0, +); +const CLOSE_REASON = "actor stopped"; +const CLOSE_CODE = 1000; +let currentPhase: + | { + title: string; + startedAt: number; + } + | undefined; +const COLOR_ENABLED = process.env.NO_COLOR === undefined && process.env.TERM !== "dumb"; +const ANSI = { + reset: "\x1b[0m", + bold: "\x1b[1m", + dim: "\x1b[2m", + red: "\x1b[31m", + green: "\x1b[32m", + yellow: "\x1b[33m", + blue: "\x1b[34m", + magenta: "\x1b[35m", + cyan: "\x1b[36m", + gray: "\x1b[90m", +}; + +interface Envoy { + envoy_key: string; + pool_name: string; + create_ts: number; + last_ping_ts: number; + stop_ts?: number | null; +} + +interface EnvoysResponse { + envoys: Envoy[]; +} + +interface ProofRow { + id: number; + event: string; + sleep_count: number; + detail: string | null; + created_at: number; +} + +interface Proof { + state: { + label: string; + wakeCount: number; + sleepCount: number; + onSleepDurationMs: number; + onSleepTickMs: number; + connectionCount: number; + messageCount: number; + onSleepStartedAt: number | null; + onSleepAsyncFinishedAt: number | null; + onSleepFinishedAt: number | null; + onSleepLastError: string | null; + }; + rows: ProofRow[]; +} + +interface CloseInfo { + code: number; + reason: string; + wasClean: boolean; + at: number; +} + +function logTimestamp(): string { + return new Date().toISOString(); +} + +function color(text: string, code: string): string { + if (!COLOR_ENABLED) return text; + return `${code}${text}${ANSI.reset}`; +} + +function colorForPrefix(prefix: string): string { + if (prefix.startsWith("[runner:one]")) return ANSI.yellow; + if (prefix.startsWith("[runner:two]")) return ANSI.green; + if (prefix.startsWith("[runner:")) return ANSI.green; + if (prefix.startsWith("[envoys]")) return ANSI.blue; + if (prefix.startsWith("[ws:error]")) return ANSI.red; + if (prefix.startsWith("[ws:close]")) return ANSI.yellow; + if (prefix.startsWith("[ws:message]")) return ANSI.gray; + if (prefix.startsWith("[ws]")) return ANSI.magenta; + if (prefix.startsWith("[test]")) return ANSI.cyan; + return ANSI.gray; +} + +function colorizeLogText(text: string, level: "log" | "warn" | "error"): string { + const colored = text.replace(/^(\[[^\]]+\])/, (prefix) => + color(prefix, level === "error" ? ANSI.red : colorForPrefix(prefix)), + ); + return level === "warn" ? color(colored, ANSI.yellow) : colored; +} + +function formatTimestamp(): string { + return color(`[${logTimestamp()}]`, ANSI.dim + ANSI.gray); +} + +function formatConsoleArgs( + level: "log" | "warn" | "error", + args: unknown[], +): unknown[] { + if (typeof args[0] !== "string") return [formatTimestamp(), ...args]; + return [formatTimestamp(), colorizeLogText(args[0], level), ...args.slice(1)]; +} + +function formatDuration(ms: number): string { + return `${ms}ms ${(ms / 1000).toFixed(3)}s`; +} + +function logPhase(title: string): void { + finishPhase(); + const line = "=".repeat(88); + const code = ANSI.bold + ANSI.magenta; + console.log(color(`[phase] ${line}`, code)); + console.log(color(`[phase] ${title.toUpperCase()}`, code)); + console.log(color(`[phase] ${line}`, code)); + currentPhase = { + title, + startedAt: Date.now(), + }; +} + +function finishPhase(): void { + if (!currentPhase) return; + const durationMs = Date.now() - currentPhase.startedAt; + const code = ANSI.bold + ANSI.green; + console.log( + color( + `[phase] complete "${currentPhase.title}" duration=${formatDuration(durationMs)}`, + code, + ), + ); + currentPhase = undefined; +} + +function installTimestampedConsole(): void { + const originalLog = console.log.bind(console); + const originalError = console.error.bind(console); + const originalWarn = console.warn.bind(console); + + console.log = (...args: unknown[]) => originalLog(...formatConsoleArgs("log", args)); + console.error = (...args: unknown[]) => + originalError(...formatConsoleArgs("error", args)); + console.warn = (...args: unknown[]) => + originalWarn(...formatConsoleArgs("warn", args)); +} + +function parseCliArgs(args: string[]): Map { + const parsed = new Map(); + for (let i = 0; i < args.length; i += 1) { + const arg = args[i]; + if (arg === "--") continue; + if (!arg.startsWith("--")) { + throw new Error(`unexpected argument "${arg}". Use --name value.`); + } + + const eqIndex = arg.indexOf("="); + if (eqIndex !== -1) { + const name = arg.slice(2, eqIndex); + const value = arg.slice(eqIndex + 1); + if (!name || value === "") { + throw new Error(`invalid argument "${arg}". Use --name=value.`); + } + parsed.set(name, value); + continue; + } + + const name = arg.slice(2); + const value = args[i + 1]; + if (!name || value === undefined || value.startsWith("--")) { + throw new Error(`missing value for --${name}`); + } + parsed.set(name, value); + i += 1; + } + return parsed; +} + +function stringFromConfig( + argName: string, + envNames: string[], + fallback: string, +): string { + const arg = CLI_ARGS.get(argName); + if (arg !== undefined) return arg; + + for (const envName of envNames) { + const raw = process.env[envName]; + if (raw !== undefined && raw !== "") return raw; + } + + return fallback; +} + +function numberFromConfig( + argName: string, + envName: string, + fallback: number, +): number { + const raw = CLI_ARGS.get(argName) ?? process.env[envName]; + if (raw === undefined || raw === "") return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 0) { + throw new Error(`--${argName} must be a finite non-negative number`); + } + return parsed; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function formatError(error: unknown): string { + if (error instanceof Error) return `${error.name}: ${error.message}`; + return String(error); +} + +function appendPath(endpoint: string, path: string): URL { + const url = new URL(endpoint); + const prefix = url.pathname.replace(/\/$/, ""); + url.pathname = `${prefix}${path}`; + url.search = ""; + url.hash = ""; + return url; +} + +function buildEnvoysUrl(): string { + const url = appendPath(ENDPOINT, "/envoys"); + url.searchParams.set("namespace", NAMESPACE); + url.searchParams.set("name", POOL_NAME); + url.searchParams.set("limit", "100"); + return url.toString(); +} + +function buildWebSocketUrl(_actorId: string): string { + const url = appendPath( + ENDPOINT, + `/gateway/sigtermSleepProbe/websocket`, + ); + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + url.searchParams.set("rvt-namespace", NAMESPACE); + url.searchParams.set("rvt-method", "getOrCreate"); + url.searchParams.set("rvt-runner", POOL_NAME); + url.searchParams.set("rvt-key", KEY); + url.searchParams.set("rvt-crash-policy", "sleep"); + url.searchParams.set("rvt-skip-ready-wait", "true"); + if (TOKEN) { + url.searchParams.set("rvt-token", TOKEN); + } + return url.toString(); +} + +function runnerEnv(port: number): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ...process.env, + RIVET_KITCHEN_SINK_MODE: "serverful", + RIVET_ENDPOINT: ENDPOINT, + RIVET_NAMESPACE: NAMESPACE, + RIVET_TOKEN: TOKEN, + RIVET_POOL: POOL_NAME, + RIVET_LOG_LEVEL: process.env.RIVET_LOG_LEVEL ?? "info", + RIVET_LOG_TARGET: process.env.RIVET_LOG_TARGET ?? "1", + RIVET_LOG_TIMESTAMP: process.env.RIVET_LOG_TIMESTAMP ?? "1", + PORT: String(port), + }; + delete env.RIVET_RUN_ENGINE; + delete env.RIVET_SERVERLESS_URL; + delete env.KITCHEN_SINK_SERVERLESS_URL; + return env; +} + +function startRunner( + label: string, + port: number, +): ChildProcessWithoutNullStreams { + const runner = spawn( + process.execPath, + [ + "--experimental-strip-types", + "--experimental-transform-types", + "--import", + "@rivetkit/sql-loader", + "src/server.ts", + ], + { + cwd: KITCHEN_SINK_ROOT, + env: runnerEnv(port), + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + runner.stdout.on("data", (chunk) => { + process.stdout.write(prefixChunk(`[runner:${label}]`, chunk)); + }); + runner.stderr.on("data", (chunk) => { + process.stderr.write(prefixChunk(`[runner:${label}]`, chunk)); + }); + + console.log(`[test] started ${label} pid=${runner.pid} port=${port}`); + return runner; +} + +function prefixChunk(prefix: string, chunk: Buffer): string { + return chunk + .toString("utf8") + .split(/\r?\n/) + .map((line, index, lines) => { + if (line === "" && index === lines.length - 1) return ""; + return `${formatTimestamp()} ${color(prefix, colorForPrefix(prefix))} ${line}`; + }) + .join("\n"); +} + +async function waitForExit( + runner: ChildProcessWithoutNullStreams, + timeoutMs: number, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + if (runner.exitCode !== null || runner.signalCode !== null) { + return { code: runner.exitCode, signal: runner.signalCode }; + } + + const exitPromise = once(runner, "exit").then(([code, signal]) => ({ + code: code as number | null, + signal: signal as NodeJS.Signals | null, + })); + const result = await Promise.race([ + exitPromise, + sleep(timeoutMs).then(() => null), + ]); + if (result) return result; + throw new Error(`runner did not exit within ${timeoutMs}ms`); +} + +async function stopRunner( + runner: ChildProcessWithoutNullStreams | undefined, + label: string, +): Promise<{ code: number | null; signal: NodeJS.Signals | null } | undefined> { + if (!runner || runner.exitCode !== null || runner.signalCode !== null) { + return undefined; + } + + if (runner.pid === undefined) { + throw new Error(`runner ${label} has no pid`); + } + + console.log(`[test] sending SIGTERM to ${label} runner pid=${runner.pid}`); + runner.kill("SIGTERM"); + + try { + const exit = await waitForExit(runner, RUNNER_EXIT_TIMEOUT_MS); + console.log( + `[test] ${label} runner exited code=${exit.code} signal=${exit.signal}`, + ); + return exit; + } catch (error) { + console.error( + `[test] ${label} runner did not exit cleanly: ${formatError(error)}`, + ); + try { + runner.kill("SIGKILL"); + } catch {} + throw error; + } +} + +async function fetchEnvoys(): Promise { + const url = buildEnvoysUrl(); + const response = await fetch(url, { + headers: TOKEN ? { Authorization: `Bearer ${TOKEN}` } : undefined, + }); + const body = await response.text(); + console.log(`[envoys] status=${response.status} url=${url}`); + if (!response.ok) { + throw new Error(`GET /envoys status=${response.status} body=${body}`); + } + const parsed = JSON.parse(body) as EnvoysResponse; + return parsed.envoys.filter((envoy) => envoy.stop_ts === undefined || envoy.stop_ts === null); +} + +async function validateEngine(): Promise { + const deadline = Date.now() + ENGINE_READY_TIMEOUT_MS; + let lastError = "not attempted"; + while (Date.now() < deadline) { + try { + await fetchEnvoys(); + console.log(`[test] engine is reachable at ${ENDPOINT}`); + return; + } catch (error) { + lastError = formatError(error); + await sleep(500); + } + } + throw new Error(`engine is not reachable at ${ENDPOINT}: ${lastError}`); +} + +async function waitForEnvoyCount( + count: number, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + let lastEnvoys: Envoy[] = []; + while (Date.now() < deadline) { + lastEnvoys = await fetchEnvoys(); + const keys = lastEnvoys.map((envoy) => envoy.envoy_key).join(","); + console.log( + `[envoys] active=${lastEnvoys.length} expected>=${count} keys=${keys}`, + ); + if (lastEnvoys.length >= count) return lastEnvoys; + await sleep(500); + } + throw new Error( + `timed out waiting for ${count} active envoys in pool ${POOL_NAME}; saw ${lastEnvoys.length}`, + ); +} + +function client() { + return createClient({ + endpoint: ENDPOINT, + namespace: NAMESPACE, + token: TOKEN, + poolName: POOL_NAME, + }); +} + +async function waitForOpen(ws: WebSocket, timeoutMs: number): Promise { + if (ws.readyState === WebSocket.OPEN) return; + await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error(`websocket open timeout after ${timeoutMs}ms`)), + timeoutMs, + ); + ws.addEventListener( + "open", + () => { + clearTimeout(timeout); + resolve(); + }, + { once: true }, + ); + ws.addEventListener( + "error", + () => { + clearTimeout(timeout); + reject(new Error("websocket error before open")); + }, + { once: true }, + ); + ws.addEventListener( + "close", + (event) => { + clearTimeout(timeout); + reject( + new Error( + `websocket closed before open code=${event.code} reason=${event.reason}`, + ), + ); + }, + { once: true }, + ); + }); +} + +async function waitForMessage( + ws: WebSocket, + predicate: (message: any) => boolean, + timeoutMs: number, + label: string, +): Promise { + return await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error(`${label} message timeout after ${timeoutMs}ms`)), + timeoutMs, + ); + const onMessage = (event: MessageEvent) => { + const data = typeof event.data === "string" ? event.data : String(event.data); + console.log(`[ws:message] ${data}`); + let parsed: any; + try { + parsed = JSON.parse(data); + } catch { + return; + } + if (!predicate(parsed)) return; + cleanup(); + resolve(parsed); + }; + const onClose = (event: CloseEvent) => { + cleanup(); + reject( + new Error( + `${label} closed while waiting code=${event.code} reason=${event.reason}`, + ), + ); + }; + const cleanup = () => { + clearTimeout(timeout); + ws.removeEventListener("message", onMessage); + ws.removeEventListener("close", onClose); + }; + ws.addEventListener("message", onMessage); + ws.addEventListener("close", onClose, { once: true }); + }); +} + +async function connectAndPingPong( + actorId: string, + label: string, + openTimeoutMs = WS_OPEN_TIMEOUT_MS, + messageTimeoutMs = WS_MESSAGE_TIMEOUT_MS, +): Promise { + const wsUrl = buildWebSocketUrl(actorId); + console.log(`[ws] ${label} connecting url=${wsUrl}`); + const ws = new WebSocket(wsUrl, ["rivet", "rivet_encoding.json"]); + ws.addEventListener("close", (event) => { + console.log( + `[ws:close] ${label} code=${event.code} reason=${event.reason} wasClean=${event.wasClean}`, + ); + }); + ws.addEventListener("error", () => { + console.error(`[ws:error] ${label}`); + }); + + try { + await waitForOpen(ws, openTimeoutMs); + console.log(`[ws] ${label} open`); + await waitForMessage( + ws, + (message) => message.type === "welcome", + messageTimeoutMs, + `${label} welcome`, + ); + ws.send(JSON.stringify({ type: "ping", label, timestamp: Date.now() })); + await waitForMessage( + ws, + (message) => message.type === "pong", + messageTimeoutMs, + `${label} pong`, + ); + ws.addEventListener("message", (event) => { + const data = typeof event.data === "string" ? event.data : String(event.data); + console.log(`[ws:message] ${label} ${data}`); + }); + console.log(`[ws] ${label} ping pong ok`); + return ws; + } catch (error) { + if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) { + ws.close(1000, `${label} retry`); + } + throw error; + } +} + +async function reconnectAndPingPong( + actorId: string, + timeoutMs: number, +): Promise { + console.log( + `[ws] reconnect strict timeoutMs=${timeoutMs} messageTimeoutMs=${RECONNECT_MESSAGE_TIMEOUT_MS} openDelayMs=${RECONNECT_OPEN_DELAY_MS}`, + ); + if (RECONNECT_OPEN_DELAY_MS > 0) { + console.log(`[ws] reconnect waiting before open delayMs=${RECONNECT_OPEN_DELAY_MS}`); + await sleep(RECONNECT_OPEN_DELAY_MS); + } + return await connectAndPingPong( + actorId, + "reconnect", + Math.min(WS_OPEN_TIMEOUT_MS, timeoutMs), + Math.min(RECONNECT_MESSAGE_TIMEOUT_MS, timeoutMs), + ); +} + +function waitForClose(ws: WebSocket, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error(`websocket close timeout after ${timeoutMs}ms`)), + timeoutMs, + ); + ws.addEventListener( + "close", + (event) => { + clearTimeout(timeout); + resolve({ + code: event.code, + reason: event.reason, + wasClean: event.wasClean, + at: Date.now(), + }); + }, + { once: true }, + ); + }); +} + +function assertClose(close: CloseInfo, sigtermAt: number): void { + const elapsedMs = close.at - sigtermAt; + if (close.code !== CLOSE_CODE || close.reason !== CLOSE_REASON) { + throw new Error( + `expected close code=${CLOSE_CODE} reason=${CLOSE_REASON}; got code=${close.code} reason=${close.reason}`, + ); + } + if (elapsedMs < ON_SLEEP_DURATION_MS) { + throw new Error( + `websocket closed too early: ${elapsedMs}ms < ${ON_SLEEP_DURATION_MS}ms`, + ); + } + if (elapsedMs > ON_SLEEP_DURATION_MS + 10_000) { + throw new Error( + `websocket closed too late: ${elapsedMs}ms > ${ON_SLEEP_DURATION_MS + 10_000}ms`, + ); + } + console.log( + `[test] shutdown close matched code=${close.code} reason=${close.reason} elapsedMs=${elapsedMs}`, + ); +} + +function assertProof(proof: Proof): void { + const events = proof.rows.map((row) => row.event); + const start = proof.rows.find((row) => row.event === "on-sleep-start"); + const afterAwait = proof.rows.find( + (row) => row.event === "on-sleep-after-await", + ); + const finish = proof.rows.find((row) => row.event === "on-sleep-finish"); + const ticks = proof.rows.filter((row) => row.event === "on-sleep-tick"); + + if (proof.state.sleepCount < 1) { + throw new Error(`expected sleepCount >= 1, got ${proof.state.sleepCount}`); + } + if (proof.state.onSleepLastError !== null) { + throw new Error(`onSleep error: ${proof.state.onSleepLastError}`); + } + if (!start || !afterAwait || !finish) { + throw new Error( + `missing onSleep proof rows. saw events=${events.join(",")}`, + ); + } + + const elapsedMs = afterAwait.created_at - start.created_at; + if (elapsedMs < ON_SLEEP_DURATION_MS) { + throw new Error( + `onSleep proof delay too short: ${elapsedMs}ms < ${ON_SLEEP_DURATION_MS}ms`, + ); + } + if (finish.created_at < afterAwait.created_at) { + throw new Error("on-sleep-finish row was written before async row"); + } + + const expectedTicks = Math.ceil(ON_SLEEP_DURATION_MS / ON_SLEEP_TICK_MS); + if (ticks.length < expectedTicks) { + throw new Error( + `expected at least ${expectedTicks} on-sleep-tick rows, got ${ticks.length}`, + ); + } +} + +async function main(): Promise { + if (ON_SLEEP_DURATION_MS <= 0) { + throw new Error("SIGTERM_SLEEP_ON_SLEEP_DURATION_MS must be positive"); + } + if (ON_SLEEP_TICK_MS <= 0) { + throw new Error("SIGTERM_SLEEP_ON_SLEEP_TICK_MS must be positive"); + } + if (RECONNECT_OPEN_DELAY_MS < 0) { + throw new Error("SIGTERM_SLEEP_RECONNECT_OPEN_DELAY_MS must be non-negative"); + } + + console.log( + `[test] endpoint=${ENDPOINT} namespace=${NAMESPACE} pool=${POOL_NAME} key=${KEY} durationMs=${ON_SLEEP_DURATION_MS} tickMs=${ON_SLEEP_TICK_MS} reconnectOpenDelayMs=${RECONNECT_OPEN_DELAY_MS}`, + ); + + let runner1: ChildProcessWithoutNullStreams | undefined; + let runner2: ChildProcessWithoutNullStreams | undefined; + let ws1: WebSocket | undefined; + let ws2: WebSocket | undefined; + + try { + logPhase("1. Validate engine"); + await validateEngine(); + + logPhase("2. Start kitchen-sink runner one"); + runner1 = startRunner("one", RUNNER_1_PORT); + await waitForEnvoyCount(1, ENVOY_READY_TIMEOUT_MS); + + logPhase("3. Create and prepare actor"); + const firstClient = client(); + const handle = firstClient.sigtermSleepProbe.getOrCreate([KEY]); + const actorId = await handle.resolve(); + console.log(`[test] actorId=${actorId}`); + const prepared = await handle.prepare( + LABEL, + ON_SLEEP_DURATION_MS, + ON_SLEEP_TICK_MS, + ); + console.log(`[test] prepared ${JSON.stringify(prepared)}`); + + logPhase("4. Initial websocket ping pong"); + ws1 = await connectAndPingPong(actorId, "initial"); + + logPhase("5. Start kitchen-sink runner two"); + runner2 = startRunner("two", RUNNER_2_PORT); + await waitForEnvoyCount(2, ENVOY_READY_TIMEOUT_MS); + + logPhase("6. SIGTERM runner one and wait for onSleep close"); + const closePromise = waitForClose( + ws1, + ON_SLEEP_DURATION_MS + RUNNER_EXIT_TIMEOUT_MS, + ); + const sigtermAt = Date.now(); + const runner1ExitPromise = stopRunner(runner1, "one"); + runner1ExitPromise.catch(() => undefined); + runner1 = undefined; + const close = await closePromise; + assertClose(close, sigtermAt); + + logPhase("7. Reconnect through runner two"); + ws2 = await reconnectAndPingPong(actorId, RECONNECT_TIMEOUT_MS); + + const runner1Exit = await runner1ExitPromise; + console.log(`[test] first runner shutdown ${JSON.stringify(runner1Exit)}`); + + logPhase("8. Verify database proof"); + const proof = (await client().sigtermSleepProbe + .getOrCreate([KEY]) + .getProof()) as Proof; + assertProof(proof); + + console.log("[test] proof rows:"); + for (const row of proof.rows) { + console.log( + `[test] #${row.id} ${row.event} sleep=${row.sleep_count} detail=${row.detail ?? ""} at=${new Date(row.created_at).toISOString()}`, + ); + } + console.log(`[test] proof state ${JSON.stringify(proof.state)}`); + console.log("[test] PASS onSleep completed during SIGTERM and actor reconnected on the second kitchen-sink envoy"); + } finally { + logPhase("9. Cleanup"); + if (ws2 && ws2.readyState === WebSocket.OPEN) ws2.close(1000, "smoke done"); + if (ws1 && ws1.readyState === WebSocket.OPEN) ws1.close(1000, "smoke done"); + await stopRunner(runner2, "two").catch((error) => { + console.error(`[test] runner two cleanup failed: ${formatError(error)}`); + }); + await stopRunner(runner1, "one").catch((error) => { + console.error(`[test] runner one cleanup failed: ${formatError(error)}`); + }); + finishPhase(); + } +} + +main().then(() => process.exit(0)); diff --git a/examples/kitchen-sink/scripts/raw-websocket-serverless-smoke.ts b/examples/kitchen-sink/scripts/raw-websocket-serverless-smoke.ts index 32d651de4a..651229529d 100644 --- a/examples/kitchen-sink/scripts/raw-websocket-serverless-smoke.ts +++ b/examples/kitchen-sink/scripts/raw-websocket-serverless-smoke.ts @@ -19,7 +19,7 @@ const SERVERLESS_URL = process.env.RIVET_SERVERLESS_URL; const NAMESPACE = process.env.SMOKE_NAMESPACE ?? process.env.RIVET_NAMESPACE ?? "default"; const TOKEN = process.env.SMOKE_TOKEN ?? process.env.RIVET_TOKEN ?? "dev"; -const POOL_NAME = process.env.SMOKE_POOL ?? process.env.RIVET_POOL ?? "default"; +const POOL_NAME = process.env.SMOKE_POOL ?? process.env.RIVET_POOL ?? "k8s"; const KEY = process.env.SMOKE_KEY ?? `raw-ws-serverless-smoke-${Date.now()}`; const DURATION_MS = Number(process.env.SMOKE_DURATION_MS ?? "120000"); const PARALLELISM = Number(process.env.SMOKE_PARALLELISM ?? "1"); diff --git a/examples/kitchen-sink/scripts/sqlite-cold-start-bench.ts b/examples/kitchen-sink/scripts/sqlite-cold-start-bench.ts index e4dd3a6f48..6f00b88dbc 100644 --- a/examples/kitchen-sink/scripts/sqlite-cold-start-bench.ts +++ b/examples/kitchen-sink/scripts/sqlite-cold-start-bench.ts @@ -452,7 +452,7 @@ async function configureLocalRunner(endpoint: string): Promise { const datacenter = datacentersBody.datacenters[0]?.name; if (!datacenter) throw new Error("local engine returned no datacenters"); - const response = await fetch(`${base}/runner-configs/default?namespace=default`, { + const response = await fetch(`${base}/runner-configs/k8s?namespace=default`, { method: "PUT", headers: { Authorization: "Bearer dev", @@ -468,7 +468,7 @@ async function configureLocalRunner(endpoint: string): Promise { }); if (!response.ok) { throw new Error( - `failed to configure local default runner: ${response.status} ${await response.text()}`, + `failed to configure local k8s runner: ${response.status} ${await response.text()}`, ); } } @@ -478,7 +478,7 @@ async function waitForEnvoy(endpoint: string): Promise { const deadline = Date.now() + 15_000; while (Date.now() < deadline) { - const response = await fetch(`${base}/envoys?namespace=default&name=default`, { + const response = await fetch(`${base}/envoys?namespace=default&name=k8s`, { headers: { Authorization: "Bearer dev" }, }); if (response.ok) { diff --git a/examples/kitchen-sink/scripts/sqlite-memory-soak.ts b/examples/kitchen-sink/scripts/sqlite-memory-soak.ts index a450f09bf6..857e8ef2f6 100644 --- a/examples/kitchen-sink/scripts/sqlite-memory-soak.ts +++ b/examples/kitchen-sink/scripts/sqlite-memory-soak.ts @@ -674,7 +674,7 @@ async function startKitchenSinkServer( RIVET_ENDPOINT: args.endpoint, RIVET_TOKEN: process.env.RIVET_TOKEN ?? "dev", RIVET_NAMESPACE: process.env.RIVET_NAMESPACE ?? "default", - RIVET_POOL: process.env.RIVET_POOL ?? "default", + RIVET_POOL: process.env.RIVET_POOL ?? "k8s", RIVET_SERVERLESS_URL: serverlessUrl, RIVET_SERVERLESS_REQUEST_LIFESPAN: args.requestLifespanSeconds.toString(), @@ -748,7 +748,7 @@ async function configureServerlessRunner( const base = args.endpoint.replace(/\/$/, ""); const namespace = process.env.RIVET_NAMESPACE ?? "default"; const token = process.env.RIVET_TOKEN ?? "dev"; - const poolName = process.env.RIVET_POOL ?? "default"; + const poolName = process.env.RIVET_POOL ?? "k8s"; const datacentersResponse = await fetch(`${base}/datacenters?namespace=${namespace}`, { headers: { Authorization: `Bearer ${token}` }, }); @@ -1912,7 +1912,7 @@ async function main(): Promise { endpoint: args.endpoint, namespace: process.env.RIVET_NAMESPACE ?? "default", token: process.env.RIVET_TOKEN ?? "dev", - poolName: process.env.RIVET_POOL ?? "default", + poolName: process.env.RIVET_POOL ?? "k8s", }); if (args.preWorkloadWaitMs > 0) { diff --git a/examples/kitchen-sink/scripts/sqlite-realworld-bench.ts b/examples/kitchen-sink/scripts/sqlite-realworld-bench.ts index 5fe05832c0..6f2cab877c 100644 --- a/examples/kitchen-sink/scripts/sqlite-realworld-bench.ts +++ b/examples/kitchen-sink/scripts/sqlite-realworld-bench.ts @@ -1048,7 +1048,7 @@ async function configureLocalRunner(endpoint: string): Promise { const datacenter = datacentersBody.datacenters[0]?.name; if (!datacenter) throw new Error("local engine returned no datacenters"); - const response = await fetch(`${base}/runner-configs/default?namespace=default`, { + const response = await fetch(`${base}/runner-configs/k8s?namespace=default`, { method: "PUT", headers: { Authorization: "Bearer dev", @@ -1064,7 +1064,7 @@ async function configureLocalRunner(endpoint: string): Promise { }); if (!response.ok) { throw new Error( - `failed to configure local default runner: ${response.status} ${await response.text()}`, + `failed to configure local k8s runner: ${response.status} ${await response.text()}`, ); } } @@ -1074,7 +1074,7 @@ async function waitForEnvoy(endpoint: string): Promise { const deadline = Date.now() + 15_000; while (Date.now() < deadline) { - const response = await fetch(`${base}/envoys?namespace=default&name=default`, { + const response = await fetch(`${base}/envoys?namespace=default&name=k8s`, { headers: { Authorization: "Bearer dev" }, }); if (response.ok) { diff --git a/examples/kitchen-sink/src/actors/counter/counter.ts b/examples/kitchen-sink/src/actors/counter/counter.ts index 0cd384f36f..a68a944b76 100644 --- a/examples/kitchen-sink/src/actors/counter/counter.ts +++ b/examples/kitchen-sink/src/actors/counter/counter.ts @@ -1,10 +1,23 @@ -import { actor, event } from "rivetkit"; +import { actor, event, type RivetMessageEvent, type UniversalWebSocket } from "rivetkit"; export const counter = actor({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 5_000, + }, state: { count: 0 }, events: { newCount: event(), }, + onWebSocket(_c, websocket: UniversalWebSocket) { + // Plain echo for the rtt counter-latency harness. Any message in → + // the same payload back out. No state mutation, no awaits — keeps the + // echo path as close to raw WS RTT as possible. + websocket.addEventListener("message", (event: RivetMessageEvent) => { + if (websocket.readyState !== 1) return; + websocket.send(event.data as string | ArrayBuffer); + }); + }, actions: { increment: (c, x: number) => { c.state.count += x; diff --git a/examples/kitchen-sink/src/actors/http/tunnel-stress.ts b/examples/kitchen-sink/src/actors/http/tunnel-stress.ts new file mode 100644 index 0000000000..ed66010965 --- /dev/null +++ b/examples/kitchen-sink/src/actors/http/tunnel-stress.ts @@ -0,0 +1,92 @@ +import { actor, type RivetMessageEvent, type UniversalWebSocket } from "rivetkit"; + +export const tunnelStress = actor({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 5_000, + }, + state: { + connectionCount: 0, + messageCount: 0, + heartbeatCount: 0, + }, + onWebSocket(c, websocket: UniversalWebSocket) { + c.state.connectionCount += 1; + const connectionId = crypto.randomUUID(); + + const sendHeartbeat = () => { + if (websocket.readyState !== 1) return; + + c.state.heartbeatCount += 1; + websocket.send( + JSON.stringify({ + type: "heartbeat", + connectionId, + heartbeatCount: c.state.heartbeatCount, + timestamp: Date.now(), + }), + ); + }; + + const heartbeat = setInterval(sendHeartbeat, 1_000); + sendHeartbeat(); + + websocket.addEventListener("message", async (event: RivetMessageEvent) => { + // Fast-path ping: echo back without touching KV so the client can measure raw RTT + // without the per-message storage write. Used by the counter-latency client's first + // two probes after WS open. + if (typeof event.data === "string") { + let parsed: unknown; + try { + parsed = JSON.parse(event.data); + } catch { + parsed = undefined; + } + if ( + parsed && + typeof parsed === "object" && + (parsed as { type?: unknown }).type === "ping" + ) { + const id = (parsed as { id?: unknown }).id; + if (websocket.readyState === 1) { + websocket.send( + JSON.stringify({ + type: "pong", + connectionId, + id, + timestamp: Date.now(), + }), + ); + } + return; + } + } + + c.state.messageCount += 1; + await c.kv.put("counter", String(c.state.messageCount)); + websocket.send( + JSON.stringify({ + type: "reply", + connectionId, + messageCount: c.state.messageCount, + timestamp: Date.now(), + received: event.data, + }), + ); + }); + + websocket.addEventListener("close", () => { + clearInterval(heartbeat); + c.state.connectionCount -= 1; + }); + }, + actions: { + getStats(c) { + return { + connectionCount: c.state.connectionCount, + messageCount: c.state.messageCount, + heartbeatCount: c.state.heartbeatCount, + }; + }, + }, +}); diff --git a/examples/kitchen-sink/src/actors/state/sqlite-drizzle/drizzle/migrations.js b/examples/kitchen-sink/src/actors/state/sqlite-drizzle/drizzle/migrations.js index 23ed81f26d..752535b815 100644 --- a/examples/kitchen-sink/src/actors/state/sqlite-drizzle/drizzle/migrations.js +++ b/examples/kitchen-sink/src/actors/state/sqlite-drizzle/drizzle/migrations.js @@ -1,4 +1,4 @@ -import journal from './meta/_journal.json'; +import journal from './meta/_journal.json' with { type: 'json' }; import m0000 from './0000_left_wrecking_crew.sql'; export default { @@ -7,4 +7,4 @@ import m0000 from './0000_left_wrecking_crew.sql'; m0000 } } - \ No newline at end of file + diff --git a/examples/kitchen-sink/src/actors/testing/bench-slow-reconnect.ts b/examples/kitchen-sink/src/actors/testing/bench-slow-reconnect.ts new file mode 100644 index 0000000000..877311095d --- /dev/null +++ b/examples/kitchen-sink/src/actors/testing/bench-slow-reconnect.ts @@ -0,0 +1,322 @@ +/** + * Driver for slowReconnectActor. + * + * Warm baseline: + * pnpm slow-reconnect -- --warm --endpoint http://localhost:6420 + * + * Default cold wake loop: + * pnpm slow-reconnect + */ + +import { createClient } from 'rivetkit/client' +import type { registry } from './slow-reconnect-actor' + +interface SlowReconnectStep { + name: string + durationMs: number + rowCount: number +} + +interface SlowReconnectWorkloadResult { + name: string + totalMs: number + steps: SlowReconnectStep[] +} + +interface SlowReconnectResultMessage { + type: 'slow_reconnect_result' + trigger: string + totalMs: number + results: SlowReconnectWorkloadResult[] +} + +interface SlowReconnectErrorMessage { + type: 'slow_reconnect_error' + trigger: string + error: string +} + +type ActorMessage = SlowReconnectResultMessage | SlowReconnectErrorMessage | { type: string } + +const args = process.argv.slice(2) +const endpoint = + readFlagValue('--endpoint') ?? + process.env.RIVET_PUBLIC_ENDPOINT ?? + process.env.RIVET_ENDPOINT ?? + 'http://localhost:6420' +const poolName = readFlagValue('--pool') ?? process.env.RIVET_POOL ?? 'k8s' +const key = readFlagValue('--key') ?? `slow-reconnect-${timestampSlug()}-${randomSuffix()}` +const runs = Number(readFlagValue('--runs') ?? '3') +const wakeLoops = Number( + readFlagValue('--wake-loops') ?? readFlagValue('--sleep-loops') ?? '5', +) +const timeoutMs = Number(readFlagValue('--timeout-ms') ?? '120000') +const mode = readFlagValue('--mode') ?? 'executor_connect' +const staggerHandleMs = Number(readFlagValue('--stagger-handle-ms') ?? '0') +const loop = args.includes('--loop') +const cold = args.includes('--warm') || args.includes('--no-cold') + ? false + : true +const prepare = !args.includes('--no-prepare') +const sleepMs = Number(readFlagValue('--sleep-ms') ?? '1000') +const reconnectDelayMs = Number(readFlagValue('--reconnect-delay-ms') ?? '1000') + +if (mode !== 'executor_connect' && mode !== 'repro_reconnect' && mode !== 'client_resume') { + console.error('Usage: --mode must be executor_connect, repro_reconnect, or client_resume') + process.exit(1) +} +if (!Number.isInteger(runs) || runs < 1) { + console.error('Usage: --runs must be an integer >= 1') + process.exit(1) +} +if (!Number.isInteger(wakeLoops) || wakeLoops < 1) { + console.error('Usage: --wake-loops/--sleep-loops must be an integer >= 1') + process.exit(1) +} + +console.log(`[slow-reconnect] endpoint=${endpoint} pool=${poolName ?? ''} key=${key}`) +console.log( + `[slow-reconnect] runs=${runs} wakeLoops=${loop ? '∞' : wakeLoops} timeout=${ms(timeoutMs)} mode=${mode} staggerHandleMs=${staggerHandleMs}`, +) +console.log( + `[slow-reconnect] cold=${cold} prepare=${prepare} sleepMs=${sleepMs} reconnectDelayMs=${reconnectDelayMs}`, +) + +const client = createClient({ + endpoint, + ...(poolName ? { poolName } : {}), +}) +let stopping = false + +process.on('SIGINT', () => { + console.log('\n[slow-reconnect] SIGINT, stopping...') + stopping = true +}) + +try { + if (prepare) { + await prepareActor() + if (cold) { + await sleepActor(1) + } + } + + let globalRun = 1 + let wakeLoop = 1 + while (!stopping && (loop || wakeLoop <= wakeLoops)) { + console.log(`\n[wake ${wakeLoop}] starting ${runs} reconnect run(s)`) + for (let reconnectRun = 1; !stopping && reconnectRun <= runs; reconnectRun++) { + try { + const result = await runOnce(globalRun, wakeLoop, reconnectRun) + printResult(globalRun, wakeLoop, reconnectRun, result) + } catch (error) { + console.error(`[wake ${wakeLoop} run ${reconnectRun}] failed:`, error) + if (!loop) { + throw error + } + } + + globalRun++ + if (reconnectRun < runs && !stopping && reconnectDelayMs > 0) { + await delay(reconnectDelayMs) + } + } + + wakeLoop++ + if (cold && !stopping && (loop || wakeLoop <= wakeLoops)) { + await sleepActor(wakeLoop) + } + } +} finally { + await client.dispose() +} + +async function prepareActor(): Promise { + const handle = client.slowReconnectActor.getOrCreate([key]) + const startedAt = performance.now() + console.log('\n[prepare] seeding slow reconnect actor...') + const result = await (handle as unknown as { + prepare: () => Promise<{ + seeded: boolean + messages: number + toolCalls: number + threadEvents: number + }> + }).prepare() + console.log( + `[prepare] seeded=${result.seeded} messages=${result.messages} toolCalls=${result.toolCalls} threadEvents=${result.threadEvents} in ${ms(performance.now() - startedAt)}`, + ) +} + +async function sleepActor(nextWakeLoop: number): Promise { + const handle = client.slowReconnectActor.getOrCreate([key]) + const startedAt = performance.now() + console.log(`\n[wake ${nextWakeLoop}] sleeping actor...`) + await (handle as unknown as { sleep: () => Promise }).sleep() + console.log( + `[wake ${nextWakeLoop}] sleep action returned in ${ms(performance.now() - startedAt)}; waiting ${ms(sleepMs)} before reconnect`, + ) + await delay(sleepMs) +} + +async function runOnce( + index: number, + wakeLoop: number, + reconnectRun: number, +): Promise { + const handle = client.slowReconnectActor.getOrCreate([key]) + const startedAt = performance.now() + console.log(`[wake ${wakeLoop} run ${reconnectRun}] opening websocket...`) + const ws = await handle.webSocket('/', undefined, { skipReadyWait: true }) + if (!ws) { + throw new Error('slowReconnectActor did not return a WebSocket') + } + try { + await waitForOpen(ws) + console.log( + `[wake ${wakeLoop} run ${reconnectRun}] websocket open in ${ms(performance.now() - startedAt)}`, + ) + const resultPromise = waitForResult(ws, timeoutMs) + ws.send(JSON.stringify(buildRequest(index))) + return await resultPromise + } finally { + ws.close() + } +} + +function buildRequest(index: number): object { + if (mode === 'client_resume') { + return { type: 'client_resume', version: 0 } + } + if (mode === 'repro_reconnect') { + return { + type: 'repro_reconnect', + clientId: `slow-reconnect-client-${index}`, + staggerHandleMs, + } + } + return { + type: 'executor_connect', + clientId: `slow-reconnect-client-${index}`, + executorType: 'local-client', + capabilities: {}, + } +} + +function waitForResult(ws: WebSocket, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error(`Timed out after ${timeoutMs}ms waiting for slowReconnectActor result`)) + try { + ws.close() + } catch {} + }, timeoutMs) + + const cleanup = () => clearTimeout(timeout) + + ws.addEventListener('message', (event: MessageEvent) => { + const data = typeof event.data === 'string' ? event.data : '' + if (data === 'pong') { + return + } + let message: ActorMessage + try { + message = JSON.parse(data) as ActorMessage + } catch { + console.log(`[slow-reconnect] <<< ${data.slice(0, 200)}`) + return + } + if (message.type === 'executor_connected') { + console.log('[slow-reconnect] <<< executor_connected') + return + } + if (message.type === 'slow_reconnect_error') { + cleanup() + reject(new Error((message as SlowReconnectErrorMessage).error)) + return + } + if (message.type === 'slow_reconnect_result') { + cleanup() + resolve(message as SlowReconnectResultMessage) + } + }) + + ws.addEventListener('close', (event: CloseEvent) => { + cleanup() + reject( + new Error( + `WebSocket closed before result: code=${event.code} reason=${event.reason || ''}`, + ), + ) + }) + ws.addEventListener('error', () => { + cleanup() + reject(new Error('WebSocket failed while waiting for result')) + }) + }) +} + +async function waitForOpen(ws: WebSocket): Promise { + if (ws.readyState === WebSocket.OPEN) { + return + } + await new Promise((resolve, reject) => { + ws.addEventListener('open', () => resolve(), { once: true }) + ws.addEventListener('error', () => reject(new Error('WebSocket failed to open')), { + once: true, + }) + ws.addEventListener('close', () => reject(new Error('WebSocket closed before open')), { + once: true, + }) + }) +} + +function printResult( + index: number, + wakeLoop: number, + reconnectRun: number, + message: SlowReconnectResultMessage, +): void { + console.log( + `\n[wake ${wakeLoop} run ${reconnectRun} global ${index}] trigger=${message.trigger} total=${ms(message.totalMs)}`, + ) + for (const workload of message.results) { + console.log(` ${workload.name.padEnd(28)} total=${ms(workload.totalMs)}`) + for (const step of workload.steps) { + console.log( + ` ${step.name.padEnd(36)} ${ms(step.durationMs).padStart(8)} rows=${step.rowCount}`, + ) + } + } +} + +function readFlagValue(flag: string): string | undefined { + const prefix = `${flag}=` + const equalsValue = args.find((arg) => arg.startsWith(prefix)) + if (equalsValue) { + return equalsValue.slice(prefix.length) + } + const index = args.indexOf(flag) + if (index === -1) { + return undefined + } + return args[index + 1] +} + +function ms(value: number): string { + return `${Math.round(value)}ms` +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms))) +} + +function timestampSlug(): string { + const d = new Date() + const pad = (n: number) => String(n).padStart(2, '0') + return `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}T${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}` +} + +function randomSuffix(): string { + return Math.random().toString(36).slice(2, 8) +} diff --git a/examples/kitchen-sink/src/actors/testing/load-test-agent-2.ts b/examples/kitchen-sink/src/actors/testing/load-test-agent-2.ts new file mode 100644 index 0000000000..8981447ddd --- /dev/null +++ b/examples/kitchen-sink/src/actors/testing/load-test-agent-2.ts @@ -0,0 +1,1524 @@ +import { actor, type RivetMessageEvent, type UniversalWebSocket } from "rivetkit"; +import { db } from "rivetkit/db"; + +type AgentConcurrent2Request = + | { type: "agent2_resume"; version: number } + | { type: "agent2_connect"; clientId: string; staggerHandleMs?: number } + | { type: "force_sleep" } + | { type: "ping"; id?: number }; + +interface AgentConcurrent2Step { + name: string; + durationMs: number; + rowCount: number; +} + +interface AgentConcurrent2QueryStats { + total: number; + reads: number; + mutations: number; + tx: number; + other: number; + rows: number; + errors: number; + slow: number; + maxMs: number; + maxStep: string; + byOperation: Record; + byTable: Record; +} + +interface AgentConcurrent2StatsSnapshot { + wakeIndex: number; + actorIteration: number; + wakeIteration: number; + cycle: AgentConcurrent2QueryStats; + wake: AgentConcurrent2QueryStats; + actor: AgentConcurrent2QueryStats; +} + +interface AgentConcurrent2WorkloadResult { + name: string; + totalMs: number; + steps: AgentConcurrent2Step[]; +} + +interface AgentConcurrent2ResultMessage { + type: "agent2_result"; + trigger: AgentConcurrent2Request["type"]; + totalMs: number; + results: AgentConcurrent2WorkloadResult[]; + stats: AgentConcurrent2StatsSnapshot; +} + +interface AgentConcurrent2ErrorMessage { + type: "agent2_error"; + trigger: AgentConcurrent2Request["type"] | "unknown"; + error: string; + stats?: AgentConcurrent2StatsSnapshot; +} + +interface AgentConcurrent2Vars { + sql: AgentConcurrent2Db | null; + wakeStats: AgentConcurrent2QueryStats | null; + wakeStartedAt: number | null; + wakeIteration: number; +} + +interface RawRivetDB { + execute: ( + query: string, + ...args: unknown[] + ) => Promise[]>; +} + +type SQLPrimitive = string | number | boolean | null; + +interface AgentConcurrent2State { + runCount: number; + wakeCount: number; + queryStats: AgentConcurrent2QueryStats; +} + +interface AgentConcurrent2QueryStatsSet { + cycle: AgentConcurrent2QueryStats; + wake: AgentConcurrent2QueryStats; + actor: AgentConcurrent2QueryStats; +} + +interface AgentConcurrent2Runtime { + sql: AgentConcurrent2Db; + wakeStats: AgentConcurrent2QueryStats; + vars: AgentConcurrent2Vars; +} + +type AgentConcurrent2Db = (>( + query: string, + ...values: SQLPrimitive[] +) => Promise) & { + withTransaction( + stats: AgentConcurrent2QueryStatsSet, + fn: (tx: AgentConcurrent2Db) => Promise, + ): Promise; +}; + +class AsyncMutex { + private locked = false; + private waiters: Array<() => void> = []; + + async acquire(): Promise { + if (!this.locked) { + this.locked = true; + return; + } + await new Promise((resolve) => this.waiters.push(resolve)); + this.locked = true; + } + + release(): void { + const next = this.waiters.shift(); + if (next) { + next(); + return; + } + this.locked = false; + } +} + +function createSerializedDb( + execute: >( + query: string, + ...values: SQLPrimitive[] + ) => Promise, +): AgentConcurrent2Db { + const mutex = new AsyncMutex(); + let activeTransaction: AgentConcurrent2Db | null = null; + + const createTransactionDb = (): AgentConcurrent2Db => { + const tx = Object.assign( + >( + query: string, + ...values: SQLPrimitive[] + ) => execute(query, ...values), + { + withTransaction: async ( + _stats: AgentConcurrent2QueryStatsSet, + fn: (tx: AgentConcurrent2Db) => Promise, + ): Promise => fn(tx), + }, + ); + return tx; + }; + + const queryWithMutex = async >( + query: string, + ...values: SQLPrimitive[] + ): Promise => { + await mutex.acquire(); + try { + return await execute(query, ...values); + } finally { + mutex.release(); + } + }; + + return Object.assign(queryWithMutex, { + withTransaction: async ( + stats: AgentConcurrent2QueryStatsSet, + fn: (tx: AgentConcurrent2Db) => Promise, + ): Promise => { + if (activeTransaction) { + return fn(activeTransaction); + } + await mutex.acquire(); + const tx = createTransactionDb(); + try { + await executeTrackedQuery(execute, stats, "transaction-begin", "BEGIN"); + activeTransaction = tx; + try { + const result = await fn(tx); + activeTransaction = null; + await executeTrackedQuery(execute, stats, "transaction-commit", "COMMIT"); + return result; + } catch (error) { + activeTransaction = null; + await executeTrackedQuery( + execute, + stats, + "transaction-rollback", + "ROLLBACK", + ); + throw error; + } + } finally { + activeTransaction = null; + mutex.release(); + } + }, + }); +} + +const MESSAGE_COUNT = 84; +const MESSAGE_TOOL_REF_COUNT = 122; +const TOOL_CALL_COUNT = 61; +const EXECUTOR_TOOL_COUNT = 42; +const THREAD_EVENT_COUNT = 233; + +const MESSAGE_CONTENT_BYTES = 2_600; +const THREAD_EVENT_PAYLOAD_BYTES = 1_000; +const TOOL_CALL_RESULT_BYTES = 2_700; +const EXECUTOR_TOOL_SCHEMA_BYTES = 550; +const SLOW_QUERY_MS = 1_000; + +function send( + websocket: UniversalWebSocket, + message: AgentConcurrent2ResultMessage | AgentConcurrent2ErrorMessage | object, +): void { + if (websocket.readyState === 1) { + websocket.send(JSON.stringify(message)); + } +} + +export const loadTestAgent2 = actor({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 1_000, + }, + state: { + runCount: 0, + wakeCount: 0, + queryStats: createAgentConcurrent2QueryStats(), + } as AgentConcurrent2State, + db: db({ + onMigrate: async (database) => { + await createAgentConcurrent2Schema(database); + await seedAgentConcurrent2Data(database); + }, + }), + vars: { + sql: null, + wakeStats: null, + wakeStartedAt: null, + wakeIteration: 0, + } as AgentConcurrent2Vars, + onWebSocket: (c, websocket: UniversalWebSocket) => { + send(websocket, { + type: "connected", + timestamp: Date.now(), + }); + + websocket.addEventListener("message", (event: RivetMessageEvent) => { + const promise = handleAgentConcurrent2Message(c, websocket, event.data); + void c.keepAwake(promise); + }); + }, + actions: { + run: async (c, clientId?: string) => { + const runtime = ensureAgentConcurrent2Runtime(c); + c.state.runCount++; + runtime.vars.wakeIteration++; + const cycleStats = createAgentConcurrent2QueryStats(); + const stats = createAgentConcurrent2StatsSet( + cycleStats, + runtime.wakeStats, + c.state.queryStats, + ); + const result = await runAgentConcurrent2Workload( + runtime.sql, + clientId ?? `agent2-action-${c.state.runCount}`, + 0, + stats, + ); + return { + ...result, + stats: snapshotAgentConcurrent2Stats(c, cycleStats), + }; + }, + getRunCount: (c) => c.state.runCount, + sleep: (c) => { + c.sleep(); + return true; + }, + }, +}); + +async function handleAgentConcurrent2Message( + c: { + db: RawRivetDB; + vars: AgentConcurrent2Vars; + state: AgentConcurrent2State; + sleep: () => void; + }, + websocket: UniversalWebSocket, + data: unknown, +): Promise { + let trigger: AgentConcurrent2Request["type"] | "unknown" = "unknown"; + let cycleStats: AgentConcurrent2QueryStats | null = null; + try { + const request = parseAgentConcurrent2Request(data); + trigger = request.type; + + if (request.type === "ping") { + send(websocket, { + type: "pong", + id: request.id, + timestamp: Date.now(), + }); + return; + } + + if (request.type === "force_sleep") { + send(websocket, { type: "sleeping", timestamp: Date.now() }); + c.sleep(); + return; + } + + const runtime = ensureAgentConcurrent2Runtime(c); + c.state.runCount++; + runtime.vars.wakeIteration++; + cycleStats = createAgentConcurrent2QueryStats(); + const stats = createAgentConcurrent2StatsSet( + cycleStats, + runtime.wakeStats, + c.state.queryStats, + ); + + if (request.type === "agent2_resume") { + const startedAt = performance.now(); + const result = await runCatchupSnapshot( + runtime.sql, + request.version, + stats, + ); + send(websocket, { + type: "agent2_result", + trigger: request.type, + totalMs: Math.round(performance.now() - startedAt), + results: [result], + stats: snapshotAgentConcurrent2Stats(c, cycleStats), + }); + return; + } + + const result = await runAgentConcurrent2Workload( + runtime.sql, + request.clientId, + request.staggerHandleMs ?? 0, + stats, + ); + send(websocket, { + type: "agent2_result", + trigger: request.type, + ...result, + stats: snapshotAgentConcurrent2Stats(c, cycleStats), + }); + } catch (error) { + send(websocket, { + type: "agent2_error", + trigger, + error: error instanceof Error ? error.message : String(error), + ...(cycleStats ? { stats: snapshotAgentConcurrent2Stats(c, cycleStats) } : {}), + }); + } +} + +function parseAgentConcurrent2Request(data: unknown): AgentConcurrent2Request { + if (typeof data !== "string") { + throw new Error("agent concurrent 2 request must be a string"); + } + const parsed = JSON.parse(data) as unknown; + if (!parsed || typeof parsed !== "object") { + throw new Error("agent concurrent 2 request must be an object"); + } + const request = parsed as Record; + + if (request.type === "ping") { + return { + type: "ping", + ...(typeof request.id === "number" ? { id: request.id } : {}), + }; + } + if (request.type === "force_sleep") { + return { type: "force_sleep" }; + } + if (request.type === "agent2_resume") { + return { type: "agent2_resume", version: numberField(request, "version") }; + } + if (request.type === "agent2_connect") { + return { + type: "agent2_connect", + clientId: stringField(request, "clientId"), + ...(typeof request.staggerHandleMs === "number" + ? { staggerHandleMs: request.staggerHandleMs } + : {}), + }; + } + throw new Error(`unknown agent concurrent 2 request type: ${String(request.type)}`); +} + +function stringField(record: Record, field: string): string { + const value = record[field]; + if (typeof value !== "string" || value.length === 0) { + throw new Error(`agent concurrent 2 request ${field} must be a string`); + } + return value; +} + +function numberField(record: Record, field: string): number { + const value = record[field]; + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`agent concurrent 2 request ${field} must be a finite number`); + } + return value; +} + +function createAgentConcurrent2Db(db: RawRivetDB): AgentConcurrent2Db { + return createSerializedDb(async >( + query: string, + ...values: SQLPrimitive[] + ): Promise => { + const converted = values.map((value) => + typeof value === "boolean" ? (value ? 1 : 0) : value, + ); + return (await db.execute(query, ...converted)) as T[]; + }); +} + +function ensureAgentConcurrent2Runtime(c: { + db: RawRivetDB; + vars: AgentConcurrent2Vars; + state: AgentConcurrent2State; +}): AgentConcurrent2Runtime { + c.vars.sql ??= createAgentConcurrent2Db(c.db); + c.state.queryStats ??= createAgentConcurrent2QueryStats(); + c.state.wakeCount ??= 0; + if (!c.vars.wakeStats) { + c.vars.wakeStats = createAgentConcurrent2QueryStats(); + c.vars.wakeStartedAt = Date.now(); + c.vars.wakeIteration = 0; + c.state.wakeCount++; + } + return { + sql: c.vars.sql, + wakeStats: c.vars.wakeStats, + vars: c.vars, + }; +} + +function createAgentConcurrent2QueryStats(): AgentConcurrent2QueryStats { + return { + total: 0, + reads: 0, + mutations: 0, + tx: 0, + other: 0, + rows: 0, + errors: 0, + slow: 0, + maxMs: 0, + maxStep: "", + byOperation: {}, + byTable: {}, + }; +} + +function createAgentConcurrent2StatsSet( + cycle: AgentConcurrent2QueryStats, + wake: AgentConcurrent2QueryStats, + actor: AgentConcurrent2QueryStats, +): AgentConcurrent2QueryStatsSet { + return { cycle, wake, actor }; +} + +function snapshotAgentConcurrent2Stats( + c: { vars: AgentConcurrent2Vars; state: AgentConcurrent2State }, + cycle: AgentConcurrent2QueryStats, +): AgentConcurrent2StatsSnapshot { + return { + wakeIndex: c.state.wakeCount, + actorIteration: c.state.runCount, + wakeIteration: c.vars.wakeIteration, + cycle: cloneAgentConcurrent2QueryStats(cycle), + wake: cloneAgentConcurrent2QueryStats( + c.vars.wakeStats ?? createAgentConcurrent2QueryStats(), + ), + actor: cloneAgentConcurrent2QueryStats(c.state.queryStats), + }; +} + +function cloneAgentConcurrent2QueryStats( + stats: AgentConcurrent2QueryStats, +): AgentConcurrent2QueryStats { + return { + total: stats.total, + reads: stats.reads, + mutations: stats.mutations, + tx: stats.tx, + other: stats.other, + rows: stats.rows, + errors: stats.errors, + slow: stats.slow, + maxMs: stats.maxMs, + maxStep: stats.maxStep, + byOperation: { ...stats.byOperation }, + byTable: { ...stats.byTable }, + }; +} + +async function runAgentConcurrent2Workload( + sql: AgentConcurrent2Db, + clientId: string, + staggerHandleMs: number, + stats: AgentConcurrent2QueryStatsSet, +): Promise> { + const startedAt = performance.now(); + const buildToolPlanContext = runBuildToolPlanContext(sql, stats); + const catchupSnapshot = runCatchupSnapshot(sql, 0, stats); + const recoverToolCalls = runRecoverToolCalls(sql, stats); + const mutationMix = runMutationMix(sql, clientId, stats); + const handleExecutorConnect = delay(staggerHandleMs).then(() => + runHandleClientConnect(sql, clientId, stats), + ); + + const results = await Promise.all([ + handleExecutorConnect, + buildToolPlanContext, + catchupSnapshot, + recoverToolCalls, + mutationMix, + ]); + return { + totalMs: Math.round(performance.now() - startedAt), + results, + }; +} + +async function runHandleClientConnect( + sql: AgentConcurrent2Db, + clientId: string, + stats: AgentConcurrent2QueryStatsSet, +): Promise { + const startedAt = performance.now(); + const steps: AgentConcurrent2Step[] = []; + const nextSeq = await sql.withTransaction(stats, async (tx) => { + const latestExecutor = await timedQuery( + tx, + stats, + steps, + "load-latest-executor-id", + `SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1`, + ); + const latestExecutorId = String( + latestExecutor[0]?.executor_id ?? "seed-executor", + ); + await timedQuery( + tx, + stats, + steps, + "select-cached-executor-tools", + `SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`, + latestExecutorId, + ); + const executorType = await timedQuery( + tx, + stats, + steps, + "select-executor-type", + `SELECT value FROM thread_meta_kv WHERE key = 'executor_type'`, + ); + if (!executorType[0]?.value) { + await timedQuery( + tx, + stats, + steps, + "set-executor-type", + `INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES ('executor_type', ?, ?)`, + "local-client", + new Date().toISOString(), + ); + } + const sandboxIntent = await timedQuery( + tx, + stats, + steps, + "select-workspace-intent", + `SELECT value FROM thread_meta_kv WHERE key = 'workspace_intent'`, + ); + if (hasPendingLaunch(sandboxIntent[0]?.value)) { + await timedQuery( + tx, + stats, + steps, + "clear-pending-launch", + `UPDATE thread_meta_kv SET value = ?, updated_at = ? WHERE key = 'workspace_intent'`, + JSON.stringify({ spec: null, pendingLaunch: null }), + new Date().toISOString(), + ); + } + const seqRows = await timedQuery( + tx, + stats, + steps, + "select-next-thread-event-seq", + `SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events`, + ); + const seq = Number(seqRows[0]?.seq ?? 1); + await timedQuery( + tx, + stats, + steps, + "insert-client-connected-event", + `INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`, + seq, + "client_connected", + JSON.stringify({ type: "client_connected", clientId }), + new Date().toISOString(), + ); + return seq; + }); + steps.push({ + name: "transaction-total", + durationMs: Math.round(performance.now() - startedAt), + rowCount: nextSeq, + }); + return { + name: "handle-client-connect", + totalMs: Math.round(performance.now() - startedAt), + steps, + }; +} + +async function runBuildToolPlanContext( + sql: AgentConcurrent2Db, + stats: AgentConcurrent2QueryStatsSet, +): Promise { + const startedAt = performance.now(); + const steps: AgentConcurrent2Step[] = []; + const latestExecutor = await timedQuery( + sql, + stats, + steps, + "load-latest-executor-id", + `SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1`, + ); + const latestExecutorId = String(latestExecutor[0]?.executor_id ?? "seed-executor"); + await timedQuery( + sql, + stats, + steps, + "select-executor-tools", + `SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`, + latestExecutorId, + ); + await timedQuery( + sql, + stats, + steps, + "count-uncancelled-top-level", + `SELECT COUNT(*) as count FROM messages WHERE cancelled = 0 AND parent_tool_use_id IS NULL`, + ); + const unresolvedRows = await timedQuery( + sql, + stats, + steps, + "find-unresolved-assistant-message", + `SELECT m.* + FROM message_tool_refs AS tool_use + JOIN messages AS m + ON m.message_id = tool_use.assistant_message_id + WHERE tool_use.block_type = 'tool_use' + AND tool_use.cancelled = 0 + AND m.cancelled = 0 + AND m.role = 'assistant' + AND m.parent_tool_use_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM message_tool_refs AS tool_result + JOIN messages AS tool_result_message + ON tool_result_message.message_id = tool_result.source_message_id + WHERE tool_result.assistant_message_id = tool_use.assistant_message_id + AND tool_result.block_type = 'tool_result' + AND tool_result.cancelled = 0 + AND tool_result.tool_use_id = tool_use.tool_use_id + AND tool_result_message.parent_tool_use_id IS NULL + ) + GROUP BY m.message_id + ORDER BY m.created_at DESC + LIMIT 1`, + ); + const unresolvedMessageId = unresolvedRows[0]?.message_id; + if (typeof unresolvedMessageId === "string") { + await timedQuery( + sql, + stats, + steps, + "get-persisted-tool-result-ids", + `SELECT tool_result.tool_use_id + FROM message_tool_refs AS tool_result + JOIN messages AS tool_result_message + ON tool_result_message.message_id = tool_result.source_message_id + WHERE tool_result.assistant_message_id = ? + AND tool_result.block_type = 'tool_result' + AND tool_result.cancelled = 0 + AND tool_result_message.parent_tool_use_id IS NULL`, + unresolvedMessageId, + ); + await timedQuery( + sql, + stats, + steps, + "get-tool-calls-by-message-id", + `SELECT * FROM tool_calls WHERE message_id = ?`, + unresolvedMessageId, + ); + } + await timedQuery( + sql, + stats, + steps, + "is-last-message-cancelled-assistant", + `SELECT role, cancelled FROM messages + WHERE parent_tool_use_id IS NULL + ORDER BY created_at DESC + LIMIT 1`, + ); + await timedQuery( + sql, + stats, + steps, + "get-last-uncancelled", + `SELECT m.* FROM messages m + WHERE m.cancelled = 0 AND m.parent_tool_use_id IS NULL + ORDER BY m.created_at DESC + LIMIT 1`, + ); + return { + name: "build-tool-plan-context", + totalMs: Math.round(performance.now() - startedAt), + steps, + }; +} + +async function runCatchupSnapshot( + sql: AgentConcurrent2Db, + version: number, + stats: AgentConcurrent2QueryStatsSet, +): Promise { + const startedAt = performance.now(); + const steps: AgentConcurrent2Step[] = []; + await Promise.all([ + timedQuery( + sql, + stats, + steps, + "thread-events-list-since-version", + `SELECT seq, event_type, payload, created_at FROM thread_events WHERE seq > ? ORDER BY seq ASC`, + version, + ), + timedQuery( + sql, + stats, + steps, + "environment-snapshot", + `SELECT snapshot FROM environment_snapshot WHERE id = 1`, + ), + timedQuery( + sql, + stats, + steps, + "thread-settings-snapshot", + `SELECT settings FROM thread_settings_snapshot WHERE id = 1`, + ), + timedQuery( + sql, + stats, + steps, + "retry-state", + `SELECT * FROM retry_state WHERE id = 1`, + ), + timedQuery( + sql, + stats, + steps, + "queued-messages", + `SELECT * FROM queued_messages ORDER BY created_at ASC`, + ), + timedQuery( + sql, + stats, + steps, + "executor-artifacts", + `SELECT artifact_key, data_type, length(content_base64) AS bytes, tool_call_id, updated_at FROM executor_artifacts ORDER BY updated_at ASC`, + ), + timedQuery( + sql, + stats, + steps, + "tool-approvals", + `SELECT * FROM tool_approvals ORDER BY timestamp ASC`, + ), + timedQuery( + sql, + stats, + steps, + "compaction-summaries", + `SELECT cut_message_id, created_at FROM compaction_summaries ORDER BY created_at ASC`, + ), + timedQuery( + sql, + stats, + steps, + "executor-status", + `SELECT value FROM thread_meta_kv WHERE key = 'executor_status'`, + ), + ]); + steps.sort((a, b) => b.durationMs - a.durationMs); + return { + name: "catchup-snapshot", + totalMs: Math.round(performance.now() - startedAt), + steps, + }; +} + +async function runRecoverToolCalls( + sql: AgentConcurrent2Db, + stats: AgentConcurrent2QueryStatsSet, +): Promise { + const startedAt = performance.now(); + const steps: AgentConcurrent2Step[] = []; + await timedQuery( + sql, + stats, + steps, + "hydrate-tool-progress", + `SELECT id, progress + FROM tool_calls + WHERE progress IS NOT NULL + AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + ); + await timedQuery( + sql, + stats, + steps, + "get-pending-tool-calls", + `SELECT * FROM tool_calls + WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running') + ORDER BY issued_at ASC`, + ); + await timedQuery( + sql, + stats, + steps, + "get-next-tool-expiry", + `SELECT MIN(expires_at) AS expires_at + FROM tool_calls + WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + ); + return { + name: "recover-tool-calls", + totalMs: Math.round(performance.now() - startedAt), + steps, + }; +} + +async function runMutationMix( + sql: AgentConcurrent2Db, + clientId: string, + stats: AgentConcurrent2QueryStatsSet, +): Promise { + const startedAt = performance.now(); + const steps: AgentConcurrent2Step[] = []; + const writeCount = await sql.withTransaction(stats, async (tx) => { + const now = new Date().toISOString(); + const suffix = safeId(clientId); + const seqRows = await timedQuery( + tx, + stats, + steps, + "select-max-thread-event-seq", + `SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events`, + ); + const seq = Number(seqRows[0]?.seq ?? 1); + const lastMessageRows = await timedQuery( + tx, + stats, + steps, + "select-last-message-created-at", + `SELECT MAX(created_at) AS created_at FROM messages`, + ); + const latestToolRows = await timedQuery( + tx, + stats, + steps, + "select-existing-tool-call", + `SELECT id FROM tool_calls ORDER BY issued_at DESC LIMIT 1`, + ); + await timedQuery( + tx, + stats, + steps, + "select-sandbox-row", + `SELECT sandbox_id, restart_attempts, traffic_access_token, project_id, repository_url, additional_repositories, setup + FROM e2b_sandbox + WHERE id = 1`, + ); + + const messageIdValue = `agent2-message-${suffix}-${seq}`; + const toolUseIdValue = `agent2-tool-${suffix}-${seq}`; + const toolCallIdValue = `agent2-call-${suffix}-${seq}`; + const latestToolCallId = String(latestToolRows[0]?.id ?? toolUseID(1)); + const lastCreatedAt = String(lastMessageRows[0]?.created_at ?? now); + + await timedQuery( + tx, + stats, + steps, + "upsert-agent-state", + `INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES (?, ?, ?)`, + "last_agent_state", + JSON.stringify({ status: "working", clientId, lastCreatedAt }), + now, + ); + await timedQuery( + tx, + stats, + steps, + "insert-work-event", + `INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`, + seq, + "message_added", + JSON.stringify({ type: "message_added", messageId: messageIdValue }), + now, + ); + await timedQuery( + tx, + stats, + steps, + "insert-message", + `INSERT INTO messages (role, content, meta, user_state, message_id, created_at, cancelled, parent_tool_use_id, tool_result_for_message_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "assistant", + "agent concurrent 2 mutation payload", + JSON.stringify({ clientId, seq }), + null, + messageIdValue, + now, + 0, + null, + null, + ); + await timedQuery( + tx, + stats, + steps, + "delete-message-tool-refs", + `DELETE FROM message_tool_refs WHERE source_message_id = ?`, + messageIdValue, + ); + await timedQuery( + tx, + stats, + steps, + "insert-message-added-event", + `INSERT OR IGNORE INTO message_added_events (message_id, seq) VALUES (?, ?)`, + messageIdValue, + seq, + ); + await timedQuery( + tx, + stats, + steps, + "insert-message-tool-ref", + `INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled) + VALUES (?, ?, ?, ?, ?)`, + messageIdValue, + messageIdValue, + toolUseIdValue, + "tool_use", + 0, + ); + await timedQuery( + tx, + stats, + steps, + "insert-tool-call", + `INSERT OR IGNORE INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + toolCallIdValue, + `provider-${toolCallIdValue}`, + "tool_1", + JSON.stringify({ path: `/tmp/${toolCallIdValue}` }), + "seed-executor", + messageIdValue, + now, + null, + "running", + null, + JSON.stringify({ pct: 0.5, clientId }), + null, + ); + await timedQuery( + tx, + stats, + steps, + "update-tool-call-progress", + `UPDATE tool_calls SET progress = ? WHERE id = ? AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + JSON.stringify({ pct: 0.75, clientId, updatedAt: now }), + toolCallIdValue, + ); + await timedQuery( + tx, + stats, + steps, + "update-existing-tool-call-progress", + `UPDATE tool_calls SET progress = ? WHERE id = ? AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + JSON.stringify({ pct: 0.25, clientId, updatedAt: now }), + latestToolCallId, + ); + + return seq; + }); + steps.push({ + name: "transaction-total", + durationMs: Math.round(performance.now() - startedAt), + rowCount: writeCount, + }); + return { + name: "mutation-mix", + totalMs: Math.round(performance.now() - startedAt), + steps, + }; +} + +async function timedQuery>( + sql: AgentConcurrent2Db, + stats: AgentConcurrent2QueryStatsSet, + steps: AgentConcurrent2Step[], + name: string, + query: string, + ...values: SQLPrimitive[] +): Promise { + const startedAt = performance.now(); + try { + const rows = await sql(query, ...values); + const durationMs = Math.round(performance.now() - startedAt); + recordAgentConcurrent2Query(stats, name, query, durationMs, rows.length, false); + steps.push({ + name, + durationMs, + rowCount: rows.length, + }); + return rows; + } catch (error) { + const durationMs = Math.round(performance.now() - startedAt); + recordAgentConcurrent2Query(stats, name, query, durationMs, 0, true); + throw error; + } +} + +async function executeTrackedQuery>( + execute: >( + query: string, + ...values: SQLPrimitive[] + ) => Promise, + stats: AgentConcurrent2QueryStatsSet, + name: string, + query: string, + ...values: SQLPrimitive[] +): Promise { + const startedAt = performance.now(); + try { + const rows = await execute(query, ...values); + recordAgentConcurrent2Query( + stats, + name, + query, + Math.round(performance.now() - startedAt), + rows.length, + false, + ); + return rows; + } catch (error) { + recordAgentConcurrent2Query( + stats, + name, + query, + Math.round(performance.now() - startedAt), + 0, + true, + ); + throw error; + } +} + +function recordAgentConcurrent2Query( + stats: AgentConcurrent2QueryStatsSet, + name: string, + query: string, + durationMs: number, + rowCount: number, + failed: boolean, +): void { + const classification = classifyAgentConcurrent2Query(query); + for (const target of [stats.cycle, stats.wake, stats.actor]) { + target.total++; + target.rows += rowCount; + if (failed) target.errors++; + if (durationMs >= SLOW_QUERY_MS) target.slow++; + if (durationMs > target.maxMs) { + target.maxMs = durationMs; + target.maxStep = `${name}:${classification.table}`; + } + target.byOperation[classification.operation] = + (target.byOperation[classification.operation] ?? 0) + 1; + target.byTable[classification.table] = + (target.byTable[classification.table] ?? 0) + 1; + if (classification.kind === "read") { + target.reads++; + } else if (classification.kind === "mutation") { + target.mutations++; + } else if (classification.kind === "tx") { + target.tx++; + } else { + target.other++; + } + } +} + +function classifyAgentConcurrent2Query(query: string): { + operation: string; + kind: "read" | "mutation" | "tx" | "other"; + table: string; +} { + const normalized = query.trim().replace(/\s+/g, " "); + const operation = normalized.match(/^([a-z]+)/i)?.[1]?.toLowerCase() ?? "other"; + const table = extractAgentConcurrent2Table(normalized, operation); + if (operation === "select") { + return { operation, kind: "read", table }; + } + if ( + operation === "insert" || + operation === "update" || + operation === "delete" || + operation === "replace" + ) { + return { operation, kind: "mutation", table }; + } + if (operation === "begin" || operation === "commit" || operation === "rollback") { + return { operation, kind: "tx", table }; + } + return { operation, kind: "other", table }; +} + +function extractAgentConcurrent2Table(query: string, operation: string): string { + const lower = query.toLowerCase(); + if (operation === "select") { + return firstMatch(lower, /\bfrom\s+([a-z0-9_]+)/) ?? "unknown"; + } + if (operation === "insert" || operation === "replace") { + return firstMatch(lower, /\binto\s+([a-z0-9_]+)/) ?? "unknown"; + } + if (operation === "update") { + return firstMatch(lower, /\bupdate\s+([a-z0-9_]+)/) ?? "unknown"; + } + if (operation === "delete") { + return firstMatch(lower, /\bfrom\s+([a-z0-9_]+)/) ?? "unknown"; + } + if (operation === "begin" || operation === "commit" || operation === "rollback") { + return "transaction"; + } + return "unknown"; +} + +function firstMatch(value: string, pattern: RegExp): string | null { + return pattern.exec(value)?.[1] ?? null; +} + +function hasPendingLaunch(value: unknown): boolean { + if (typeof value !== "string" || value.length === 0) { + return false; + } + try { + const parsed = JSON.parse(value) as { pendingLaunch?: unknown }; + return parsed.pendingLaunch !== null && parsed.pendingLaunch !== undefined; + } catch { + return false; + } +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms))); +} + +async function createAgentConcurrent2Schema(database: RawRivetDB): Promise { + await database.execute(`CREATE TABLE IF NOT EXISTS executor_tools ( + executor_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + schema TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (executor_id, tool_name) + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_executor_tools_executor ON executor_tools(executor_id)`, + ); + await database.execute(`CREATE TABLE IF NOT EXISTS thread_meta_kv ( + key TEXT PRIMARY KEY, + value TEXT, + updated_at TEXT NOT NULL + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS thread_events ( + seq INTEGER PRIMARY KEY, + event_type TEXT NOT NULL, + payload TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_thread_events_seq ON thread_events(seq)`, + ); + await database.execute(`CREATE TABLE IF NOT EXISTS message_added_events ( + message_id TEXT PRIMARY KEY, + seq INTEGER NOT NULL UNIQUE + )`); + await database.execute(`CREATE TABLE IF NOT EXISTS messages ( + message_id TEXT PRIMARY KEY, + role TEXT NOT NULL CHECK(role IN ('user', 'assistant', 'info')), + content TEXT NOT NULL, + meta TEXT, + user_state TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + cancelled INTEGER NOT NULL DEFAULT 0, + read_at TEXT, + parent_tool_use_id TEXT, + tool_result_for_message_id TEXT + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_role_cancelled_created_at ON messages(parent_tool_use_id, role, cancelled, created_at DESC)`, + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_cancelled_created_at ON messages(parent_tool_use_id, cancelled, created_at DESC)`, + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_created_at ON messages(parent_tool_use_id, created_at DESC)`, + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_role_created_at ON messages(role, created_at)`, + ); + await database.execute(`CREATE TABLE IF NOT EXISTS message_tool_refs ( + source_message_id TEXT NOT NULL, + assistant_message_id TEXT NOT NULL, + tool_use_id TEXT NOT NULL, + block_type TEXT NOT NULL CHECK(block_type IN ('tool_use', 'tool_result')), + cancelled INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (source_message_id, block_type, tool_use_id) + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_assistant_lookup ON message_tool_refs(assistant_message_id, block_type, cancelled, tool_use_id)`, + ); + await database.execute( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_message_tool_refs_live_tool_result ON message_tool_refs(assistant_message_id, tool_use_id) WHERE block_type = 'tool_result' AND cancelled = 0`, + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_source_message ON message_tool_refs(source_message_id)`, + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_tool_use_lookup ON message_tool_refs(tool_use_id, assistant_message_id) WHERE block_type = 'tool_use' AND cancelled = 0`, + ); + await database.execute(`CREATE TABLE IF NOT EXISTS tool_calls ( + id TEXT PRIMARY KEY, + provider_tool_use_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + args TEXT NOT NULL, + executor_id TEXT, + message_id TEXT NOT NULL, + issued_at TEXT NOT NULL, + expires_at TEXT, + state TEXT NOT NULL CHECK(state IN ('queued', 'pending_reconnect', 'pending_ack', 'running', 'completed', 'expired', 'revoked')), + result TEXT, + progress TEXT, + completed_at TEXT + )`); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_message_id ON tool_calls(message_id)`, + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_state ON tool_calls(state)`, + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_expires_at ON tool_calls(expires_at) WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS environment_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), snapshot TEXT NOT NULL, updated_at TEXT NOT NULL)`, + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS thread_settings_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), settings TEXT NOT NULL, updated_at TEXT NOT NULL)`, + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS retry_state (id INTEGER PRIMARY KEY CHECK (id = 1), attempt INTEGER NOT NULL DEFAULT 0, scheduled_at INTEGER NOT NULL, reason TEXT NOT NULL)`, + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS queued_messages (message_id TEXT PRIMARY KEY, content TEXT NOT NULL, user_state TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), steer INTEGER NOT NULL DEFAULT 0, user_meta TEXT)`, + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS executor_artifacts (artifact_key TEXT PRIMARY KEY, data_type TEXT NOT NULL, content_base64 TEXT NOT NULL, tool_call_id TEXT, updated_at TEXT NOT NULL)`, + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS e2b_sandbox ( + id INTEGER PRIMARY KEY CHECK (id = 1), + sandbox_id TEXT, + restart_attempts INTEGER NOT NULL DEFAULT 0, + traffic_access_token TEXT, + project_id TEXT, + repository_url TEXT, + additional_repositories TEXT, + setup TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )`, + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS tool_approvals (id TEXT PRIMARY KEY, tool_call_id TEXT NOT NULL UNIQUE, tool_name TEXT NOT NULL, args TEXT NOT NULL, reason TEXT, to_allow TEXT, context TEXT NOT NULL CHECK(context IN ('thread', 'subagent')), subagent_tool_name TEXT, parent_tool_call_id TEXT, timestamp INTEGER NOT NULL, matched_rule TEXT, rule_source TEXT CHECK(rule_source IN ('user', 'built-in')))`, + ); + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_approvals_timestamp ON tool_approvals(timestamp)`, + ); + await database.execute( + `CREATE TABLE IF NOT EXISTS compaction_summaries (summary_id TEXT PRIMARY KEY, summary_text TEXT NOT NULL, cut_message_id TEXT NOT NULL, created_at TEXT NOT NULL)`, + ); +} + +async function seedAgentConcurrent2Data(database: RawRivetDB): Promise { + const existing = await database.execute(`SELECT COUNT(*) AS count FROM messages`); + if (Number(existing[0]?.count ?? 0) > 0) { + return; + } + + const now = new Date("2026-05-16T03:58:18.661Z").getTime(); + const text = (size: number) => "x".repeat(size); + const isoAt = (index: number) => new Date(now + index * 1_000).toISOString(); + + await batchInsert(database, `INSERT INTO thread_meta_kv (key, value, updated_at)`, [ + ["executor_type", "local-client", isoAt(0)], + ["workspace_intent", JSON.stringify({ spec: null, pendingLaunch: null }), isoAt(0)], + ["executor_status", JSON.stringify({ available: true, message: "ready" }), isoAt(0)], + ]); + + const messageRows: unknown[][] = []; + for (let index = 1; index <= MESSAGE_COUNT; index++) { + const role = index % 2 === 0 ? "assistant" : "user"; + messageRows.push([ + messageId(index), + role, + text(MESSAGE_CONTENT_BYTES), + null, + null, + isoAt(index), + 0, + null, + null, + null, + ]); + } + await batchInsert( + database, + `INSERT INTO messages (message_id, role, content, meta, user_state, created_at, cancelled, read_at, parent_tool_use_id, tool_result_for_message_id)`, + messageRows, + 20, + ); + + const messageToolRefRows: unknown[][] = []; + for (let index = 0; index < MESSAGE_TOOL_REF_COUNT / 2; index++) { + const assistantIndex = 2 + (index % 42) * 2; + const sourceIndex = Math.max(1, assistantIndex - 1); + const resultIndex = Math.min(MESSAGE_COUNT, assistantIndex + 1); + const toolUseId = toolUseID(index + 1); + messageToolRefRows.push([ + messageId(sourceIndex), + messageId(assistantIndex), + toolUseId, + "tool_use", + 0, + ]); + messageToolRefRows.push([ + messageId(resultIndex), + messageId(assistantIndex), + toolUseId, + "tool_result", + 0, + ]); + } + await batchInsert( + database, + `INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled)`, + messageToolRefRows, + 50, + ); + + const toolCallRows: unknown[][] = []; + for (let index = 1; index <= TOOL_CALL_COUNT; index++) { + const assistantIndex = 2 + ((index - 1) % 42) * 2; + toolCallRows.push([ + toolUseID(index), + `provider-${index}`, + `tool_${index % 21}`, + JSON.stringify({ path: `/tmp/file-${index}` }), + "seed-executor", + messageId(assistantIndex), + isoAt(index), + null, + "completed", + JSON.stringify({ + ok: true, + run: { status: "done", result: text(TOOL_CALL_RESULT_BYTES) }, + }), + null, + isoAt(index + 100), + ]); + } + await batchInsert( + database, + `INSERT INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at)`, + toolCallRows, + 20, + ); + + const executorToolRows: unknown[][] = []; + for (let index = 1; index <= EXECUTOR_TOOL_COUNT; index++) { + const schema = JSON.stringify({ + name: `tool_${index}`, + description: text(EXECUTOR_TOOL_SCHEMA_BYTES), + input_schema: { type: "object", properties: {} }, + }); + executorToolRows.push(["seed-executor", `tool_${index}`, schema, isoAt(index)]); + } + await batchInsert( + database, + `INSERT INTO executor_tools (executor_id, tool_name, schema, updated_at)`, + executorToolRows, + 42, + ); + + const threadEventRows: unknown[][] = []; + for (let index = 1; index <= THREAD_EVENT_COUNT; index++) { + threadEventRows.push([ + index, + index % 3 === 0 ? "message_added" : "agent_state_changed", + JSON.stringify({ type: "seed_event", body: text(THREAD_EVENT_PAYLOAD_BYTES) }), + isoAt(index), + ]); + } + await batchInsert( + database, + `INSERT INTO thread_events (seq, event_type, payload, created_at)`, + threadEventRows, + 25, + ); + + const messageAddedRows: unknown[][] = []; + for (let index = 1; index <= MESSAGE_COUNT; index++) { + messageAddedRows.push([messageId(index), index]); + } + await batchInsert( + database, + `INSERT INTO message_added_events (message_id, seq)`, + messageAddedRows, + 50, + ); + + await database.execute( + `INSERT INTO environment_snapshot (id, snapshot, updated_at) VALUES (1, ?, ?)`, + JSON.stringify({ cwd: "/workspace", body: text(3_620) }), + isoAt(0), + ); + await database.execute( + `INSERT INTO thread_settings_snapshot (id, settings, updated_at) VALUES (1, ?, ?)`, + JSON.stringify({ maxTokens: 20_000, body: text(55) }), + isoAt(0), + ); + await database.execute( + `INSERT INTO e2b_sandbox (id, sandbox_id, restart_attempts, traffic_access_token, project_id, repository_url, additional_repositories, setup, created_at, updated_at) + VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "sandbox-seed", + 0, + "token-seed", + "project-seed", + "https://example.invalid/repo.git", + JSON.stringify([]), + JSON.stringify({ commands: [] }), + isoAt(0), + isoAt(0), + ); +} + +async function batchInsert( + database: RawRivetDB, + insertPrefix: string, + rows: unknown[][], + batchSize = 100, +): Promise { + if (rows.length === 0) { + return; + } + const columnCount = rows[0]?.length ?? 0; + if (columnCount === 0) { + return; + } + const rowPlaceholder = `(${"?,".repeat(columnCount).slice(0, -1)})`; + for (let index = 0; index < rows.length; index += batchSize) { + const chunk = rows.slice(index, index + batchSize); + const values = chunk.map(() => rowPlaceholder).join(","); + const bindings = chunk.flat(); + await database.execute(`${insertPrefix} VALUES ${values}`, ...bindings); + } +} + +function messageId(index: number): string { + return `M-${String(index).padStart(22, "0")}`; +} + +function toolUseID(index: number): string { + return `toolu_${String(index).padStart(22, "0")}`; +} + +function safeId(value: string): string { + return value.replace(/[^a-zA-Z0-9_-]/g, "-").slice(0, 80); +} diff --git a/examples/kitchen-sink/src/actors/testing/load-test-agent.ts b/examples/kitchen-sink/src/actors/testing/load-test-agent.ts new file mode 100644 index 0000000000..0b6682d249 --- /dev/null +++ b/examples/kitchen-sink/src/actors/testing/load-test-agent.ts @@ -0,0 +1,205 @@ +import { actor, type RivetMessageEvent, type UniversalWebSocket } from "rivetkit"; +import { db } from "rivetkit/db"; + +const DEFAULT_TOKENS_PER_SECOND = 20; +const DEFAULT_DURATION_MS = 5_000; + +function send(websocket: UniversalWebSocket, payload: unknown): void { + if (websocket.readyState !== 1) return; + websocket.send(JSON.stringify(payload)); +} + +function parsePositiveNumber( + value: unknown, + name: string, + fallback: number, +): number { + if (value === undefined || value === null) return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive number`); + } + return parsed; +} + +function sleep(ms: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const timeout = setTimeout(resolve, ms); + signal.addEventListener( + "abort", + () => { + clearTimeout(timeout); + resolve(); + }, + { once: true }, + ); + }); +} + +export const loadTestAgent = actor({ + options: { + canHibernateWebSocket: false, + sleepGracePeriod: 30_000, + }, + db: db({ + onMigrate: async (db) => { + await db.execute(` + CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + connection_id TEXT NOT NULL, + request_id TEXT NOT NULL, + token_index INTEGER NOT NULL, + token TEXT NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await db.execute(` + CREATE INDEX IF NOT EXISTS messages_request_idx + ON messages (request_id, token_index) + `); + }, + }), + state: { + connectionCount: 0, + inferenceCount: 0, + tokenCount: 0, + }, + onWebSocket(c, websocket: UniversalWebSocket) { + c.state.connectionCount += 1; + const connectionId = crypto.randomUUID(); + + send(websocket, { + type: "connected", + connectionId, + connectionCount: c.state.connectionCount, + timestamp: Date.now(), + }); + + websocket.addEventListener("message", async (event: RivetMessageEvent) => { + try { + const message = + typeof event.data === "string" + ? JSON.parse(event.data) + : undefined; + + // Fast-path ping: echo back without touching SQLite so the client can measure raw + // RTT without the per-message storage write. Used by the counter-latency client's + // first two probes after WS open. + if (message && message.type === "ping") { + send(websocket, { + type: "pong", + connectionId, + id: message.id, + timestamp: Date.now(), + }); + return; + } + + if (!message || message.type !== "inference") { + throw new Error("expected inference message"); + } + + const requestId = + typeof message.requestId === "string" && message.requestId + ? message.requestId + : crypto.randomUUID(); + const tokensPerSecond = parsePositiveNumber( + message.tokensPerSecond, + "tokensPerSecond", + DEFAULT_TOKENS_PER_SECOND, + ); + const durationMs = parsePositiveNumber( + message.durationMs, + "durationMs", + DEFAULT_DURATION_MS, + ); + const intervalMs = 1_000 / tokensPerSecond; + const targetTokens = Math.max( + 1, + Math.floor((durationMs / 1_000) * tokensPerSecond), + ); + + const inference = (async () => { + c.state.inferenceCount += 1; + send(websocket, { + type: "inference-start", + connectionId, + requestId, + tokensPerSecond, + durationMs, + targetTokens, + timestamp: Date.now(), + }); + + const startedAt = performance.now(); + for (let i = 0; i < targetTokens; i++) { + if (c.abortSignal.aborted || websocket.readyState !== 1) { + break; + } + + const tokenIndex = i + 1; + const token = `token-${tokenIndex}`; + const createdAt = Date.now(); + await c.db.execute( + "INSERT INTO messages (connection_id, request_id, token_index, token, created_at) VALUES (?, ?, ?, ?, ?)", + connectionId, + requestId, + tokenIndex, + token, + createdAt, + ); + c.state.tokenCount += 1; + + send(websocket, { + type: "token", + connectionId, + requestId, + tokenIndex, + token, + timestamp: createdAt, + }); + + const nextAt = startedAt + tokenIndex * intervalMs; + const delayMs = Math.max(0, nextAt - performance.now()); + if (delayMs > 0) { + await sleep(delayMs, c.abortSignal); + } + } + + send(websocket, { + type: "inference-complete", + connectionId, + requestId, + tokenCount: targetTokens, + timestamp: Date.now(), + }); + })(); + + await c.keepAwake(inference); + } catch (error) { + send(websocket, { + type: "error", + message: + error instanceof Error + ? error.message + : "unknown websocket error", + timestamp: Date.now(), + }); + } + }); + + websocket.addEventListener("close", () => { + c.state.connectionCount -= 1; + }); + }, + actions: { + getStats(c) { + return { + connectionCount: c.state.connectionCount, + inferenceCount: c.state.inferenceCount, + tokenCount: c.state.tokenCount, + }; + }, + }, +}); diff --git a/examples/kitchen-sink/src/actors/testing/sigterm-sleep-probe.ts b/examples/kitchen-sink/src/actors/testing/sigterm-sleep-probe.ts new file mode 100644 index 0000000000..4aaf1666c8 --- /dev/null +++ b/examples/kitchen-sink/src/actors/testing/sigterm-sleep-probe.ts @@ -0,0 +1,329 @@ +import { actor, type RivetMessageEvent, type UniversalWebSocket } from "rivetkit"; +import { db } from "rivetkit/db"; + +export const DEFAULT_ON_SLEEP_DURATION_MS = 5_000; +export const DEFAULT_ON_SLEEP_TICK_MS = 1_000; +const SLEEP_TIMEOUT_MS = 10 * 60 * 1000; +const SLEEP_GRACE_PERIOD_MS = 30 * 60 * 1000; +const ACTOR_STOPPED_CLOSE_CODE = 1000; +const ACTOR_STOPPED_CLOSE_REASON = "actor stopped"; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function formatError(error: unknown): string { + if (error instanceof Error) return error.stack ?? error.message; + return String(error); +} + +export const sigtermSleepProbe = actor({ + state: { + label: "unprepared", + wakeCount: 0, + sleepCount: 0, + onSleepDurationMs: DEFAULT_ON_SLEEP_DURATION_MS, + onSleepTickMs: DEFAULT_ON_SLEEP_TICK_MS, + connectionCount: 0, + messageCount: 0, + onSleepStartedAt: null as number | null, + onSleepAsyncFinishedAt: null as number | null, + onSleepFinishedAt: null as number | null, + onSleepLastError: null as string | null, + }, + createVars: () => ({ + websockets: new Set(), + }), + db: db({ + onMigrate: async (database) => { + await database.execute(` + CREATE TABLE IF NOT EXISTS sigterm_sleep_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + event TEXT NOT NULL, + sleep_count INTEGER NOT NULL, + detail TEXT, + created_at INTEGER NOT NULL + ) + `); + }, + }), + onWebSocket: (c, websocket: UniversalWebSocket) => { + c.vars.websockets.add(websocket); + c.state.connectionCount += 1; + const connectionId = crypto.randomUUID(); + + c.log.info({ + msg: "sigterm sleep probe websocket connected", + label: c.state.label, + connectionId, + connectionCount: c.state.connectionCount, + }); + + websocket.send( + JSON.stringify({ + type: "welcome", + connectionId, + label: c.state.label, + connectionCount: c.state.connectionCount, + }), + ); + + websocket.addEventListener("message", (event: RivetMessageEvent) => { + c.state.messageCount += 1; + const data = event.data; + if (typeof data !== "string") return; + + try { + const parsed = JSON.parse(data); + if (parsed.type === "ping") { + websocket.send( + JSON.stringify({ + type: "pong", + connectionId, + messageCount: c.state.messageCount, + timestamp: Date.now(), + }), + ); + return; + } + } catch {} + + websocket.send( + JSON.stringify({ + type: "echo", + connectionId, + received: data, + messageCount: c.state.messageCount, + timestamp: Date.now(), + }), + ); + }); + + websocket.addEventListener("close", (event) => { + c.vars.websockets.delete(websocket); + c.state.connectionCount -= 1; + c.log.info({ + msg: "sigterm sleep probe websocket closed", + label: c.state.label, + connectionId, + connectionCount: c.state.connectionCount, + code: event.code, + reason: event.reason, + }); + }); + }, + onWake: async (c) => { + c.state.wakeCount += 1; + c.log.info({ + msg: "sigterm sleep probe onWake", + label: c.state.label, + wakeCount: c.state.wakeCount, + sleepCount: c.state.sleepCount, + }); + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "wake", + c.state.sleepCount, + `wake-${c.state.wakeCount}`, + Date.now(), + ); + }, + onSleep: async (c) => { + const sleepCount = c.state.sleepCount + 1; + const startedAt = Date.now(); + c.state.sleepCount = sleepCount; + c.state.onSleepStartedAt = startedAt; + c.state.onSleepAsyncFinishedAt = null; + c.state.onSleepFinishedAt = null; + c.state.onSleepLastError = null; + + c.log.info({ + msg: "sigterm sleep probe onSleep start", + label: c.state.label, + sleepCount, + onSleepDurationMs: c.state.onSleepDurationMs, + onSleepTickMs: c.state.onSleepTickMs, + }); + + try { + for (const websocket of c.vars.websockets) { + if (websocket.readyState !== 1) continue; + websocket.send( + JSON.stringify({ + type: "onSleepStarted", + sleepCount, + onSleepDurationMs: c.state.onSleepDurationMs, + onSleepTickMs: c.state.onSleepTickMs, + timestamp: startedAt, + }), + ); + } + + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "on-sleep-start", + sleepCount, + c.state.label, + startedAt, + ); + + const deadline = startedAt + c.state.onSleepDurationMs; + let tickIndex = 0; + while (Date.now() < deadline) { + const waitMs = Math.min( + c.state.onSleepTickMs, + Math.max(0, deadline - Date.now()), + ); + if (waitMs > 0) await sleep(waitMs); + + tickIndex += 1; + const tickAt = Date.now(); + const detail = `tick=${tickIndex} elapsed-ms=${tickAt - startedAt}`; + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "on-sleep-tick", + sleepCount, + detail, + tickAt, + ); + c.log.info({ + msg: "sigterm sleep probe onSleep tick", + label: c.state.label, + sleepCount, + tickIndex, + elapsedMs: tickAt - startedAt, + }); + + for (const websocket of c.vars.websockets) { + if (websocket.readyState !== 1) continue; + websocket.send( + JSON.stringify({ + type: "onSleepTick", + sleepCount, + tickIndex, + elapsedMs: tickAt - startedAt, + timestamp: tickAt, + }), + ); + } + } + + const asyncFinishedAt = Date.now(); + c.state.onSleepAsyncFinishedAt = asyncFinishedAt; + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "on-sleep-after-await", + sleepCount, + `delay-ms=${asyncFinishedAt - startedAt}`, + asyncFinishedAt, + ); + + const finishedAt = Date.now(); + c.state.onSleepFinishedAt = finishedAt; + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "on-sleep-finish", + sleepCount, + c.state.label, + finishedAt, + ); + + for (const websocket of c.vars.websockets) { + if (websocket.readyState !== 1) continue; + websocket.send( + JSON.stringify({ + type: "onSleepFinished", + sleepCount, + elapsedMs: finishedAt - startedAt, + timestamp: finishedAt, + }), + ); + websocket.close( + ACTOR_STOPPED_CLOSE_CODE, + ACTOR_STOPPED_CLOSE_REASON, + ); + } + + c.log.info({ + msg: "sigterm sleep probe onSleep finish", + label: c.state.label, + sleepCount, + elapsedMs: finishedAt - startedAt, + }); + } catch (error) { + const message = formatError(error); + c.state.onSleepLastError = message; + c.log.error({ + msg: "sigterm sleep probe onSleep error", + label: c.state.label, + sleepCount, + error: message, + }); + throw error; + } + }, + actions: { + prepare: async ( + c, + label = `sigterm-sleep-probe-${Date.now()}`, + onSleepDurationMs = DEFAULT_ON_SLEEP_DURATION_MS, + onSleepTickMs = DEFAULT_ON_SLEEP_TICK_MS, + ) => { + if (!Number.isFinite(onSleepDurationMs) || onSleepDurationMs < 0) { + throw new Error("onSleepDurationMs must be a finite non-negative number"); + } + if (!Number.isFinite(onSleepTickMs) || onSleepTickMs <= 0) { + throw new Error("onSleepTickMs must be a finite positive number"); + } + c.state.label = label; + c.state.onSleepDurationMs = onSleepDurationMs; + c.state.onSleepTickMs = onSleepTickMs; + await c.db.execute( + "INSERT INTO sigterm_sleep_log (event, sleep_count, detail, created_at) VALUES (?, ?, ?, ?)", + "prepared", + c.state.sleepCount, + label, + Date.now(), + ); + return { + label: c.state.label, + onSleepDurationMs: c.state.onSleepDurationMs, + onSleepTickMs: c.state.onSleepTickMs, + wakeCount: c.state.wakeCount, + sleepCount: c.state.sleepCount, + connectionCount: c.state.connectionCount, + messageCount: c.state.messageCount, + }; + }, + getProof: async (c) => { + const rows = await c.db.execute<{ + id: number; + event: string; + sleep_count: number; + detail: string | null; + created_at: number; + }>("SELECT * FROM sigterm_sleep_log ORDER BY id"); + return { + state: { + label: c.state.label, + wakeCount: c.state.wakeCount, + sleepCount: c.state.sleepCount, + onSleepDurationMs: c.state.onSleepDurationMs, + onSleepTickMs: c.state.onSleepTickMs, + connectionCount: c.state.connectionCount, + messageCount: c.state.messageCount, + onSleepStartedAt: c.state.onSleepStartedAt, + onSleepAsyncFinishedAt: c.state.onSleepAsyncFinishedAt, + onSleepFinishedAt: c.state.onSleepFinishedAt, + onSleepLastError: c.state.onSleepLastError, + }, + rows, + }; + }, + }, + options: { + canHibernateWebSocket: false, + sleepTimeout: SLEEP_TIMEOUT_MS, + sleepGracePeriod: SLEEP_GRACE_PERIOD_MS, + }, +}); diff --git a/examples/kitchen-sink/src/actors/testing/slow-reconnect-actor.ts b/examples/kitchen-sink/src/actors/testing/slow-reconnect-actor.ts new file mode 100644 index 0000000000..7e781f8bb3 --- /dev/null +++ b/examples/kitchen-sink/src/actors/testing/slow-reconnect-actor.ts @@ -0,0 +1,976 @@ +import { actor, setup } from 'rivetkit' +import { db } from 'rivetkit/db' + +export type SlowReconnectRequest = + | { type: 'client_resume'; version: number } + | { + type: 'executor_connect' + clientId: string + executorType?: 'local-client' | 'sandbox' | 'virtual' + } + | { + type: 'repro_reconnect' + clientId?: string + staggerHandleMs?: number + } + +export interface SlowReconnectStep { + name: string + durationMs: number + rowCount: number +} + +export interface SlowReconnectWorkloadResult { + name: string + totalMs: number + steps: SlowReconnectStep[] +} + +export interface SlowReconnectResultMessage { + type: 'slow_reconnect_result' + trigger: SlowReconnectRequest['type'] + totalMs: number + results: SlowReconnectWorkloadResult[] +} + +export interface SlowReconnectErrorMessage { + type: 'slow_reconnect_error' + trigger: SlowReconnectRequest['type'] | 'unknown' + error: string +} + +export interface SlowReconnectVars { + sql: Db | null +} + +interface RawRivetDB { + execute: (query: string, ...args: unknown[]) => Promise[]> +} + +type SQLPrimitive = string | number | boolean | null + +type Db = (>( + query: string, + ...values: SQLPrimitive[] +) => Promise) & { + withTransaction(fn: (tx: Db) => Promise): Promise +} + +class AsyncMutex { + private locked = false + private waiters: Array<() => void> = [] + + async acquire(): Promise { + if (!this.locked) { + this.locked = true + return + } + await new Promise((resolve) => this.waiters.push(resolve)) + this.locked = true + } + + release(): void { + const next = this.waiters.shift() + if (next) { + next() + return + } + this.locked = false + } +} + +function createDb(execute: >( + query: string, + ...values: SQLPrimitive[] +) => Promise): Db { + const mutex = new AsyncMutex() + let activeTransaction: Db | null = null + + const createTransactionDb = (): Db => { + const tx = Object.assign( + >(query: string, ...values: SQLPrimitive[]) => + execute(query, ...values), + { + withTransaction: async (fn: (tx: Db) => Promise): Promise => fn(tx), + }, + ) + return tx + } + + const queryWithMutex = async >( + query: string, + ...values: SQLPrimitive[] + ): Promise => { + if (activeTransaction) { + return activeTransaction(query, ...values) + } + await mutex.acquire() + try { + return await execute(query, ...values) + } finally { + mutex.release() + } + } + + const sql = Object.assign(queryWithMutex, { + withTransaction: async (fn: (tx: Db) => Promise): Promise => { + if (activeTransaction) { + return fn(activeTransaction) + } + await mutex.acquire() + const tx = createTransactionDb() + try { + await execute('BEGIN') + activeTransaction = tx + try { + const result = await fn(tx) + activeTransaction = null + await execute('COMMIT') + return result + } catch (error) { + activeTransaction = null + await execute('ROLLBACK') + throw error + } + } finally { + activeTransaction = null + mutex.release() + } + }, + }) + return sql +} + +const MESSAGE_COUNT = 84 +const MESSAGE_TOOL_REF_COUNT = 122 +const TOOL_CALL_COUNT = 61 +const EXECUTOR_TOOL_COUNT = 42 +const THREAD_EVENT_COUNT = 233 + +const MESSAGE_CONTENT_BYTES = 10_620 +const THREAD_EVENT_PAYLOAD_BYTES = 4_036 +const TOOL_CALL_RESULT_BYTES = 10_975 +const EXECUTOR_TOOL_SCHEMA_BYTES = 2_235 + +export const slowReconnectActor = actor({ + state: { runCount: 0 }, + db: db({ + onMigrate: async (database) => { + await createSlowReconnectSchema(database) + }, + }), + vars: { sql: null } as SlowReconnectVars, + onWebSocket: (c, ws) => { + const sock = ws as unknown as WebSocket + if (sock.readyState === WebSocket.OPEN) { + sock.send('pong') + } + + ws.addEventListener('message', (event) => { + const promise = handleSlowReconnectWebSocketMessage(c, sock, event.data) + void c.keepAwake(promise) + }) + }, + actions: { + prepare: async (c) => { + await createSlowReconnectSchema(c.db) + return await seedSlowReconnectData(c.db) + }, + reproReconnect: async (c, clientId?: string) => { + c.vars.sql ??= createSlowReconnectDb(c.db) + c.state.runCount++ + return await runReconnectRepro(c.vars.sql, clientId ?? `action-${c.state.runCount}`, 0) + }, + getRunCount: (c) => c.state.runCount, + sleep: (c) => { + c.sleep() + return true + }, + }, +}) + +async function handleSlowReconnectWebSocketMessage( + c: { db: RawRivetDB; vars: SlowReconnectVars; state: { runCount: number } }, + sock: WebSocket, + data: unknown, +): Promise { + if (data === 'ping') { + if (sock.readyState === WebSocket.OPEN) { + sock.send('pong') + } + return + } + + let trigger: SlowReconnectRequest['type'] | 'unknown' = 'unknown' + try { + const request = parseSlowReconnectRequest(data) + trigger = request.type + c.vars.sql ??= createSlowReconnectDb(c.db) + c.state.runCount++ + + if (request.type === 'client_resume') { + const startedAt = performance.now() + const result = await runCatchupSnapshot(c.vars.sql, request.version) + sendJSON(sock, { + type: 'slow_reconnect_result', + trigger: request.type, + totalMs: Math.round(performance.now() - startedAt), + results: [result], + }) + return + } + + const clientId = + request.type === 'executor_connect' + ? request.clientId + : (request.clientId ?? `slow-reconnect-${c.state.runCount}`) + const staggerHandleMs = request.type === 'repro_reconnect' ? (request.staggerHandleMs ?? 0) : 0 + const result = await runReconnectRepro(c.vars.sql, clientId, staggerHandleMs) + + if (request.type === 'executor_connect') { + sendJSON(sock, { + type: 'executor_connected', + executorId: clientId, + registeredToolCount: EXECUTOR_TOOL_COUNT, + guidanceInventory: [], + resumeBootstrap: true, + }) + } + + sendJSON(sock, { + type: 'slow_reconnect_result', + trigger: request.type, + ...result, + }) + } catch (error) { + sendJSON(sock, { + type: 'slow_reconnect_error', + trigger, + error: error instanceof Error ? error.message : String(error), + }) + } +} + +function parseSlowReconnectRequest(data: unknown): SlowReconnectRequest { + if (typeof data !== 'string') { + throw new Error('slowReconnectActor request must be a string') + } + const parsed = JSON.parse(data) as unknown + if (!parsed || typeof parsed !== 'object') { + throw new Error('slowReconnectActor request must be an object') + } + const request = parsed as Record + if (request.type === 'client_resume') { + return { type: 'client_resume', version: numberField(request, 'version') } + } + if (request.type === 'executor_connect') { + const executorType = request.executorType + return { + type: 'executor_connect', + clientId: stringField(request, 'clientId'), + ...(executorType === 'local-client' || + executorType === 'sandbox' || + executorType === 'virtual' + ? { executorType } + : {}), + } + } + if (request.type === 'repro_reconnect') { + return { + type: 'repro_reconnect', + ...(typeof request.clientId === 'string' ? { clientId: request.clientId } : {}), + ...(typeof request.staggerHandleMs === 'number' + ? { staggerHandleMs: request.staggerHandleMs } + : {}), + } + } + throw new Error(`Unknown slowReconnectActor request type: ${String(request.type)}`) +} + +function stringField(record: Record, field: string): string { + const value = record[field] + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`slowReconnectActor request ${field} must be a non-empty string`) + } + return value +} + +function numberField(record: Record, field: string): number { + const value = record[field] + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`slowReconnectActor request ${field} must be a finite number`) + } + return value +} + +function sendJSON( + sock: WebSocket, + message: SlowReconnectResultMessage | SlowReconnectErrorMessage | object, +): void { + if (sock.readyState === WebSocket.OPEN) { + sock.send(JSON.stringify(message)) + } +} + +function createSlowReconnectDb(db: RawRivetDB): Db { + return createDb(async >( + query: string, + ...values: SQLPrimitive[] + ): Promise => { + const converted = values.map((value) => + typeof value === 'boolean' ? (value ? 1 : 0) : value, + ) + return (await db.execute(query, ...converted)) as T[] + }) +} + +async function runReconnectRepro( + sql: Db, + clientId: string, + staggerHandleMs: number, +): Promise> { + const startedAt = performance.now() + const buildToolPlanContext = runBuildToolPlanContext(sql) + const catchupSnapshot = runCatchupSnapshot(sql, 0) + const recoverToolCalls = runRecoverToolCalls(sql) + const handleExecutorConnect = delay(staggerHandleMs).then(() => + runHandleExecutorConnect(sql, clientId), + ) + + const results = await Promise.all([ + handleExecutorConnect, + buildToolPlanContext, + catchupSnapshot, + recoverToolCalls, + ]) + return { + totalMs: Math.round(performance.now() - startedAt), + results, + } +} + +async function runHandleExecutorConnect( + sql: Db, + clientId: string, +): Promise { + const startedAt = performance.now() + const steps: SlowReconnectStep[] = [] + const nextSeq = await sql.withTransaction(async (tx) => { + const latestExecutor = await timedQuery( + tx, + steps, + 'load-latest-executor-id', + `SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1`, + ) + const latestExecutorId = String(latestExecutor[0]?.executor_id ?? 'seed-executor') + await timedQuery( + tx, + steps, + 'select-cached-executor-tools', + `SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`, + latestExecutorId, + ) + const executorType = await timedQuery( + tx, + steps, + 'select-executor-type', + `SELECT value FROM thread_meta_kv WHERE key = 'executor_type'`, + ) + if (!executorType[0]?.value) { + await timedQuery( + tx, + steps, + 'set-executor-type', + `INSERT OR REPLACE INTO thread_meta_kv (key, value, updated_at) VALUES ('executor_type', ?, ?)`, + 'local-client', + new Date().toISOString(), + ) + } + const sandboxIntent = await timedQuery( + tx, + steps, + 'select-sandbox-intent', + `SELECT value FROM thread_meta_kv WHERE key = 'sandbox_intent'`, + ) + if (hasPendingLaunch(sandboxIntent[0]?.value)) { + await timedQuery( + tx, + steps, + 'clear-pending-launch', + `UPDATE thread_meta_kv SET value = ?, updated_at = ? WHERE key = 'sandbox_intent'`, + JSON.stringify({ spec: null, pendingLaunch: null }), + new Date().toISOString(), + ) + } + const seqRows = await timedQuery( + tx, + steps, + 'select-next-thread-event-seq', + `SELECT COALESCE(MAX(seq), 0) + 1 AS seq FROM thread_events`, + ) + const seq = Number(seqRows[0]?.seq ?? 1) + await timedQuery( + tx, + steps, + 'insert-executor-connected-event', + `INSERT INTO thread_events (seq, event_type, payload, created_at) VALUES (?, ?, ?, ?)`, + seq, + 'executor_connected', + JSON.stringify({ type: 'executor_connected', executorId: clientId }), + new Date().toISOString(), + ) + return seq + }) + steps.push({ + name: 'transaction-total', + durationMs: Math.round(performance.now() - startedAt), + rowCount: nextSeq, + }) + return { + name: 'handle-executor-connect', + totalMs: Math.round(performance.now() - startedAt), + steps, + } +} + +async function runBuildToolPlanContext(sql: Db): Promise { + const startedAt = performance.now() + const steps: SlowReconnectStep[] = [] + const latestExecutor = await timedQuery( + sql, + steps, + 'load-latest-executor-id', + `SELECT executor_id FROM executor_tools ORDER BY updated_at DESC LIMIT 1`, + ) + const latestExecutorId = String(latestExecutor[0]?.executor_id ?? 'seed-executor') + await timedQuery( + sql, + steps, + 'select-executor-tools', + `SELECT tool_name, schema FROM executor_tools WHERE executor_id = ? ORDER BY tool_name ASC`, + latestExecutorId, + ) + await timedQuery( + sql, + steps, + 'count-uncancelled-top-level', + `SELECT COUNT(*) as count FROM messages WHERE cancelled = 0 AND parent_tool_use_id IS NULL`, + ) + const unresolvedRows = await timedQuery( + sql, + steps, + 'find-unresolved-assistant-message', + `SELECT m.* + FROM message_tool_refs AS tool_use + JOIN messages AS m + ON m.message_id = tool_use.assistant_message_id + WHERE tool_use.block_type = 'tool_use' + AND tool_use.cancelled = 0 + AND m.cancelled = 0 + AND m.role = 'assistant' + AND m.parent_tool_use_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM message_tool_refs AS tool_result + JOIN messages AS tool_result_message + ON tool_result_message.message_id = tool_result.source_message_id + WHERE tool_result.assistant_message_id = tool_use.assistant_message_id + AND tool_result.block_type = 'tool_result' + AND tool_result.cancelled = 0 + AND tool_result.tool_use_id = tool_use.tool_use_id + AND tool_result_message.parent_tool_use_id IS NULL + ) + GROUP BY m.message_id + ORDER BY m.created_at DESC + LIMIT 1`, + ) + const unresolvedMessageId = unresolvedRows[0]?.message_id + if (typeof unresolvedMessageId === 'string') { + await timedQuery( + sql, + steps, + 'get-persisted-tool-result-ids', + `SELECT tool_result.tool_use_id + FROM message_tool_refs AS tool_result + JOIN messages AS tool_result_message + ON tool_result_message.message_id = tool_result.source_message_id + WHERE tool_result.assistant_message_id = ? + AND tool_result.block_type = 'tool_result' + AND tool_result.cancelled = 0 + AND tool_result_message.parent_tool_use_id IS NULL`, + unresolvedMessageId, + ) + await timedQuery( + sql, + steps, + 'get-tool-calls-by-message-id', + `SELECT * FROM tool_calls WHERE message_id = ?`, + unresolvedMessageId, + ) + } + await timedQuery( + sql, + steps, + 'is-last-message-cancelled-assistant', + `SELECT role, cancelled FROM messages + WHERE parent_tool_use_id IS NULL + ORDER BY created_at DESC + LIMIT 1`, + ) + await timedQuery( + sql, + steps, + 'get-last-uncancelled', + `SELECT m.* FROM messages m + WHERE m.cancelled = 0 AND m.parent_tool_use_id IS NULL + ORDER BY m.created_at DESC + LIMIT 1`, + ) + return { + name: 'build-tool-plan-context', + totalMs: Math.round(performance.now() - startedAt), + steps, + } +} + +async function runCatchupSnapshot(sql: Db, version: number): Promise { + const startedAt = performance.now() + const steps: SlowReconnectStep[] = [] + await Promise.all([ + timedQuery( + sql, + steps, + 'thread-events-list-since-version', + `SELECT seq, event_type, payload, created_at FROM thread_events WHERE seq > ? ORDER BY seq ASC`, + version, + ), + timedQuery( + sql, + steps, + 'environment-snapshot', + `SELECT snapshot FROM environment_snapshot WHERE id = 1`, + ), + timedQuery( + sql, + steps, + 'thread-settings-snapshot', + `SELECT settings FROM thread_settings_snapshot WHERE id = 1`, + ), + timedQuery(sql, steps, 'retry-state', `SELECT * FROM retry_state WHERE id = 1`), + timedQuery( + sql, + steps, + 'queued-messages', + `SELECT * FROM queued_messages ORDER BY created_at ASC`, + ), + timedQuery( + sql, + steps, + 'executor-artifacts', + `SELECT artifact_key, data_type, length(content_base64) AS bytes, tool_call_id, updated_at FROM executor_artifacts ORDER BY updated_at ASC`, + ), + timedQuery(sql, steps, 'tool-approvals', `SELECT * FROM tool_approvals ORDER BY timestamp ASC`), + timedQuery( + sql, + steps, + 'compaction-summaries', + `SELECT cut_message_id, created_at FROM compaction_summaries ORDER BY created_at ASC`, + ), + timedQuery( + sql, + steps, + 'executor-status', + `SELECT value FROM thread_meta_kv WHERE key = 'executor_status'`, + ), + ]) + steps.sort((a, b) => b.durationMs - a.durationMs) + return { name: 'catchup-snapshot', totalMs: Math.round(performance.now() - startedAt), steps } +} + +async function runRecoverToolCalls(sql: Db): Promise { + const startedAt = performance.now() + const steps: SlowReconnectStep[] = [] + await timedQuery( + sql, + steps, + 'hydrate-tool-progress', + `SELECT id, progress + FROM tool_calls + WHERE progress IS NOT NULL + AND state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + ) + await timedQuery( + sql, + steps, + 'get-pending-tool-calls', + `SELECT * FROM tool_calls + WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running') + ORDER BY issued_at ASC`, + ) + await timedQuery( + sql, + steps, + 'get-next-tool-expiry', + `SELECT MIN(expires_at) AS expires_at + FROM tool_calls + WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + ) + return { name: 'recover-tool-calls', totalMs: Math.round(performance.now() - startedAt), steps } +} + +async function timedQuery>( + sql: Db, + steps: SlowReconnectStep[], + name: string, + query: string, + ...values: SQLPrimitive[] +): Promise { + const startedAt = performance.now() + const rows = await sql(query, ...values) + steps.push({ + name, + durationMs: Math.round(performance.now() - startedAt), + rowCount: rows.length, + }) + return rows +} + +function hasPendingLaunch(value: unknown): boolean { + if (typeof value !== 'string' || value.length === 0) { + return false + } + try { + const parsed = JSON.parse(value) as { pendingLaunch?: unknown } + return parsed.pendingLaunch !== null && parsed.pendingLaunch !== undefined + } catch { + return false + } +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms))) +} + +async function createSlowReconnectSchema(database: RawRivetDB): Promise { + await database.execute(`CREATE TABLE IF NOT EXISTS executor_tools ( + executor_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + schema TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (executor_id, tool_name) + )`) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_executor_tools_executor ON executor_tools(executor_id)`, + ) + await database.execute(`CREATE TABLE IF NOT EXISTS thread_meta_kv ( + key TEXT PRIMARY KEY, + value TEXT, + updated_at TEXT NOT NULL + )`) + await database.execute(`CREATE TABLE IF NOT EXISTS thread_events ( + seq INTEGER PRIMARY KEY, + event_type TEXT NOT NULL, + payload TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + )`) + await database.execute(`CREATE INDEX IF NOT EXISTS idx_thread_events_seq ON thread_events(seq)`) + await database.execute(`CREATE TABLE IF NOT EXISTS message_added_events ( + message_id TEXT PRIMARY KEY, + seq INTEGER NOT NULL UNIQUE + )`) + await database.execute(`CREATE TABLE IF NOT EXISTS messages ( + message_id TEXT PRIMARY KEY, + role TEXT NOT NULL CHECK(role IN ('user', 'assistant', 'info')), + content TEXT NOT NULL, + meta TEXT, + user_state TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + cancelled INTEGER NOT NULL DEFAULT 0, + read_at TEXT, + parent_tool_use_id TEXT, + tool_result_for_message_id TEXT + )`) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_role_cancelled_created_at ON messages(parent_tool_use_id, role, cancelled, created_at DESC)`, + ) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_cancelled_created_at ON messages(parent_tool_use_id, cancelled, created_at DESC)`, + ) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_parent_created_at ON messages(parent_tool_use_id, created_at DESC)`, + ) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_messages_role_created_at ON messages(role, created_at)`, + ) + await database.execute(`CREATE TABLE IF NOT EXISTS message_tool_refs ( + source_message_id TEXT NOT NULL, + assistant_message_id TEXT NOT NULL, + tool_use_id TEXT NOT NULL, + block_type TEXT NOT NULL CHECK(block_type IN ('tool_use', 'tool_result')), + cancelled INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (source_message_id, block_type, tool_use_id) + )`) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_assistant_lookup ON message_tool_refs(assistant_message_id, block_type, cancelled, tool_use_id)`, + ) + await database.execute( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_message_tool_refs_live_tool_result ON message_tool_refs(assistant_message_id, tool_use_id) WHERE block_type = 'tool_result' AND cancelled = 0`, + ) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_source_message ON message_tool_refs(source_message_id)`, + ) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_message_tool_refs_tool_use_lookup ON message_tool_refs(tool_use_id, assistant_message_id) WHERE block_type = 'tool_use' AND cancelled = 0`, + ) + await database.execute(`CREATE TABLE IF NOT EXISTS tool_calls ( + id TEXT PRIMARY KEY, + provider_tool_use_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + args TEXT NOT NULL, + executor_id TEXT, + message_id TEXT NOT NULL, + issued_at TEXT NOT NULL, + expires_at TEXT, + state TEXT NOT NULL CHECK(state IN ('queued', 'pending_reconnect', 'pending_ack', 'running', 'completed', 'expired', 'revoked')), + result TEXT, + progress TEXT, + completed_at TEXT + )`) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_message_id ON tool_calls(message_id)`, + ) + await database.execute(`CREATE INDEX IF NOT EXISTS idx_tool_calls_state ON tool_calls(state)`) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_expires_at ON tool_calls(expires_at) WHERE state IN ('queued', 'pending_reconnect', 'pending_ack', 'running')`, + ) + await database.execute( + `CREATE TABLE IF NOT EXISTS environment_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), snapshot TEXT NOT NULL, updated_at TEXT NOT NULL)`, + ) + await database.execute( + `CREATE TABLE IF NOT EXISTS thread_settings_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), settings TEXT NOT NULL, updated_at TEXT NOT NULL)`, + ) + await database.execute( + `CREATE TABLE IF NOT EXISTS retry_state (id INTEGER PRIMARY KEY CHECK (id = 1), attempt INTEGER NOT NULL DEFAULT 0, scheduled_at INTEGER NOT NULL, reason TEXT NOT NULL)`, + ) + await database.execute( + `CREATE TABLE IF NOT EXISTS queued_messages (message_id TEXT PRIMARY KEY, content TEXT NOT NULL, user_state TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), steer INTEGER NOT NULL DEFAULT 0, user_meta TEXT)`, + ) + await database.execute( + `CREATE TABLE IF NOT EXISTS executor_artifacts (artifact_key TEXT PRIMARY KEY, data_type TEXT NOT NULL, content_base64 TEXT NOT NULL, tool_call_id TEXT, updated_at TEXT NOT NULL)`, + ) + await database.execute( + `CREATE TABLE IF NOT EXISTS tool_approvals (id TEXT PRIMARY KEY, tool_call_id TEXT NOT NULL UNIQUE, tool_name TEXT NOT NULL, args TEXT NOT NULL, reason TEXT, to_allow TEXT, context TEXT NOT NULL CHECK(context IN ('thread', 'subagent')), subagent_tool_name TEXT, parent_tool_call_id TEXT, timestamp INTEGER NOT NULL, matched_rule TEXT, rule_source TEXT CHECK(rule_source IN ('user', 'built-in')))`, + ) + await database.execute( + `CREATE INDEX IF NOT EXISTS idx_tool_approvals_timestamp ON tool_approvals(timestamp)`, + ) + await database.execute( + `CREATE TABLE IF NOT EXISTS compaction_summaries (summary_id TEXT PRIMARY KEY, summary_text TEXT NOT NULL, cut_message_id TEXT NOT NULL, created_at TEXT NOT NULL)`, + ) +} + +async function seedSlowReconnectData(database: RawRivetDB): Promise<{ + seeded: boolean + messages: number + toolCalls: number + threadEvents: number +}> { + const existing = await database.execute(`SELECT COUNT(*) AS count FROM messages`) + if (Number(existing[0]?.count ?? 0) > 0) { + return { + seeded: false, + messages: MESSAGE_COUNT, + toolCalls: TOOL_CALL_COUNT, + threadEvents: THREAD_EVENT_COUNT, + } + } + + const now = new Date('2026-05-16T03:58:18.661Z').getTime() + const text = (size: number) => 'x'.repeat(size) + const isoAt = (index: number) => new Date(now + index * 1_000).toISOString() + + await batchInsert(database, `INSERT INTO thread_meta_kv (key, value, updated_at)`, [ + ['executor_type', 'local-client', isoAt(0)], + ['sandbox_intent', JSON.stringify({ spec: null, pendingLaunch: null }), isoAt(0)], + ['executor_status', JSON.stringify({ available: true, message: 'ready' }), isoAt(0)], + ]) + + const messageRows: unknown[][] = [] + for (let index = 1; index <= MESSAGE_COUNT; index++) { + const role = index % 2 === 0 ? 'assistant' : 'user' + messageRows.push([ + messageId(index), + role, + text(MESSAGE_CONTENT_BYTES), + null, + null, + isoAt(index), + 0, + null, + null, + null, + ]) + } + await batchInsert( + database, + `INSERT INTO messages (message_id, role, content, meta, user_state, created_at, cancelled, read_at, parent_tool_use_id, tool_result_for_message_id)`, + messageRows, + 20, + ) + + const messageToolRefRows: unknown[][] = [] + for (let index = 0; index < MESSAGE_TOOL_REF_COUNT / 2; index++) { + const assistantIndex = 2 + (index % 42) * 2 + const sourceIndex = Math.max(1, assistantIndex - 1) + const resultIndex = Math.min(MESSAGE_COUNT, assistantIndex + 1) + const toolUseId = toolUseID(index + 1) + messageToolRefRows.push([ + messageId(sourceIndex), + messageId(assistantIndex), + toolUseId, + 'tool_use', + 0, + ]) + messageToolRefRows.push([ + messageId(resultIndex), + messageId(assistantIndex), + toolUseId, + 'tool_result', + 0, + ]) + } + await batchInsert( + database, + `INSERT INTO message_tool_refs (source_message_id, assistant_message_id, tool_use_id, block_type, cancelled)`, + messageToolRefRows, + 50, + ) + + const toolCallRows: unknown[][] = [] + for (let index = 1; index <= TOOL_CALL_COUNT; index++) { + const assistantIndex = 2 + ((index - 1) % 42) * 2 + toolCallRows.push([ + toolUseID(index), + `provider-${index}`, + `tool_${index % 21}`, + JSON.stringify({ path: `/tmp/file-${index}` }), + 'seed-executor', + messageId(assistantIndex), + isoAt(index), + null, + 'completed', + JSON.stringify({ + ok: true, + run: { status: 'done', result: text(TOOL_CALL_RESULT_BYTES) }, + }), + null, + isoAt(index + 100), + ]) + } + await batchInsert( + database, + `INSERT INTO tool_calls (id, provider_tool_use_id, tool_name, args, executor_id, message_id, issued_at, expires_at, state, result, progress, completed_at)`, + toolCallRows, + 20, + ) + + const executorToolRows: unknown[][] = [] + for (let index = 1; index <= EXECUTOR_TOOL_COUNT; index++) { + const schema = JSON.stringify({ + name: `tool_${index}`, + description: text(EXECUTOR_TOOL_SCHEMA_BYTES), + input_schema: { type: 'object', properties: {} }, + }) + executorToolRows.push(['seed-executor', `tool_${index}`, schema, isoAt(index)]) + } + await batchInsert( + database, + `INSERT INTO executor_tools (executor_id, tool_name, schema, updated_at)`, + executorToolRows, + 42, + ) + + const threadEventRows: unknown[][] = [] + for (let index = 1; index <= THREAD_EVENT_COUNT; index++) { + threadEventRows.push([ + index, + index % 3 === 0 ? 'message_added' : 'agent_state_changed', + JSON.stringify({ type: 'seed_event', body: text(THREAD_EVENT_PAYLOAD_BYTES) }), + isoAt(index), + ]) + } + await batchInsert( + database, + `INSERT INTO thread_events (seq, event_type, payload, created_at)`, + threadEventRows, + 25, + ) + + const messageAddedRows: unknown[][] = [] + for (let index = 1; index <= MESSAGE_COUNT; index++) { + messageAddedRows.push([messageId(index), index]) + } + await batchInsert( + database, + `INSERT INTO message_added_events (message_id, seq)`, + messageAddedRows, + 50, + ) + + await database.execute( + `INSERT INTO environment_snapshot (id, snapshot, updated_at) VALUES (1, ?, ?)`, + JSON.stringify({ cwd: '/workspace', body: text(3_620) }), + isoAt(0), + ) + await database.execute( + `INSERT INTO thread_settings_snapshot (id, settings, updated_at) VALUES (1, ?, ?)`, + JSON.stringify({ maxTokens: 20_000, body: text(55) }), + isoAt(0), + ) + return { + seeded: true, + messages: MESSAGE_COUNT, + toolCalls: TOOL_CALL_COUNT, + threadEvents: THREAD_EVENT_COUNT, + } +} + +async function batchInsert( + database: RawRivetDB, + insertPrefix: string, + rows: unknown[][], + batchSize = 100, +): Promise { + if (rows.length === 0) { + return + } + const columnCount = rows[0]?.length ?? 0 + if (columnCount === 0) { + return + } + const rowPlaceholder = `(${'?,'.repeat(columnCount).slice(0, -1)})` + for (let index = 0; index < rows.length; index += batchSize) { + const chunk = rows.slice(index, index + batchSize) + const values = chunk.map(() => rowPlaceholder).join(',') + const bindings = chunk.flat() + await database.execute(`${insertPrefix} VALUES ${values}`, ...bindings) + } +} + +function messageId(index: number): string { + return `M-${String(index).padStart(22, '0')}` +} + +function toolUseID(index: number): string { + return `toolu_${String(index).padStart(22, '0')}` +} + +export const registry = setup({ + use: { slowReconnectActor }, + maxIncomingMessageSize: 5 * 1024 * 1024, + maxOutgoingMessageSize: 5 * 1024 * 1024, +}) + +if (import.meta.main) { + registry.start() +} diff --git a/examples/kitchen-sink/src/index.ts b/examples/kitchen-sink/src/index.ts index a4d5f2fb04..9a1ca53a4f 100644 --- a/examples/kitchen-sink/src/index.ts +++ b/examples/kitchen-sink/src/index.ts @@ -123,6 +123,10 @@ import { rawSqliteFuzzer } from "./actors/testing/raw-sqlite-fuzzer.ts"; import { sqliteMemoryPressure } from "./actors/testing/sqlite-memory-pressure.ts"; import { mockAgenticLoop } from "./actors/testing/mock-agentic-loop.ts"; import { sleepCloseFuzz } from "./actors/testing/sleep-close-fuzz.ts"; +import { loadTestAgent } from "./actors/testing/load-test-agent.ts"; +import { loadTestAgent2 } from "./actors/testing/load-test-agent-2.ts"; +import { sigtermSleepProbe } from "./actors/testing/sigterm-sleep-probe.ts"; +import { slowReconnectActor } from "./actors/testing/slow-reconnect-actor.ts"; // AI import { aiAgent } from "./actors/ai/ai-agent.ts"; @@ -139,6 +143,19 @@ function numberFromEnv(name: string, fallback: number): number { } function serverlessPoolConfig() { + // Running under a platform-managed serverless host (Rivet Cloud managed + // pool, or Cloud Run via Rivet's deploy pipeline). The platform configures + // the serverless runner pool on its side and issues per-namespace `sk_` + // tokens that do not have permission to list datacenters, which is what + // `configurePool` would try to do. Skip our in-process pool configuration + // in that case. + if ( + process.env._RIVET_COMPUTE === "1" || + process.env.SANDBOX_MODE === "serverless" + ) { + return undefined; + } + const url = process.env.RIVET_SERVERLESS_URL ?? process.env.KITCHEN_SINK_SERVERLESS_URL ?? @@ -279,6 +296,10 @@ export const registry = setup({ sqliteMemoryPressure, mockAgenticLoop, sleepCloseFuzz, + loadTestAgent, + loadTestAgent2, + sigtermSleepProbe, + slowReconnectActor, // AI aiAgent, }, diff --git a/examples/kitchen-sink/tsup.server.config.ts b/examples/kitchen-sink/tsup.server.config.ts new file mode 100644 index 0000000000..e57bcbe4e0 --- /dev/null +++ b/examples/kitchen-sink/tsup.server.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: { + server: "src/server.ts", + }, + format: ["esm"], + outDir: "dist-server", + outExtension: () => ({ js: ".mjs" }), + platform: "node", + loader: { + ".sql": "text", + }, + sourcemap: true, + splitting: false, + clean: true, +}); diff --git a/justfile b/justfile index 4a01794285..fe03d162c3 100644 --- a/justfile +++ b/justfile @@ -3,8 +3,8 @@ release *ARGS: pnpm --filter=publish release {{ ARGS }} [group('release')] -preview-publish: - gh workflow run .github/workflows/publish.yaml --ref "$(git rev-parse --abbrev-ref HEAD)" +preview-publish REF: + gh workflow run .github/workflows/publish.yaml --ref "{{ REF }}" [group('docker')] docker-build: diff --git a/package.json b/package.json index c1883a112f..7f62368d96 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,8 @@ "test": "npx turbo test", "test:watch": "npx turbo watch test", "check-types": "npx turbo check-types", + "counter-latency": "pnpm --dir examples/kitchen-sink counter-latency", + "slow-reconnect": "pnpm --dir examples/kitchen-sink slow-reconnect", "lint": "pnpm biome check .", "fmt": "pnpm biome check --write --diagnostic-level=error ." }, diff --git a/rivetkit-rust/packages/client-protocol/src/versioned.rs b/rivetkit-rust/packages/client-protocol/src/versioned.rs index 713ebb8e6c..e70e995664 100644 --- a/rivetkit-rust/packages/client-protocol/src/versioned.rs +++ b/rivetkit-rust/packages/client-protocol/src/versioned.rs @@ -268,10 +268,14 @@ macro_rules! impl_to_server_pair { macro_rules! impl_common_pair { ($left:ident, $right:ident) => { impl_same_fields_pair!($left, $right, ActionRequest { id, name, args }); - impl_same_fields_pair!($left, $right, SubscriptionRequest { - event_name, - subscribe, - }); + impl_same_fields_pair!( + $left, + $right, + SubscriptionRequest { + event_name, + subscribe, + } + ); impl_to_server_pair!($left, $right); impl_same_fields_pair!($left, $right, HttpActionRequest { args }); impl_same_fields_pair!($left, $right, HttpActionResponse { output }); @@ -281,17 +285,25 @@ macro_rules! impl_common_pair { macro_rules! impl_to_client_v2_v3_pair { () => { - impl_same_fields_pair!(v2, v3, Init { - actor_id, - connection_id, - }); - impl_same_fields_pair!(v2, v3, Error { - group, - code, - message, - metadata, - action_id, - }); + impl_same_fields_pair!( + v2, + v3, + Init { + actor_id, + connection_id, + } + ); + impl_same_fields_pair!( + v2, + v3, + Error { + group, + code, + message, + metadata, + action_id, + } + ); impl_same_fields_pair!(v2, v3, ActionResponse { id, output }); impl_same_fields_pair!(v2, v3, Event { name, args }); @@ -343,24 +355,36 @@ impl_common_pair!(v1, v2); impl_common_pair!(v2, v3); impl_common_pair!(v3, v4); impl_to_client_v2_v3_pair!(); -impl_same_fields_pair!(v1, v2, HttpResponseError { - group, - code, - message, - metadata, -}); -impl_same_fields_pair!(v2, v3, HttpResponseError { - group, - code, - message, - metadata, -}); -impl_same_fields_pair!(v3, v4, HttpQueueSendRequest { - body, - name, - wait, - timeout, -}); +impl_same_fields_pair!( + v1, + v2, + HttpResponseError { + group, + code, + message, + metadata, + } +); +impl_same_fields_pair!( + v2, + v3, + HttpResponseError { + group, + code, + message, + metadata, + } +); +impl_same_fields_pair!( + v3, + v4, + HttpQueueSendRequest { + body, + name, + wait, + timeout, + } +); impl_same_fields_pair!(v3, v4, HttpQueueSendResponse { status, response }); macro_rules! impl_versioned_manual { @@ -609,7 +633,9 @@ impl OwnedVersionedData for HttpResponseError { (Self::V2(data), 2) => serde_bare::to_vec(&data).map_err(Into::into), (Self::V3(data), 3) => serde_bare::to_vec(&data).map_err(Into::into), (Self::V4(data), 4) => serde_bare::to_vec(&data).map_err(Into::into), - (_, version) => bail!("unexpected client protocol version for HttpResponseError: {version}"), + (_, version) => { + bail!("unexpected client protocol version for HttpResponseError: {version}") + } } } diff --git a/rivetkit-rust/packages/inspector-protocol/src/versioned.rs b/rivetkit-rust/packages/inspector-protocol/src/versioned.rs index c58f4db48f..35c757f235 100644 --- a/rivetkit-rust/packages/inspector-protocol/src/versioned.rs +++ b/rivetkit-rust/packages/inspector-protocol/src/versioned.rs @@ -69,18 +69,12 @@ impl ToServer { v1::ToServerBody::PatchStateRequest(req) => { v2::ToServerBody::PatchStateRequest(req.into()) } - v1::ToServerBody::StateRequest(req) => { - v2::ToServerBody::StateRequest(req.into()) - } + v1::ToServerBody::StateRequest(req) => v2::ToServerBody::StateRequest(req.into()), v1::ToServerBody::ConnectionsRequest(req) => { v2::ToServerBody::ConnectionsRequest(req.into()) } - v1::ToServerBody::ActionRequest(req) => { - v2::ToServerBody::ActionRequest(req.into()) - } - v1::ToServerBody::RpcsListRequest(req) => { - v2::ToServerBody::RpcsListRequest(req.into()) - } + v1::ToServerBody::ActionRequest(req) => v2::ToServerBody::ActionRequest(req.into()), + v1::ToServerBody::RpcsListRequest(req) => v2::ToServerBody::RpcsListRequest(req.into()), v1::ToServerBody::EventsRequest(_) | v1::ToServerBody::ClearEventsRequest(_) => { bail!("cannot convert inspector v1 events requests to v2") } @@ -105,24 +99,16 @@ impl ToServer { v3::ToServerBody::PatchStateRequest(req) => { v4::ToServerBody::PatchStateRequest(req.into()) } - v3::ToServerBody::StateRequest(req) => { - v4::ToServerBody::StateRequest(req.into()) - } + v3::ToServerBody::StateRequest(req) => v4::ToServerBody::StateRequest(req.into()), v3::ToServerBody::ConnectionsRequest(req) => { v4::ToServerBody::ConnectionsRequest(req.into()) } - v3::ToServerBody::ActionRequest(req) => { - v4::ToServerBody::ActionRequest(req.into()) - } - v3::ToServerBody::RpcsListRequest(req) => { - v4::ToServerBody::RpcsListRequest(req.into()) - } + v3::ToServerBody::ActionRequest(req) => v4::ToServerBody::ActionRequest(req.into()), + v3::ToServerBody::RpcsListRequest(req) => v4::ToServerBody::RpcsListRequest(req.into()), v3::ToServerBody::TraceQueryRequest(req) => { v4::ToServerBody::TraceQueryRequest(req.into()) } - v3::ToServerBody::QueueRequest(req) => { - v4::ToServerBody::QueueRequest(req.into()) - } + v3::ToServerBody::QueueRequest(req) => v4::ToServerBody::QueueRequest(req.into()), v3::ToServerBody::WorkflowHistoryRequest(req) => { v4::ToServerBody::WorkflowHistoryRequest(req.into()) } @@ -146,24 +132,16 @@ impl ToServer { v4::ToServerBody::PatchStateRequest(req) => { v3::ToServerBody::PatchStateRequest(req.into()) } - v4::ToServerBody::StateRequest(req) => { - v3::ToServerBody::StateRequest(req.into()) - } + v4::ToServerBody::StateRequest(req) => v3::ToServerBody::StateRequest(req.into()), v4::ToServerBody::ConnectionsRequest(req) => { v3::ToServerBody::ConnectionsRequest(req.into()) } - v4::ToServerBody::ActionRequest(req) => { - v3::ToServerBody::ActionRequest(req.into()) - } - v4::ToServerBody::RpcsListRequest(req) => { - v3::ToServerBody::RpcsListRequest(req.into()) - } + v4::ToServerBody::ActionRequest(req) => v3::ToServerBody::ActionRequest(req.into()), + v4::ToServerBody::RpcsListRequest(req) => v3::ToServerBody::RpcsListRequest(req.into()), v4::ToServerBody::TraceQueryRequest(req) => { v3::ToServerBody::TraceQueryRequest(req.into()) } - v4::ToServerBody::QueueRequest(req) => { - v3::ToServerBody::QueueRequest(req.into()) - } + v4::ToServerBody::QueueRequest(req) => v3::ToServerBody::QueueRequest(req.into()), v4::ToServerBody::WorkflowHistoryRequest(req) => { v3::ToServerBody::WorkflowHistoryRequest(req.into()) } @@ -190,24 +168,16 @@ impl ToServer { v3::ToServerBody::PatchStateRequest(req) => { v2::ToServerBody::PatchStateRequest(req.into()) } - v3::ToServerBody::StateRequest(req) => { - v2::ToServerBody::StateRequest(req.into()) - } + v3::ToServerBody::StateRequest(req) => v2::ToServerBody::StateRequest(req.into()), v3::ToServerBody::ConnectionsRequest(req) => { v2::ToServerBody::ConnectionsRequest(req.into()) } - v3::ToServerBody::ActionRequest(req) => { - v2::ToServerBody::ActionRequest(req.into()) - } - v3::ToServerBody::RpcsListRequest(req) => { - v2::ToServerBody::RpcsListRequest(req.into()) - } + v3::ToServerBody::ActionRequest(req) => v2::ToServerBody::ActionRequest(req.into()), + v3::ToServerBody::RpcsListRequest(req) => v2::ToServerBody::RpcsListRequest(req.into()), v3::ToServerBody::TraceQueryRequest(req) => { v2::ToServerBody::TraceQueryRequest(req.into()) } - v3::ToServerBody::QueueRequest(req) => { - v2::ToServerBody::QueueRequest(req.into()) - } + v3::ToServerBody::QueueRequest(req) => v2::ToServerBody::QueueRequest(req.into()), v3::ToServerBody::WorkflowHistoryRequest(req) => { v2::ToServerBody::WorkflowHistoryRequest(req.into()) } @@ -229,18 +199,12 @@ impl ToServer { v2::ToServerBody::PatchStateRequest(req) => { v1::ToServerBody::PatchStateRequest(req.into()) } - v2::ToServerBody::StateRequest(req) => { - v1::ToServerBody::StateRequest(req.into()) - } + v2::ToServerBody::StateRequest(req) => v1::ToServerBody::StateRequest(req.into()), v2::ToServerBody::ConnectionsRequest(req) => { v1::ToServerBody::ConnectionsRequest(req.into()) } - v2::ToServerBody::ActionRequest(req) => { - v1::ToServerBody::ActionRequest(req.into()) - } - v2::ToServerBody::RpcsListRequest(req) => { - v1::ToServerBody::RpcsListRequest(req.into()) - } + v2::ToServerBody::ActionRequest(req) => v1::ToServerBody::ActionRequest(req.into()), + v2::ToServerBody::RpcsListRequest(req) => v1::ToServerBody::RpcsListRequest(req.into()), v2::ToServerBody::TraceQueryRequest(_) | v2::ToServerBody::QueueRequest(_) | v2::ToServerBody::WorkflowHistoryRequest(_) => { @@ -309,24 +273,18 @@ impl ToClient { }; let body = match data.body { - v1::ToClientBody::StateResponse(resp) => { - v2::ToClientBody::StateResponse(resp.into()) - } + v1::ToClientBody::StateResponse(resp) => v2::ToClientBody::StateResponse(resp.into()), v1::ToClientBody::ConnectionsResponse(resp) => { v2::ToClientBody::ConnectionsResponse(resp.into()) } - v1::ToClientBody::ActionResponse(resp) => { - v2::ToClientBody::ActionResponse(resp.into()) - } + v1::ToClientBody::ActionResponse(resp) => v2::ToClientBody::ActionResponse(resp.into()), v1::ToClientBody::RpcsListResponse(resp) => { v2::ToClientBody::RpcsListResponse(resp.into()) } v1::ToClientBody::ConnectionsUpdated(update) => { v2::ToClientBody::ConnectionsUpdated(update.into()) } - v1::ToClientBody::StateUpdated(update) => { - v2::ToClientBody::StateUpdated(update.into()) - } + v1::ToClientBody::StateUpdated(update) => v2::ToClientBody::StateUpdated(update.into()), v1::ToClientBody::Error(error) => v2::ToClientBody::Error(error.into()), v1::ToClientBody::Init(init) => v2::ToClientBody::Init(v2::Init { connections: convert_vec(init.connections), @@ -359,24 +317,16 @@ impl ToClient { }; let body = match data.body { - v3::ToClientBody::StateResponse(resp) => { - v4::ToClientBody::StateResponse(resp.into()) - } + v3::ToClientBody::StateResponse(resp) => v4::ToClientBody::StateResponse(resp.into()), v3::ToClientBody::ConnectionsResponse(resp) => { v4::ToClientBody::ConnectionsResponse(resp.into()) } - v3::ToClientBody::ActionResponse(resp) => { - v4::ToClientBody::ActionResponse(resp.into()) - } + v3::ToClientBody::ActionResponse(resp) => v4::ToClientBody::ActionResponse(resp.into()), v3::ToClientBody::ConnectionsUpdated(update) => { v4::ToClientBody::ConnectionsUpdated(update.into()) } - v3::ToClientBody::QueueUpdated(update) => { - v4::ToClientBody::QueueUpdated(update.into()) - } - v3::ToClientBody::StateUpdated(update) => { - v4::ToClientBody::StateUpdated(update.into()) - } + v3::ToClientBody::QueueUpdated(update) => v4::ToClientBody::QueueUpdated(update.into()), + v3::ToClientBody::StateUpdated(update) => v4::ToClientBody::StateUpdated(update.into()), v3::ToClientBody::WorkflowHistoryUpdated(update) => { v4::ToClientBody::WorkflowHistoryUpdated(update.into()) } @@ -386,9 +336,7 @@ impl ToClient { v3::ToClientBody::TraceQueryResponse(resp) => { v4::ToClientBody::TraceQueryResponse(resp.into()) } - v3::ToClientBody::QueueResponse(resp) => { - v4::ToClientBody::QueueResponse(resp.into()) - } + v3::ToClientBody::QueueResponse(resp) => v4::ToClientBody::QueueResponse(resp.into()), v3::ToClientBody::WorkflowHistoryResponse(resp) => { v4::ToClientBody::WorkflowHistoryResponse(resp.into()) } @@ -411,24 +359,16 @@ impl ToClient { }; let body = match data.body { - v4::ToClientBody::StateResponse(resp) => { - v3::ToClientBody::StateResponse(resp.into()) - } + v4::ToClientBody::StateResponse(resp) => v3::ToClientBody::StateResponse(resp.into()), v4::ToClientBody::ConnectionsResponse(resp) => { v3::ToClientBody::ConnectionsResponse(resp.into()) } - v4::ToClientBody::ActionResponse(resp) => { - v3::ToClientBody::ActionResponse(resp.into()) - } + v4::ToClientBody::ActionResponse(resp) => v3::ToClientBody::ActionResponse(resp.into()), v4::ToClientBody::ConnectionsUpdated(update) => { v3::ToClientBody::ConnectionsUpdated(update.into()) } - v4::ToClientBody::QueueUpdated(update) => { - v3::ToClientBody::QueueUpdated(update.into()) - } - v4::ToClientBody::StateUpdated(update) => { - v3::ToClientBody::StateUpdated(update.into()) - } + v4::ToClientBody::QueueUpdated(update) => v3::ToClientBody::QueueUpdated(update.into()), + v4::ToClientBody::StateUpdated(update) => v3::ToClientBody::StateUpdated(update.into()), v4::ToClientBody::WorkflowHistoryUpdated(update) => { v3::ToClientBody::WorkflowHistoryUpdated(update.into()) } @@ -438,9 +378,7 @@ impl ToClient { v4::ToClientBody::TraceQueryResponse(resp) => { v3::ToClientBody::TraceQueryResponse(resp.into()) } - v4::ToClientBody::QueueResponse(resp) => { - v3::ToClientBody::QueueResponse(resp.into()) - } + v4::ToClientBody::QueueResponse(resp) => v3::ToClientBody::QueueResponse(resp.into()), v4::ToClientBody::WorkflowHistoryResponse(resp) => { v3::ToClientBody::WorkflowHistoryResponse(resp.into()) } @@ -466,24 +404,16 @@ impl ToClient { }; let body = match data.body { - v3::ToClientBody::StateResponse(resp) => { - v2::ToClientBody::StateResponse(resp.into()) - } + v3::ToClientBody::StateResponse(resp) => v2::ToClientBody::StateResponse(resp.into()), v3::ToClientBody::ConnectionsResponse(resp) => { v2::ToClientBody::ConnectionsResponse(resp.into()) } - v3::ToClientBody::ActionResponse(resp) => { - v2::ToClientBody::ActionResponse(resp.into()) - } + v3::ToClientBody::ActionResponse(resp) => v2::ToClientBody::ActionResponse(resp.into()), v3::ToClientBody::ConnectionsUpdated(update) => { v2::ToClientBody::ConnectionsUpdated(update.into()) } - v3::ToClientBody::QueueUpdated(update) => { - v2::ToClientBody::QueueUpdated(update.into()) - } - v3::ToClientBody::StateUpdated(update) => { - v2::ToClientBody::StateUpdated(update.into()) - } + v3::ToClientBody::QueueUpdated(update) => v2::ToClientBody::QueueUpdated(update.into()), + v3::ToClientBody::StateUpdated(update) => v2::ToClientBody::StateUpdated(update.into()), v3::ToClientBody::WorkflowHistoryUpdated(update) => { v2::ToClientBody::WorkflowHistoryUpdated(update.into()) } @@ -493,9 +423,7 @@ impl ToClient { v3::ToClientBody::TraceQueryResponse(resp) => { v2::ToClientBody::TraceQueryResponse(resp.into()) } - v3::ToClientBody::QueueResponse(resp) => { - v2::ToClientBody::QueueResponse(resp.into()) - } + v3::ToClientBody::QueueResponse(resp) => v2::ToClientBody::QueueResponse(resp.into()), v3::ToClientBody::WorkflowHistoryResponse(resp) => { v2::ToClientBody::WorkflowHistoryResponse(resp.into()) } @@ -516,21 +444,15 @@ impl ToClient { }; let body = match data.body { - v2::ToClientBody::StateResponse(resp) => { - v1::ToClientBody::StateResponse(resp.into()) - } + v2::ToClientBody::StateResponse(resp) => v1::ToClientBody::StateResponse(resp.into()), v2::ToClientBody::ConnectionsResponse(resp) => { v1::ToClientBody::ConnectionsResponse(resp.into()) } - v2::ToClientBody::ActionResponse(resp) => { - v1::ToClientBody::ActionResponse(resp.into()) - } + v2::ToClientBody::ActionResponse(resp) => v1::ToClientBody::ActionResponse(resp.into()), v2::ToClientBody::ConnectionsUpdated(update) => { v1::ToClientBody::ConnectionsUpdated(update.into()) } - v2::ToClientBody::StateUpdated(update) => { - v1::ToClientBody::StateUpdated(update.into()) - } + v2::ToClientBody::StateUpdated(update) => v1::ToClientBody::StateUpdated(update.into()), v2::ToClientBody::RpcsListResponse(resp) => { v1::ToClientBody::RpcsListResponse(resp.into()) } @@ -720,11 +642,15 @@ macro_rules! impl_common_actor_pair { impl_same_fields_pair!($left, $right, Connection { id, details }); impl_connections_response_pair!($left, $right); impl_connection_list_pair!($left, $right, ConnectionsUpdated); - impl_same_fields_pair!($left, $right, StateResponse { - rid, - state, - is_state_enabled, - }); + impl_same_fields_pair!( + $left, + $right, + StateResponse { + rid, + state, + is_state_enabled, + } + ); impl_same_fields_pair!($left, $right, ActionResponse { rid, output }); impl_same_fields_pair!($left, $right, StateUpdated { state }); impl_same_fields_pair!($left, $right, RpcsListResponse { rid, rpcs }); @@ -734,28 +660,40 @@ macro_rules! impl_common_actor_pair { macro_rules! impl_queue_workflow_pair { ($left:ident, $right:ident) => { - impl_same_fields_pair!($left, $right, TraceQueryRequest { - id, - start_ms, - end_ms, - limit, - }); + impl_same_fields_pair!( + $left, + $right, + TraceQueryRequest { + id, + start_ms, + end_ms, + limit, + } + ); impl_same_fields_pair!($left, $right, TraceQueryResponse { rid, payload }); impl_same_fields_pair!($left, $right, QueueRequest { id, limit }); - impl_same_fields_pair!($left, $right, QueueMessageSummary { - id, - name, - created_at_ms, - }); + impl_same_fields_pair!( + $left, + $right, + QueueMessageSummary { + id, + name, + created_at_ms, + } + ); impl_queue_status_pair!($left, $right); impl_queue_response_pair!($left, $right); impl_same_fields_pair!($left, $right, QueueUpdated { queue_size }); impl_same_fields_pair!($left, $right, WorkflowHistoryRequest { id }); - impl_same_fields_pair!($left, $right, WorkflowHistoryResponse { - rid, - history, - is_workflow_enabled, - }); + impl_same_fields_pair!( + $left, + $right, + WorkflowHistoryResponse { + rid, + history, + is_workflow_enabled, + } + ); impl_same_fields_pair!($left, $right, WorkflowHistoryUpdated { history }); impl_init_pair!($left, $right); }; @@ -765,12 +703,16 @@ macro_rules! impl_database_pair { ($left:ident, $right:ident) => { impl_same_fields_pair!($left, $right, DatabaseSchemaRequest { id }); impl_same_fields_pair!($left, $right, DatabaseSchemaResponse { rid, schema }); - impl_same_fields_pair!($left, $right, DatabaseTableRowsRequest { - id, - table, - limit, - offset, - }); + impl_same_fields_pair!( + $left, + $right, + DatabaseTableRowsRequest { + id, + table, + limit, + offset, + } + ); impl_same_fields_pair!($left, $right, DatabaseTableRowsResponse { rid, result }); }; } @@ -813,9 +755,7 @@ impl From for v3::ToClientBody { v2::ToClientBody::StateResponse(resp) => Self::StateResponse(resp.into()), v2::ToClientBody::ConnectionsResponse(resp) => Self::ConnectionsResponse(resp.into()), v2::ToClientBody::ActionResponse(resp) => Self::ActionResponse(resp.into()), - v2::ToClientBody::ConnectionsUpdated(update) => { - Self::ConnectionsUpdated(update.into()) - } + v2::ToClientBody::ConnectionsUpdated(update) => Self::ConnectionsUpdated(update.into()), v2::ToClientBody::QueueUpdated(update) => Self::QueueUpdated(update.into()), v2::ToClientBody::StateUpdated(update) => Self::StateUpdated(update.into()), v2::ToClientBody::WorkflowHistoryUpdated(update) => { diff --git a/rivetkit-rust/packages/rivetkit-core/src/actor/connection.rs b/rivetkit-rust/packages/rivetkit-core/src/actor/connection.rs index 17e119f963..ec5a551e62 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/actor/connection.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/actor/connection.rs @@ -3,7 +3,7 @@ use std::fmt; use std::ops::Bound::{Excluded, Unbounded}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use futures::future::BoxFuture; @@ -41,6 +41,41 @@ pub(crate) struct OutgoingEvent { pub args: Vec, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ConnectionCloseReason { + WsClose, + WsError, + Shutdown, + Orphaned, +} + +impl ConnectionCloseReason { + fn as_label(self) -> &'static str { + match self { + Self::WsClose => "ws_close", + Self::WsError => "ws_error", + Self::Shutdown => "shutdown", + Self::Orphaned => "orphaned", + } + } + + fn from_disconnect_reason(reason: Option<&str>) -> Self { + let Some(reason) = reason else { + return Self::WsClose; + }; + let reason = reason.to_ascii_lowercase(); + if reason.contains("shutdown") { + Self::Shutdown + } else if reason.contains("orphan") { + Self::Orphaned + } else if reason.contains("error") { + Self::WsError + } else { + Self::WsClose + } + } +} + #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct HibernatableConnectionMetadata { pub gateway_id: [u8; 4], @@ -151,6 +186,7 @@ pub struct ConnHandle(Arc); struct ConnHandleInner { id: ConnId, params: Vec, + created_at: Instant, // Forced-sync: connection handles expose synchronous state and callback // methods to foreign runtimes; callbacks are cloned before async work. state: RwLock>, @@ -174,6 +210,7 @@ impl ConnHandle { Self(Arc::new(ConnHandleInner { id: id.into(), params, + created_at: Instant::now(), state: RwLock::new(state), is_hibernatable, dirty: AtomicBool::new(false), @@ -194,6 +231,10 @@ impl ConnHandle { self.0.params.clone() } + pub(crate) fn lifetime(&self) -> Duration { + self.0.created_at.elapsed() + } + pub fn state(&self) -> Vec { self.0.state.read().clone() } @@ -517,7 +558,16 @@ impl ActorContext { removed } + #[cfg(test)] fn remove_existing_for_disconnect(&self, conn_id: &str) -> Option { + self.remove_existing_for_disconnect_with_reason(conn_id, ConnectionCloseReason::WsClose) + } + + fn remove_existing_for_disconnect_with_reason( + &self, + conn_id: &str, + reason: ConnectionCloseReason, + ) -> Option { let _disconnect_state = self.0.connection_disconnect_state.lock(); let (removed, active_count) = { let mut connections = self.0.connections.write(); @@ -534,6 +584,9 @@ impl ActorContext { (removed, connections.len()) }; self.0.metrics.set_active_connections(active_count); + self.0 + .metrics + .record_connection_closed(reason.as_label(), removed.lifetime()); tracing::debug!( actor_id = %self.actor_id(), conn_id, @@ -842,7 +895,10 @@ impl ActorContext { } async fn disconnect_managed(&self, conn_id: &str, reason: Option) -> Result<()> { - let Some(conn) = self.remove_existing_for_disconnect(conn_id) else { + let close_reason = ConnectionCloseReason::from_disconnect_reason(reason.as_deref()); + let Some(conn) = + self.remove_existing_for_disconnect_with_reason(conn_id, close_reason) + else { tracing::debug!( actor_id = %self.actor_id(), conn_id, @@ -954,7 +1010,10 @@ impl ActorContext { let mut removed_any = false; for conn_id in disconnected_ids { - let Some(conn) = self.remove_existing_for_disconnect(&conn_id) else { + let Some(conn) = self.remove_existing_for_disconnect_with_reason( + &conn_id, + ConnectionCloseReason::WsClose, + ) else { tracing::debug!( actor_id = %self.actor_id(), conn_id = %conn_id, diff --git a/rivetkit-rust/packages/rivetkit-core/src/actor/context.rs b/rivetkit-rust/packages/rivetkit-core/src/actor/context.rs index e67c096f3a..69a19bc821 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/actor/context.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/actor/context.rs @@ -10,9 +10,9 @@ use crate::time::{Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context as AnyhowContext, Result}; use futures::future::BoxFuture; use parking_lot::{Mutex, RwLock}; -use rivet_error::ActorSpecifier; use rivet_envoy_client::handle::EnvoyHandle; use rivet_envoy_client::tunnel::HibernatingWebSocketMetadata; +use rivet_error::ActorSpecifier; use scc::HashMap as SccHashMap; use tokio::runtime::Handle; use tokio::sync::{Mutex as AsyncMutex, Notify, OnceCell, broadcast, mpsc, oneshot}; diff --git a/rivetkit-rust/packages/rivetkit-core/src/actor/metrics.rs b/rivetkit-rust/packages/rivetkit-core/src/actor/metrics.rs index 6d5f6e11c3..5191892d75 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/actor/metrics.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/actor/metrics.rs @@ -22,6 +22,7 @@ const DIRECT_SHUTDOWN_LABELS: &[&str] = &[ "subsystem", "operation", ]; +const CONNECTION_CLOSE_LABELS: &[&str] = &["actor_id_gen", "actor_key", "envoy_key", "reason"]; #[cfg(feature = "sqlite-local")] const SQLITE_COMMIT_PHASE_LABELS: &[&str] = &["actor_id_gen", "actor_key", "envoy_key", "phase"]; @@ -68,6 +69,8 @@ struct ActorMetricCollectors { queue_messages_received_total: IntCounterVec, active_connections: IntGaugeVec, connections_total: IntCounterVec, + connection_closed_total: IntCounterVec, + connection_lifetime_seconds: HistogramVec, lifecycle_inbox_depth: IntGaugeVec, dispatch_inbox_depth: IntGaugeVec, lifecycle_event_inbox_depth: IntGaugeVec, @@ -185,6 +188,22 @@ impl ActorMetricCollectors { ACTOR_LABELS, ) .expect("create actor_connections_total counter"); + let connection_closed_total = IntCounterVec::new( + Opts::new( + "actor_connection_closed_total", + "total actor connections closed by close reason", + ), + CONNECTION_CLOSE_LABELS, + ) + .expect("create actor_connection_closed_total counter"); + let connection_lifetime_seconds = HistogramVec::new( + HistogramOpts::new( + "actor_connection_lifetime_seconds", + "actor connection lifetime in seconds", + ), + ACTOR_LABELS, + ) + .expect("create actor_connection_lifetime_seconds histogram"); let lifecycle_inbox_depth = IntGaugeVec::new( Opts::new( "actor_lifecycle_inbox_depth", @@ -455,6 +474,11 @@ impl ActorMetricCollectors { register_metric(&rivet_metrics::REGISTRY, queue_messages_received_total.clone()); register_metric(&rivet_metrics::REGISTRY, active_connections.clone()); register_metric(&rivet_metrics::REGISTRY, connections_total.clone()); + register_metric(&rivet_metrics::REGISTRY, connection_closed_total.clone()); + register_metric( + &rivet_metrics::REGISTRY, + connection_lifetime_seconds.clone(), + ); register_metric(&rivet_metrics::REGISTRY, lifecycle_inbox_depth.clone()); register_metric(&rivet_metrics::REGISTRY, dispatch_inbox_depth.clone()); register_metric(&rivet_metrics::REGISTRY, lifecycle_event_inbox_depth.clone()); @@ -522,6 +546,8 @@ impl ActorMetricCollectors { queue_messages_received_total, active_connections, connections_total, + connection_closed_total, + connection_lifetime_seconds, lifecycle_inbox_depth, dispatch_inbox_depth, lifecycle_event_inbox_depth, @@ -653,6 +679,18 @@ impl ActorMetrics { .inc(); } + pub(crate) fn record_connection_closed(&self, reason: &str, lifetime: Duration) { + let labels = self.actor_labels(); + METRICS + .connection_closed_total + .with_label_values(&[labels[0], labels[1], labels[2], reason]) + .inc(); + METRICS + .connection_lifetime_seconds + .with_label_values(&labels) + .observe(lifetime.as_secs_f64()); + } + pub(crate) fn set_lifecycle_inbox_depth(&self, depth: usize) { METRICS .lifecycle_inbox_depth diff --git a/rivetkit-rust/packages/rivetkit-core/src/actor/sqlite.rs b/rivetkit-rust/packages/rivetkit-core/src/actor/sqlite.rs index a401cefe2b..35b3de3bbf 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/actor/sqlite.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/actor/sqlite.rs @@ -10,9 +10,9 @@ use depot_client_types::is_head_fence_mismatch; pub use depot_client_types::{BindParam, ColumnValue, ExecResult, ExecuteResult, QueryResult}; #[cfg(feature = "sqlite-local")] use parking_lot::Mutex; -use rivet_error::{ActorSpecifier, RivetError}; use rivet_envoy_client::protocol; use rivet_envoy_client::{handle::EnvoyHandle, utils::RemoteSqliteIndeterminateResultError}; +use rivet_error::{ActorSpecifier, RivetError}; use serde::Serialize; use serde_json::{Map as JsonMap, Value as JsonValue}; #[cfg(feature = "sqlite-local")] @@ -507,8 +507,7 @@ impl SqliteDb { } fn actor_specifier(&self) -> Option { - let mut specifier = - ActorSpecifier::new(self.actor_id.as_ref()?.clone(), self.generation?); + let mut specifier = ActorSpecifier::new(self.actor_id.as_ref()?.clone(), self.generation?); if let Some(key) = self.actor_key.as_ref() { specifier = specifier.with_key(key.clone()); } diff --git a/rivetkit-rust/packages/rivetkit-core/src/actor/task.rs b/rivetkit-rust/packages/rivetkit-core/src/actor/task.rs index 2f771e5c28..1fec4ec76d 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/actor/task.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/actor/task.rs @@ -280,6 +280,7 @@ enum LiveExit { struct SleepGraceState { deadline: Instant, reason: ShutdownKind, + started_at: Instant, } struct PersistedStartup { @@ -890,7 +891,8 @@ impl ActorTask { let _action_keep_awake = action_keep_awake; match tracked_reply_rx.await { Ok(result) => { - let result = result.map_err(|error| ctx.attach_actor_to_error(error)); + let result = + result.map_err(|error| ctx.attach_actor_to_error(error)); tracing::info!( actor_id = %actor_id, action_name = %action_name_for_log, @@ -1459,9 +1461,11 @@ impl ActorTask { self.sleep_deadline = None; self.ctx.cancel_sleep_timer(); self.ctx.cancel_actor_abort_signal(); + let started_at = Instant::now(); self.sleep_grace = Some(SleepGraceState { - deadline: Instant::now() + grace_period, + deadline: started_at + grace_period, reason, + started_at, }); self.ctx.reset_sleep_timer(); } @@ -1498,6 +1502,23 @@ impl ActorTask { }; if self.ctx.can_finalize_shutdown(grace.reason) { let reason = grace.reason; + let grace_elapsed_ms = Instant::now() + .saturating_duration_since(grace.started_at) + .as_millis() as u64; + tracing::debug!( + actor_id = %self.ctx.actor_id(), + reason = shutdown_reason_label(reason), + grace_elapsed_ms, + core_dispatched_hook_count = self.ctx.core_dispatched_hook_count(), + shutdown_task_count = self.ctx.shutdown_task_count(), + sleep_keep_awake_count = self.ctx.sleep_keep_awake_count(), + sleep_internal_keep_awake_count = self.ctx.sleep_internal_keep_awake_count(), + active_http_request_count = self.ctx.active_http_request_count(), + websocket_callback_count = self.ctx.websocket_callback_count(), + pending_disconnect_count = self.ctx.pending_disconnect_count(), + connection_count = self.ctx.conns().len(), + "actor shutdown grace drained, all gates satisfied" + ); self.sleep_grace = None; return Some(LiveExit::Shutdown { reason }); } diff --git a/rivetkit-rust/packages/rivetkit-core/src/lib.rs b/rivetkit-rust/packages/rivetkit-core/src/lib.rs index 76e4bce759..eb954fa224 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/lib.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/lib.rs @@ -11,8 +11,10 @@ pub mod actor; pub mod engine_process; pub mod error; pub mod inspector; +pub mod metrics_endpoint; pub mod registry; pub mod runtime; +pub mod runtime_metrics; pub mod serverless; pub(crate) mod time { use std::fmt; diff --git a/rivetkit-rust/packages/rivetkit-core/src/metrics_endpoint.rs b/rivetkit-rust/packages/rivetkit-core/src/metrics_endpoint.rs new file mode 100644 index 0000000000..8c8d326c25 --- /dev/null +++ b/rivetkit-rust/packages/rivetkit-core/src/metrics_endpoint.rs @@ -0,0 +1,159 @@ +use std::collections::HashMap; +use std::sync::LazyLock; + +use anyhow::{Context, Result}; +use rivet_metrics::prometheus::{Encoder, Gauge, IntGauge, Opts, Registry, TextEncoder}; +use subtle::ConstantTimeEq; + +use crate::registry::CoreEnvoyStatus; + +const METRICS_ENABLED_ENV: &str = "RIVETKIT_METRICS_ENABLED"; +const METRICS_TOKEN_ENV: &str = "RIVETKIT_METRICS_TOKEN"; + +struct EnvoyMetricCollectors { + last_ping_timestamp_seconds: Gauge, + ping_healthy: IntGauge, +} + +static ENVOY_METRICS: LazyLock = + LazyLock::new(EnvoyMetricCollectors::new); + +pub struct RenderedMetrics { + pub content_type: String, + pub body: Vec, +} + +pub enum MetricsAccessError { + NotEnabled, + Unauthorized, +} + +pub fn authorize_metrics_request( + bearer_token: Option<&str>, +) -> std::result::Result<(), MetricsAccessError> { + let Some(configured_token) = configured_metrics_token() else { + return Err(MetricsAccessError::NotEnabled); + }; + + let Some(bearer_token) = bearer_token.filter(|token| !token.is_empty()) else { + return Err(MetricsAccessError::Unauthorized); + }; + + if bearer_token + .as_bytes() + .ct_eq(configured_token.as_bytes()) + .into() + { + Ok(()) + } else { + Err(MetricsAccessError::Unauthorized) + } +} + +pub fn render_prometheus_metrics( + envoy_status: Option<&CoreEnvoyStatus>, +) -> Result { + ENVOY_METRICS.refresh(envoy_status); + + let encoder = TextEncoder::new(); + let metric_families = rivet_metrics::REGISTRY.gather(); + let mut body = Vec::new(); + encoder + .encode(&metric_families, &mut body) + .context("encode prometheus metrics")?; + + Ok(RenderedMetrics { + content_type: encoder.format_type().to_owned(), + body, + }) +} + +impl EnvoyMetricCollectors { + fn new() -> Self { + let last_ping_timestamp_seconds = Gauge::with_opts(Opts::new( + "rivetkit_envoy_last_ping_timestamp_seconds", + "unix timestamp of the most recent engine ping received by rivetkit", + )) + .expect("create envoy last ping timestamp gauge"); + let ping_healthy = IntGauge::with_opts(Opts::new( + "rivetkit_envoy_ping_healthy", + "whether rivetkit has received a recent engine ping", + )) + .expect("create envoy ping healthy gauge"); + + register_metric(&rivet_metrics::REGISTRY, last_ping_timestamp_seconds.clone()); + register_metric(&rivet_metrics::REGISTRY, ping_healthy.clone()); + + Self { + last_ping_timestamp_seconds, + ping_healthy, + } + } + + fn refresh(&self, envoy_status: Option<&CoreEnvoyStatus>) { + let last_ping_timestamp_seconds = envoy_status + .and_then(|status| status.last_ping_at_ms) + .map(|ts| ts as f64 / 1_000.0) + .unwrap_or(0.0); + let ping_healthy = envoy_status + .map(|status| if status.ping_healthy { 1 } else { 0 }) + .unwrap_or(0); + + self.last_ping_timestamp_seconds + .set(last_ping_timestamp_seconds); + self.ping_healthy.set(ping_healthy); + } +} + +fn register_metric(registry: &Registry, metric: M) +where + M: rivet_metrics::prometheus::core::Collector + Clone + Send + Sync + 'static, +{ + if let Err(error) = registry.register(Box::new(metric)) { + tracing::warn!( + ?error, + "envoy metric registration failed, using existing collector" + ); + } +} + +pub fn authorization_bearer_token(headers: &http::HeaderMap) -> Option<&str> { + headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(bearer_token_from_authorization) +} + +pub fn authorization_bearer_token_map(headers: &HashMap) -> Option<&str> { + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(http::header::AUTHORIZATION.as_str())) + .and_then(|(_, value)| bearer_token_from_authorization(value)) +} + +fn configured_metrics_token() -> Option { + let enabled = std::env::var(METRICS_ENABLED_ENV).ok()?; + if enabled != "1" { + return None; + } + + std::env::var(METRICS_TOKEN_ENV) + .ok() + .filter(|token| !token.is_empty()) +} + +fn bearer_token_from_authorization(value: &str) -> Option<&str> { + let value = value.trim_start(); + let scheme = value.get(..6)?; + if !scheme.eq_ignore_ascii_case("bearer") { + return None; + } + + let rest = value.get(6..)?; + if !rest.chars().next().is_some_and(char::is_whitespace) { + return None; + } + + let token = rest.trim_start(); + if token.is_empty() { None } else { Some(token) } +} diff --git a/rivetkit-rust/packages/rivetkit-core/src/registry/actor_connect.rs b/rivetkit-rust/packages/rivetkit-core/src/registry/actor_connect.rs index f70cfce65d..1b9c43bd31 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/registry/actor_connect.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/registry/actor_connect.rs @@ -64,13 +64,14 @@ pub(super) fn encode_actor_connect_message(message: &ActorConnectToClient) -> Re .as_ref() .map(|metadata| metadata.as_ref().to_vec()), action_id: payload.action_id.map(serde_bare::Uint), - actor: payload.actor.as_ref().map(|actor| { - client_protocol::ActorSpecifier { + actor: payload + .actor + .as_ref() + .map(|actor| client_protocol::ActorSpecifier { actor_id: actor.actor_id.clone(), generation: serde_bare::Uint(actor.generation), key: actor.key.clone(), - } - }), + }), }) } ActorConnectToClient::ActionResponse(payload) => { diff --git a/rivetkit-rust/packages/rivetkit-core/src/registry/http.rs b/rivetkit-rust/packages/rivetkit-core/src/registry/http.rs index 3527621953..818f55331b 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/registry/http.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/registry/http.rs @@ -21,6 +21,10 @@ impl RegistryDispatcher { request.uri().path(), self.handle_inspector_http_in_runtime, )?; + if matches!(route, RegistryHttpRoute::Framework(FrameworkHttpRoute::Metrics)) { + let envoy_status = self.envoy_status(); + return handle_metrics_fetch(&request, envoy_status.as_ref()); + } let instance = match self.active_actor(actor_id).await { Ok(instance) => instance, Err(error) => { @@ -114,6 +118,10 @@ impl RegistryDispatcher { } FrameworkHttpRoute::Metadata => handle_metadata_fetch(&request, Some(&actor)), FrameworkHttpRoute::Health => handle_health_fetch(&request, Some(&actor)), + FrameworkHttpRoute::Metrics => { + let envoy_status = self.envoy_status(); + handle_metrics_fetch(&request, envoy_status.as_ref()) + } FrameworkHttpRoute::Root => handle_root_fetch(&request, Some(&actor)), FrameworkHttpRoute::NotFound => handle_not_found_fetch(&request, Some(&actor)), } @@ -416,6 +424,7 @@ impl RegistryHttpRoute { match normalized_path { "/metadata" => Ok(Self::Framework(FrameworkHttpRoute::Metadata)), "/health" => Ok(Self::Framework(FrameworkHttpRoute::Health)), + "/metrics" => Ok(Self::Framework(FrameworkHttpRoute::Metrics)), "/" => Ok(Self::Framework(FrameworkHttpRoute::Root)), _ => Ok(Self::Framework(FrameworkHttpRoute::NotFound)), } @@ -427,6 +436,7 @@ pub(super) enum FrameworkHttpRoute { Queue(String), Metadata, Health, + Metrics, Root, NotFound, } @@ -466,6 +476,29 @@ fn handle_health_fetch(request: &Request, actor: Option<&ActorSpecifier>) -> Res text_response(StatusCode::OK, "ok") } +fn handle_metrics_fetch( + request: &Request, + envoy_status: Option<&CoreEnvoyStatus>, +) -> Result { + if request.method() != http::Method::GET { + return method_not_allowed_response(request, None); + } + + let metrics = crate::metrics_endpoint::render_prometheus_metrics(envoy_status)?; + bytes_response(StatusCode::OK, &metrics.content_type, metrics.body) +} + +fn bytes_response(status: StatusCode, content_type: &str, body: Vec) -> Result { + let mut headers = HashMap::new(); + headers.insert(http::header::CONTENT_TYPE.to_string(), content_type.to_owned()); + Ok(HttpResponse { + status: status.as_u16(), + headers, + body: Some(body), + body_stream: None, + }) +} + fn handle_root_fetch(request: &Request, actor: Option<&ActorSpecifier>) -> Result { if request.method() != http::Method::GET { return method_not_allowed_response(request, actor); diff --git a/rivetkit-rust/packages/rivetkit-core/src/registry/inspector.rs b/rivetkit-rust/packages/rivetkit-core/src/registry/inspector.rs index 56816c4aac..1c2929df00 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/registry/inspector.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/registry/inspector.rs @@ -1,7 +1,7 @@ use super::dispatch::*; use super::http::*; use super::*; -use crate::error::{client_error_message, ProtocolError}; +use crate::error::{ProtocolError, client_error_message}; use ::http; #[derive(rivet_error::RivetError, serde::Serialize)] @@ -346,9 +346,7 @@ impl RegistryDispatcher { ) -> Result<(bool, Option>)> { let result = instance .ctx - .internal_keep_awake(dispatch_workflow_history_through_task( - &instance.dispatch, - )) + .internal_keep_awake(dispatch_workflow_history_through_task(&instance.dispatch)) .await .context("load inspector workflow history"); diff --git a/rivetkit-rust/packages/rivetkit-core/src/registry/mod.rs b/rivetkit-rust/packages/rivetkit-core/src/registry/mod.rs index 3c98819220..6a103fe9cd 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/registry/mod.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/registry/mod.rs @@ -16,7 +16,7 @@ use rivet_envoy_client::config::{ WebSocketHandler, WebSocketMessage, WebSocketSender, }; use rivet_envoy_client::envoy::start_envoy; -use rivet_envoy_client::handle::EnvoyHandle; +use rivet_envoy_client::handle::{EnvoyHandle, EnvoyStatusHandle}; use rivet_envoy_client::protocol; use rivet_error::{ActorSpecifier, RivetError}; use rivetkit_client_protocol as client_protocol; @@ -81,6 +81,52 @@ pub struct CoreRegistry { factories: HashMap>, } +#[derive(Clone)] +pub struct CoreEnvoyHandle { + handle: EnvoyHandle, +} + +#[derive(Clone, Debug)] +pub struct CoreEnvoyStatus { + pub active_actor_count: usize, + pub ping_healthy: bool, + pub last_ping_at_ms: Option, + pub last_ping_age_ms: Option, +} + +impl CoreEnvoyStatus { + fn from_status_handle(handle: &EnvoyStatusHandle) -> Option { + Some(Self { + active_actor_count: handle.active_actor_count()?, + ping_healthy: handle.is_ping_healthy(), + last_ping_at_ms: handle.last_ping_at_ms(), + last_ping_age_ms: handle.last_ping_age_ms(), + }) + } +} + +impl CoreEnvoyHandle { + pub(crate) fn new(handle: EnvoyHandle) -> Self { + Self { handle } + } + + pub fn status(&self) -> CoreEnvoyStatus { + CoreEnvoyStatus { + active_actor_count: self.handle.active_actor_count(), + ping_healthy: self.handle.is_ping_healthy(), + last_ping_at_ms: self.handle.last_ping_at_ms(), + last_ping_age_ms: self.handle.last_ping_age_ms(), + } + } + + pub async fn actor_stop_threshold_ms(&self) -> Option { + self.handle + .get_protocol_metadata() + .await + .map(|metadata| metadata.actor_stop_threshold) + } +} + #[derive(Clone)] struct ActorTaskHandle { actor_id: String, @@ -130,6 +176,7 @@ pub(crate) struct RegistryDispatcher { actor_instances: SccHashMap, starting_instances: SccHashMap>, pending_stops: SccHashMap, + envoy_status_handle: Mutex>, region: String, handle_inspector_http_in_runtime: bool, } @@ -468,6 +515,7 @@ impl CoreRegistry { callbacks, }) .await; + dispatcher.set_envoy_status_handle(handle.status_handle()); // Do not install `tokio::signal::ctrl_c()` here. It calls // `sigaction(SIGINT, ...)` at the POSIX level, which overrides the @@ -516,11 +564,23 @@ impl RegistryDispatcher { actor_instances: SccHashMap::new(), starting_instances: SccHashMap::new(), pending_stops: SccHashMap::new(), + envoy_status_handle: Mutex::new(None), region: env::var("RIVET_REGION").unwrap_or_default(), handle_inspector_http_in_runtime, } } + pub(crate) fn set_envoy_status_handle(&self, handle: EnvoyStatusHandle) { + *self.envoy_status_handle.lock() = Some(handle); + } + + pub(crate) fn envoy_status(&self) -> Option { + self.envoy_status_handle + .lock() + .as_ref() + .and_then(CoreEnvoyStatus::from_status_handle) + } + pub(crate) fn build_actor_metadata_map(&self) -> HashMap { self.factories .iter() @@ -905,7 +965,7 @@ impl RegistryDispatcher { instance.ctx.mark_destroy_requested(); } - tracing::debug!( + tracing::info!( actor_id, handle_actor_id = %instance.actor_id, actor_name = %instance.actor_name, diff --git a/rivetkit-rust/packages/rivetkit-core/src/runtime_metrics.rs b/rivetkit-rust/packages/rivetkit-core/src/runtime_metrics.rs new file mode 100644 index 0000000000..796add1ac8 --- /dev/null +++ b/rivetkit-rust/packages/rivetkit-core/src/runtime_metrics.rs @@ -0,0 +1,175 @@ +use std::sync::LazyLock; + +use rivet_metrics::prometheus::{ + CounterVec, Gauge, GaugeVec, HistogramOpts, HistogramVec, IntGauge, IntGaugeVec, Opts, + Registry, +}; + +const QUANTILE_LABELS: &[&str] = &["quantile"]; +const CPU_MODE_LABELS: &[&str] = &["mode"]; +const HEAP_STATE_LABELS: &[&str] = &["state"]; +const GC_KIND_LABELS: &[&str] = &["kind"]; + +struct RuntimeMetricCollectors { + // Event loop quantile snapshot. Periodic gauge populated from + // `monitorEventLoopDelay()` percentiles every scrape interval. Not a real + // Prometheus histogram. + eventloop_lag_seconds: GaugeVec, + eventloop_utilization: Gauge, + // Last-heartbeat epoch (ms). JS-side `setInterval(100ms)` updates this; the + // scraping dashboard computes `now - this` to get heartbeat age. + eventloop_heartbeat_ts_ms: IntGauge, + process_cpu_seconds_total: CounterVec, + process_resident_memory_bytes: IntGauge, + heap_bytes: IntGaugeVec, + gc_duration_seconds: HistogramVec, + active_handles: IntGauge, + active_requests: IntGauge, +} + +static METRICS: LazyLock = LazyLock::new(RuntimeMetricCollectors::new); + +impl RuntimeMetricCollectors { + fn new() -> Self { + let eventloop_lag_seconds = GaugeVec::new( + Opts::new( + "rivetkit_js_eventloop_lag_seconds", + "event loop delay quantile snapshot (seconds) from monitorEventLoopDelay; periodic gauge, not a true histogram", + ), + QUANTILE_LABELS, + ) + .expect("create js_eventloop_lag_seconds gauge"); + let eventloop_utilization = Gauge::new( + "rivetkit_js_eventloop_utilization", + "event loop utilization fraction (0.0..1.0) from performance.eventLoopUtilization delta", + ) + .expect("create js_eventloop_utilization gauge"); + let eventloop_heartbeat_ts_ms = IntGauge::new( + "rivetkit_js_eventloop_heartbeat_ts_ms", + "epoch millisecond timestamp of last JS-side event loop heartbeat; dashboards compute now - this for age", + ) + .expect("create js_eventloop_heartbeat_ts_ms gauge"); + let process_cpu_seconds_total = CounterVec::new( + Opts::new( + "rivetkit_js_process_cpu_seconds_total", + "total CPU time consumed by the Node.js process in seconds", + ), + CPU_MODE_LABELS, + ) + .expect("create js_process_cpu_seconds_total counter"); + let process_resident_memory_bytes = IntGauge::new( + "rivetkit_js_process_resident_memory_bytes", + "Node.js process resident set size in bytes", + ) + .expect("create js_process_resident_memory_bytes gauge"); + let heap_bytes = IntGaugeVec::new( + Opts::new( + "rivetkit_js_heap_bytes", + "V8 heap size in bytes by state (used|total|limit)", + ), + HEAP_STATE_LABELS, + ) + .expect("create js_heap_bytes gauge"); + let gc_duration_seconds = HistogramVec::new( + HistogramOpts::new( + "rivetkit_js_gc_duration_seconds", + "V8 garbage collection pause duration in seconds by kind", + ) + .buckets(rivet_metrics::MICRO_BUCKETS.to_vec()), + GC_KIND_LABELS, + ) + .expect("create js_gc_duration_seconds histogram"); + let active_handles = IntGauge::new( + "rivetkit_js_active_handles", + "number of active libuv handles from process._getActiveHandles()", + ) + .expect("create js_active_handles gauge"); + let active_requests = IntGauge::new( + "rivetkit_js_active_requests", + "number of active libuv requests from process._getActiveRequests()", + ) + .expect("create js_active_requests gauge"); + + register_metric(&rivet_metrics::REGISTRY, eventloop_lag_seconds.clone()); + register_metric(&rivet_metrics::REGISTRY, eventloop_utilization.clone()); + register_metric(&rivet_metrics::REGISTRY, eventloop_heartbeat_ts_ms.clone()); + register_metric(&rivet_metrics::REGISTRY, process_cpu_seconds_total.clone()); + register_metric( + &rivet_metrics::REGISTRY, + process_resident_memory_bytes.clone(), + ); + register_metric(&rivet_metrics::REGISTRY, heap_bytes.clone()); + register_metric(&rivet_metrics::REGISTRY, gc_duration_seconds.clone()); + register_metric(&rivet_metrics::REGISTRY, active_handles.clone()); + register_metric(&rivet_metrics::REGISTRY, active_requests.clone()); + + Self { + eventloop_lag_seconds, + eventloop_utilization, + eventloop_heartbeat_ts_ms, + process_cpu_seconds_total, + process_resident_memory_bytes, + heap_bytes, + gc_duration_seconds, + active_handles, + active_requests, + } + } +} + +pub fn set_eventloop_lag_quantile(quantile: &str, seconds: f64) { + METRICS + .eventloop_lag_seconds + .with_label_values(&[quantile]) + .set(seconds); +} + +pub fn set_eventloop_utilization(value: f64) { + METRICS.eventloop_utilization.set(value); +} + +pub fn set_eventloop_heartbeat_ts_ms(epoch_ms: i64) { + METRICS.eventloop_heartbeat_ts_ms.set(epoch_ms); +} + +pub fn add_process_cpu_seconds(mode: &str, seconds: f64) { + METRICS + .process_cpu_seconds_total + .with_label_values(&[mode]) + .inc_by(seconds); +} + +pub fn set_process_resident_memory_bytes(bytes: i64) { + METRICS.process_resident_memory_bytes.set(bytes); +} + +pub fn set_heap_bytes(state: &str, bytes: i64) { + METRICS.heap_bytes.with_label_values(&[state]).set(bytes); +} + +pub fn observe_gc_duration(kind: &str, seconds: f64) { + METRICS + .gc_duration_seconds + .with_label_values(&[kind]) + .observe(seconds); +} + +pub fn set_active_handles(count: i64) { + METRICS.active_handles.set(count); +} + +pub fn set_active_requests(count: i64) { + METRICS.active_requests.set(count); +} + +fn register_metric(registry: &Registry, metric: M) +where + M: rivet_metrics::prometheus::core::Collector + Clone + Send + Sync + 'static, +{ + if let Err(error) = registry.register(Box::new(metric)) { + tracing::warn!( + ?error, + "runtime metric registration failed, using existing collector" + ); + } +} diff --git a/rivetkit-rust/packages/rivetkit-core/src/serverless.rs b/rivetkit-rust/packages/rivetkit-core/src/serverless.rs index 284fc6c166..9c2c707a61 100644 --- a/rivetkit-rust/packages/rivetkit-core/src/serverless.rs +++ b/rivetkit-rust/packages/rivetkit-core/src/serverless.rs @@ -21,7 +21,9 @@ use url::Url; use crate::actor::factory::ActorFactory; #[cfg(feature = "native-runtime")] use crate::engine_process::EngineProcessManager; -use crate::registry::{RegistryCallbacks, RegistryDispatcher, ServeConfig}; +use crate::registry::{ + CoreEnvoyHandle, CoreEnvoyStatus, RegistryCallbacks, RegistryDispatcher, ServeConfig, +}; use crate::runtime::RuntimeSpawner; use crate::time::{sleep, timeout}; @@ -224,6 +226,16 @@ impl CoreServerlessRuntime { .map(EnvoyHandle::active_actor_count) } + pub async fn active_envoy_actor_stop_threshold_ms(&self) -> Option { + let handle = self.envoy.lock().await.as_ref().cloned()?; + CoreEnvoyHandle::new(handle).actor_stop_threshold_ms().await + } + + pub async fn active_envoy_status(&self) -> Option { + let handle = self.envoy.lock().await.as_ref().cloned()?; + Some(CoreEnvoyHandle::new(handle).status()) + } + pub async fn handle_request(&self, req: ServerlessRequest) -> ServerlessResponse { let cors = cors_headers(&req); match self.handle_request_inner(req).await { @@ -279,6 +291,10 @@ impl CoreServerlessRuntime { } } ("GET", "/metadata") => Ok(self.metadata_response()), + ("GET", "/metrics") => { + let envoy_status = self.active_envoy_status().await; + Ok(metrics_response(&req.headers, envoy_status.as_ref())) + } ("GET", "/start") | ("POST", "/start") => self.start_response(req).await, ("OPTIONS", _) => Ok(bytes_response( StatusCode::NO_CONTENT, @@ -625,6 +641,33 @@ fn json_response(status: StatusCode, body: serde_json::Value) -> ServerlessRespo ) } +fn metrics_response( + headers: &HashMap, + envoy_status: Option<&CoreEnvoyStatus>, +) -> ServerlessResponse { + let bearer_token = crate::metrics_endpoint::authorization_bearer_token_map(headers); + match crate::metrics_endpoint::authorize_metrics_request(bearer_token) { + Ok(()) => match crate::metrics_endpoint::render_prometheus_metrics(envoy_status) { + Ok(metrics) => bytes_response( + StatusCode::OK, + HashMap::from([("content-type".to_owned(), metrics.content_type)]), + metrics.body, + ), + Err(error) => error_response(error), + }, + Err(crate::metrics_endpoint::MetricsAccessError::NotEnabled) => text_response( + StatusCode::FORBIDDEN, + "text/plain; charset=utf-8", + "metrics not enabled\n", + ), + Err(crate::metrics_endpoint::MetricsAccessError::Unauthorized) => text_response( + StatusCode::UNAUTHORIZED, + "text/plain; charset=utf-8", + "metrics request requires a valid bearer token\n", + ), + } +} + fn bytes_response( status: StatusCode, headers: HashMap, diff --git a/rivetkit-rust/packages/rivetkit-core/tests/integration.rs b/rivetkit-rust/packages/rivetkit-core/tests/integration.rs index 1fa129c13a..b4e7c6e7b5 100644 --- a/rivetkit-rust/packages/rivetkit-core/tests/integration.rs +++ b/rivetkit-rust/packages/rivetkit-core/tests/integration.rs @@ -4,5 +4,8 @@ mod common; #[path = "integration/counter.rs"] mod counter; +#[path = "integration/sqlite_corruption_fuzz.rs"] +mod sqlite_corruption_fuzz; + #[path = "migration/v2_2_1/mod.rs"] mod migration_v2_2_1; diff --git a/rivetkit-rust/packages/rivetkit-core/tests/integration/common/ctx.rs b/rivetkit-rust/packages/rivetkit-core/tests/integration/common/ctx.rs index d1ee97f2f0..c7ce56558b 100644 --- a/rivetkit-rust/packages/rivetkit-core/tests/integration/common/ctx.rs +++ b/rivetkit-rust/packages/rivetkit-core/tests/integration/common/ctx.rs @@ -22,6 +22,12 @@ pub struct IntegrationCtx { endpoint: String, child: Child, client: reqwest::Client, + binary_path: PathBuf, + db_path: PathBuf, + database: EngineDatabase, + guard_port: u16, + api_peer_port: u16, + metrics_port: u16, stdout_path: PathBuf, stderr_path: PathBuf, } @@ -168,6 +174,10 @@ impl IntegrationCtx { } pub async fn create_actor(&self, name: &str) -> Result { + self.create_actor_with_key(name, None).await + } + + pub async fn create_actor_with_key(&self, name: &str, key: Option<&str>) -> Result { let response = self .client .post(format!("{}/actors", self.endpoint)) @@ -176,7 +186,7 @@ impl IntegrationCtx { .json(&serde_json::json!({ "name": name, "runner_name_selector": DEFAULT_POOL, - "key": null, + "key": key, "input": null, "datacenter": null, "crash_policy": "destroy", @@ -198,6 +208,51 @@ impl IntegrationCtx { Ok(response.actor) } + pub async fn create_or_get_actor_with_key(&self, name: &str, key: &str) -> Result { + let response = self + .client + .post(format!("{}/actors", self.endpoint)) + .query(&[("namespace", DEFAULT_NAMESPACE)]) + .bearer_auth(TOKEN) + .json(&serde_json::json!({ + "name": name, + "runner_name_selector": DEFAULT_POOL, + "key": key, + "input": null, + "datacenter": null, + "crash_policy": "destroy", + })) + .send() + .await + .context("create or get actor")?; + let status = response.status(); + let body = response + .text() + .await + .context("read create or get actor response")?; + if status.is_success() { + let response: CreateActorResponse = + serde_json::from_str(&body).context("decode create actor response")?; + return Ok(response.actor); + } + + let value: serde_json::Value = + serde_json::from_str(&body).context("decode create actor error response")?; + if value.get("code").and_then(serde_json::Value::as_str) == Some("duplicate_key") { + if let Some(actor_id) = value + .get("metadata") + .and_then(|metadata| metadata.get("existing_actor_id")) + .and_then(serde_json::Value::as_str) + { + return Ok(ApiActor { + actor_id: actor_id.to_owned(), + }); + } + } + + bail!("create or get actor failed with {status}: {body}"); + } + async fn envoy_ready(&self) -> Result { let envoys_response = self .client @@ -322,6 +377,10 @@ impl IntegrationCtx { &self.endpoint } + pub fn client(&self) -> reqwest::Client { + self.client.clone() + } + pub fn engine_stdout_tail(&self) -> String { tail_file(&self.stdout_path) } @@ -330,6 +389,29 @@ impl IntegrationCtx { tail_file(&self.stderr_path) } + pub async fn restart_engine(&mut self) -> Result<()> { + shutdown_child(&mut self.child).await; + self.child = spawn_engine_child( + &self.binary_path, + &self.db_path, + &self.database, + self.guard_port, + self.api_peer_port, + self.metrics_port, + &self.stdout_path, + &self.stderr_path, + ) + .await?; + wait_for_engine_health( + &self.client, + &self.endpoint, + &mut self.child, + &self.stderr_path, + ) + .await?; + Ok(()) + } + pub async fn shutdown(mut self) -> Result<()> { shutdown_child(&mut self.child).await; Ok(()) @@ -369,25 +451,20 @@ impl IntegrationCtxBuilder { let metrics_port = pick_port("metrics")?; let endpoint = format!("http://127.0.0.1:{guard_port}"); let binary_path = engine_binary_path()?; + let database = EngineDatabase::from_env(); let stdout_path = temp_dir.path().join("engine.stdout.log"); let stderr_path = temp_dir.path().join("engine.stderr.log"); - let stdout = File::create(&stdout_path).context("create engine stdout log")?; - let stderr = File::create(&stderr_path).context("create engine stderr log")?; - - let mut child = Command::new(&binary_path) - .arg("start") - .env("RIVET__GUARD__HOST", "127.0.0.1") - .env("RIVET__GUARD__PORT", guard_port.to_string()) - .env("RIVET__API_PEER__HOST", "127.0.0.1") - .env("RIVET__API_PEER__PORT", api_peer_port.to_string()) - .env("RIVET__METRICS__HOST", "127.0.0.1") - .env("RIVET__METRICS__PORT", metrics_port.to_string()) - .env("RIVET__FILE_SYSTEM__PATH", &db_path) - .stdin(Stdio::null()) - .stdout(Stdio::from(stdout)) - .stderr(Stdio::from(stderr)) - .spawn() - .with_context(|| format!("spawn engine binary `{}`", binary_path.display()))?; + let mut child = spawn_engine_child( + &binary_path, + &db_path, + &database, + guard_port, + api_peer_port, + metrics_port, + &stdout_path, + &stderr_path, + ) + .await?; let client = reqwest::Client::new(); wait_for_engine_health(&client, &endpoint, &mut child, &stderr_path).await?; @@ -397,6 +474,12 @@ impl IntegrationCtxBuilder { endpoint, child, client, + binary_path, + db_path, + database, + guard_port, + api_peer_port, + metrics_port, stdout_path, stderr_path, }) @@ -448,6 +531,96 @@ fn pick_port(label: &str) -> Result { portpicker::pick_unused_port().with_context(|| format!("pick {label} port")) } +#[derive(Clone, Debug)] +enum EngineDatabase { + FileSystem, + FoundationDb { + cluster_description: String, + cluster_id: String, + addresses: String, + }, +} + +impl EngineDatabase { + fn from_env() -> Self { + match std::env::var("SQLITE_CORRUPTION_FUZZ_ENGINE_DATABASE") + .ok() + .as_deref() + { + Some("foundationdb") => Self::FoundationDb { + cluster_description: std::env::var( + "SQLITE_CORRUPTION_FUZZ_FOUNDATIONDB_CLUSTER_DESCRIPTION", + ) + .unwrap_or_else(|_| "docker".to_owned()), + cluster_id: std::env::var("SQLITE_CORRUPTION_FUZZ_FOUNDATIONDB_CLUSTER_ID") + .unwrap_or_else(|_| "docker".to_owned()), + addresses: std::env::var("SQLITE_CORRUPTION_FUZZ_FOUNDATIONDB_ADDRESSES") + .unwrap_or_else(|_| "127.0.0.1:4500".to_owned()), + }, + _ => Self::FileSystem, + } + } +} + +async fn spawn_engine_child( + binary_path: &Path, + db_path: &Path, + database: &EngineDatabase, + guard_port: u16, + api_peer_port: u16, + metrics_port: u16, + stdout_path: &Path, + stderr_path: &Path, +) -> Result { + let stdout = File::options() + .create(true) + .append(true) + .open(stdout_path) + .context("open engine stdout log")?; + let stderr = File::options() + .create(true) + .append(true) + .open(stderr_path) + .context("open engine stderr log")?; + + let mut command = Command::new(binary_path); + command + .arg("start") + .env("RIVET__GUARD__HOST", "127.0.0.1") + .env("RIVET__GUARD__PORT", guard_port.to_string()) + .env("RIVET__API_PEER__HOST", "127.0.0.1") + .env("RIVET__API_PEER__PORT", api_peer_port.to_string()) + .env("RIVET__METRICS__HOST", "127.0.0.1") + .env("RIVET__METRICS__PORT", metrics_port.to_string()) + .env("RIVET__TELEMETRY__ENABLED", "false") + .stdin(Stdio::null()) + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)); + + match database { + EngineDatabase::FileSystem => { + command.env("RIVET__FILE_SYSTEM__PATH", db_path); + } + EngineDatabase::FoundationDb { + cluster_description, + cluster_id, + addresses, + } => { + command + .env( + "RIVET__FOUNDATIONDB__CLUSTER_DESCRIPTION", + cluster_description, + ) + .env("RIVET__FOUNDATIONDB__CLUSTER_ID", cluster_id) + .env("RIVET__FOUNDATIONDB__ADDRESSES", addresses); + } + } + + command + .spawn() + .with_context(|| format!("spawn engine binary `{}`", binary_path.display())) +} + async fn wait_for_engine_health( client: &reqwest::Client, endpoint: &str, diff --git a/rivetkit-rust/packages/rivetkit-core/tests/integration/sqlite_corruption_fuzz.rs b/rivetkit-rust/packages/rivetkit-core/tests/integration/sqlite_corruption_fuzz.rs new file mode 100644 index 0000000000..cf53fac376 --- /dev/null +++ b/rivetkit-rust/packages/rivetkit-core/tests/integration/sqlite_corruption_fuzz.rs @@ -0,0 +1,1035 @@ +use std::collections::{HashMap, HashSet}; +use std::env; +use std::io::Cursor; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, bail}; +use futures::future::try_join_all; +use rivetkit_core::{ + ActorConfig, ActorEvent, ActorFactory, CoreRegistry, RequestSaveOpts, SerializeStateReason, + StateDelta, +}; +use serde_json::{Value as JsonValue, json}; + +use crate::common::ctx::IntegrationCtx; + +const ACTOR_NAME: &str = "sqlite-fuzz"; +const DEFAULT_ACTOR_COUNT: usize = 6; +const DEFAULT_STEPS_PER_ACTOR: usize = 80; +const DEFAULT_RESTART_EVERY_ROUNDS: usize = 20; +const DEFAULT_ACTION_TIMEOUT_SECS: usize = 20; +const DEFAULT_MID_ROUND_RESTART_EVERY_ROUNDS: usize = 0; +const DEFAULT_MID_ROUND_RESTART_DELAY_MS: usize = 75; +const DEFAULT_SAVE_EVERY_STEPS: usize = 1; +const DEFAULT_ENGINE_RESTART_EVERY_ROUNDS: usize = 0; +const DEFAULT_MID_ROUND_ENGINE_RESTART_EVERY_ROUNDS: usize = 0; +const DEFAULT_MID_ROUND_ENGINE_RESTART_DELAY_MS: usize = 75; +const DEFAULT_SUSPECT_PROBE_ROUNDS: usize = 3; +const DEFAULT_PAYLOAD_MULTIPLIER: usize = 1; +const DEFAULT_FINAL_CHECK_TIMEOUT_SECS: usize = 30; +const DEFAULT_FINAL_CHECK_ATTEMPTS: usize = 2; + +struct FuzzActor { + actor_id: String, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "diagnostic fuzz harness. Run manually with RIVET_ENGINE_BINARY_PATH pointing at the target engine."] +async fn sqlite_lifecycle_fuzz_real_engine() -> Result<()> { + let mut ctx = IntegrationCtx::builder().start().await?; + ctx.create_default_namespace().await?; + let actor_count = env_usize("SQLITE_CORRUPTION_FUZZ_ACTORS", DEFAULT_ACTOR_COUNT); + let steps_per_actor = env_usize("SQLITE_CORRUPTION_FUZZ_STEPS", DEFAULT_STEPS_PER_ACTOR); + let restart_every_rounds = env_usize( + "SQLITE_CORRUPTION_FUZZ_RESTART_EVERY_ROUNDS", + DEFAULT_RESTART_EVERY_ROUNDS, + ); + let action_timeout = Duration::from_secs(env_usize( + "SQLITE_CORRUPTION_FUZZ_ACTION_TIMEOUT_SECS", + DEFAULT_ACTION_TIMEOUT_SECS, + ) as u64); + let final_check_timeout = Duration::from_secs(env_usize( + "SQLITE_CORRUPTION_FUZZ_FINAL_CHECK_TIMEOUT_SECS", + DEFAULT_FINAL_CHECK_TIMEOUT_SECS, + ) as u64); + let final_check_attempts = env_usize( + "SQLITE_CORRUPTION_FUZZ_FINAL_CHECK_ATTEMPTS", + DEFAULT_FINAL_CHECK_ATTEMPTS, + ) + .max(1); + let mid_round_restart_every_rounds = env_usize( + "SQLITE_CORRUPTION_FUZZ_MID_ROUND_RESTART_EVERY_ROUNDS", + DEFAULT_MID_ROUND_RESTART_EVERY_ROUNDS, + ); + let mid_round_restart_delay = Duration::from_millis(env_usize( + "SQLITE_CORRUPTION_FUZZ_MID_ROUND_RESTART_DELAY_MS", + DEFAULT_MID_ROUND_RESTART_DELAY_MS, + ) as u64); + let engine_restart_every_rounds = env_usize( + "SQLITE_CORRUPTION_FUZZ_ENGINE_RESTART_EVERY_ROUNDS", + DEFAULT_ENGINE_RESTART_EVERY_ROUNDS, + ); + let mid_round_engine_restart_every_rounds = env_usize( + "SQLITE_CORRUPTION_FUZZ_MID_ROUND_ENGINE_RESTART_EVERY_ROUNDS", + DEFAULT_MID_ROUND_ENGINE_RESTART_EVERY_ROUNDS, + ); + let mid_round_engine_restart_delay = Duration::from_millis(env_usize( + "SQLITE_CORRUPTION_FUZZ_MID_ROUND_ENGINE_RESTART_DELAY_MS", + DEFAULT_MID_ROUND_ENGINE_RESTART_DELAY_MS, + ) as u64); + let suspect_probe_rounds = env_usize( + "SQLITE_CORRUPTION_FUZZ_SUSPECT_PROBE_ROUNDS", + DEFAULT_SUSPECT_PROBE_ROUNDS, + ); + let run_id = fuzz_run_id(); + tracing::info!(run_id, "starting sqlite corruption fuzz run"); + let mut registry_task = ctx.serve_registry(sqlite_fuzz_registry()); + + ctx.wait_for_envoy_ready().await?; + let mut actors = Vec::new(); + let mut all_actor_ids = HashSet::new(); + let mut all_actor_keys = HashSet::new(); + for index in 0..actor_count { + let key = format!("sqlite-fuzz-{run_id}-{index}"); + let actor = ctx + .create_actor_with_key(ACTOR_NAME, Some(&key)) + .await + .with_context(|| format!("create fuzz actor {key}"))?; + all_actor_ids.insert(actor.actor_id.clone()); + all_actor_keys.insert(key.clone()); + actors.push(FuzzActor { + actor_id: actor.actor_id, + }); + } + + let mut replacement_index = 0; + let mut suspect_unavailable_counts = HashMap::new(); + for round in 0..steps_per_actor { + let mut targets = Vec::with_capacity(actors.len() + actors.len() / 2); + targets.extend(actors.iter().map(|actor| actor.actor_id.clone())); + if round % 7 == 0 { + targets.extend(actors.iter().step_by(2).map(|actor| actor.actor_id.clone())); + } + + let client = ctx.client(); + let endpoint = ctx.endpoint().to_owned(); + let action_futures = targets.into_iter().map(|actor_id| { + let client = client.clone(); + let endpoint = endpoint.clone(); + async move { run_actor_step_direct(client, endpoint, actor_id, action_timeout).await } + }); + let attempts_future = try_join_all(action_futures); + let should_restart_mid_engine = mid_round_engine_restart_every_rounds != 0 + && (round + 1) % mid_round_engine_restart_every_rounds == 0 + && round + 1 < steps_per_actor; + let should_restart_mid_round = mid_round_restart_every_rounds != 0 + && (round + 1) % mid_round_restart_every_rounds == 0 + && round + 1 < steps_per_actor; + let mut restarted_mid_round = false; + let attempts = if should_restart_mid_engine { + tokio::pin!(attempts_future); + tokio::select! { + attempts = &mut attempts_future => attempts?, + _ = tokio::time::sleep(mid_round_engine_restart_delay) => { + restarted_mid_round = true; + tracing::warn!( + round, + delay_ms = mid_round_engine_restart_delay.as_millis(), + "restarting sqlite fuzz engine child while actions are in flight" + ); + registry_task.shutdown().await?; + ctx.restart_engine().await?; + ctx.create_default_namespace().await?; + registry_task = ctx.serve_registry(sqlite_fuzz_registry()); + ctx.wait_for_envoy_ready().await?; + attempts_future.await? + } + } + } else if should_restart_mid_round { + tokio::pin!(attempts_future); + tokio::select! { + attempts = &mut attempts_future => attempts?, + _ = tokio::time::sleep(mid_round_restart_delay) => { + restarted_mid_round = true; + tracing::warn!( + round, + delay_ms = mid_round_restart_delay.as_millis(), + "restarting sqlite fuzz registry while actions are in flight" + ); + registry_task.shutdown().await?; + tokio::time::sleep(Duration::from_millis(150)).await; + registry_task = ctx.serve_registry(sqlite_fuzz_registry()); + ctx.wait_for_envoy_ready().await?; + attempts_future.await? + } + } + } else { + attempts_future.await? + }; + let unavailable = attempts + .into_iter() + .filter_map(|attempt| match attempt { + ActionAttempt::Ok => None, + ActionAttempt::Unavailable { actor_id, reason } => Some((actor_id, reason)), + }) + .collect::>(); + let unavailable_actor_ids = unavailable + .iter() + .map(|(actor_id, _)| actor_id.clone()) + .collect::>(); + if !unavailable_actor_ids.is_empty() { + tracing::warn!( + ?unavailable, + "sqlite fuzz actor unavailable after lifecycle churn" + ); + let mut replacement_actor_ids = HashSet::new(); + for actor_id in unavailable.iter().map(|(actor_id, _)| actor_id) { + let client = ctx.client(); + let endpoint = ctx.endpoint().to_owned(); + match run_actor_check_direct(client, endpoint, actor_id.clone(), action_timeout) + .await? + { + ActionAttempt::Ok => { + suspect_unavailable_counts.remove(actor_id); + } + ActionAttempt::Unavailable { .. } => { + let unavailable_count = suspect_unavailable_counts + .entry(actor_id.clone()) + .or_insert(0); + *unavailable_count += 1; + if *unavailable_count >= suspect_probe_rounds { + replacement_actor_ids.insert(actor_id.clone()); + } else { + tracing::warn!( + actor_id, + unavailable_count = *unavailable_count, + suspect_probe_rounds, + "retaining sqlite fuzz suspect actor for later recheck" + ); + } + } + } + } + for actor_id in &replacement_actor_ids { + suspect_unavailable_counts.remove(actor_id); + } + actors.retain(|actor| !replacement_actor_ids.contains(&actor.actor_id)); + for _ in 0..replacement_actor_ids.len() { + let key = format!("sqlite-fuzz-{run_id}-replacement-{round}-{replacement_index}"); + replacement_index += 1; + let actor = ctx + .create_actor_with_key(ACTOR_NAME, Some(&key)) + .await + .with_context(|| format!("create replacement fuzz actor {key}"))?; + all_actor_ids.insert(actor.actor_id.clone()); + all_actor_keys.insert(key.clone()); + actors.push(FuzzActor { + actor_id: actor.actor_id, + }); + } + } + + if restart_every_rounds != 0 + && (round + 1) % restart_every_rounds == 0 + && round + 1 < steps_per_actor + && !restarted_mid_round + && !(engine_restart_every_rounds != 0 && (round + 1) % engine_restart_every_rounds == 0) + { + registry_task.shutdown().await?; + tokio::time::sleep(Duration::from_millis(150)).await; + registry_task = ctx.serve_registry(sqlite_fuzz_registry()); + ctx.wait_for_envoy_ready().await?; + } + + if engine_restart_every_rounds != 0 + && (round + 1) % engine_restart_every_rounds == 0 + && round + 1 < steps_per_actor + { + tracing::warn!(round, "restarting sqlite fuzz engine child between rounds"); + registry_task.shutdown().await?; + ctx.restart_engine().await?; + ctx.create_default_namespace().await?; + registry_task = ctx.serve_registry(sqlite_fuzz_registry()); + ctx.wait_for_envoy_ready().await?; + } + } + + registry_task.shutdown().await?; + ctx.restart_engine().await?; + ctx.create_default_namespace().await?; + registry_task = ctx.serve_registry(sqlite_fuzz_registry()); + ctx.wait_for_envoy_ready().await?; + + for actor_id in all_actor_ids { + let client = ctx.client(); + let endpoint = ctx.endpoint().to_owned(); + match run_actor_check_with_retries( + client, + endpoint, + actor_id.clone(), + final_check_timeout, + final_check_attempts, + ) + .await? + { + ActionAttempt::Ok => {} + ActionAttempt::Unavailable { reason, .. } => { + tracing::warn!( + actor_id, + reason, + "sqlite fuzz final actor check unavailable" + ); + } + } + } + + for key in all_actor_keys { + let actor = match create_or_get_actor_with_retries(&ctx, &key, final_check_attempts).await { + Ok(actor) => actor, + Err(err) => { + tracing::warn!( + key, + error = format!("{err:#}"), + "sqlite fuzz final actor key resolve failed" + ); + continue; + } + }; + let client = ctx.client(); + let endpoint = ctx.endpoint().to_owned(); + match run_actor_check_with_retries( + client, + endpoint, + actor.actor_id.clone(), + final_check_timeout, + final_check_attempts, + ) + .await? + { + ActionAttempt::Ok => {} + ActionAttempt::Unavailable { reason, .. } => { + tracing::warn!( + key, + actor_id = actor.actor_id, + reason, + "sqlite fuzz final actor key check unavailable" + ); + } + } + } + + registry_task.shutdown().await?; + ctx.shutdown().await?; + + Ok(()) +} + +enum ActionAttempt { + Ok, + Unavailable { actor_id: String, reason: String }, +} + +async fn run_actor_step_direct( + client: reqwest::Client, + endpoint: String, + actor_id: String, + action_timeout: Duration, +) -> Result { + run_actor_action_direct(client, endpoint, actor_id, "step", action_timeout).await +} + +async fn run_actor_check_direct( + client: reqwest::Client, + endpoint: String, + actor_id: String, + action_timeout: Duration, +) -> Result { + run_actor_action_direct(client, endpoint, actor_id, "check", action_timeout).await +} + +async fn run_actor_check_with_retries( + client: reqwest::Client, + endpoint: String, + actor_id: String, + action_timeout: Duration, + attempts: usize, +) -> Result { + let mut last_unavailable = None; + for attempt in 0..attempts { + match run_actor_check_direct( + client.clone(), + endpoint.clone(), + actor_id.clone(), + action_timeout, + ) + .await? + { + ActionAttempt::Ok => return Ok(ActionAttempt::Ok), + ActionAttempt::Unavailable { reason, .. } => { + last_unavailable = Some(reason); + if attempt + 1 < attempts { + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + } + + Ok(ActionAttempt::Unavailable { + actor_id, + reason: last_unavailable.unwrap_or_else(|| "final check unavailable".to_owned()), + }) +} + +async fn create_or_get_actor_with_retries( + ctx: &IntegrationCtx, + key: &str, + attempts: usize, +) -> Result { + let mut last_error = None; + for attempt in 0..attempts { + match ctx.create_or_get_actor_with_key(ACTOR_NAME, key).await { + Ok(actor) => return Ok(actor), + Err(err) => { + last_error = Some(err); + if attempt + 1 < attempts { + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + } + + Err(last_error.expect("at least one create-or-get attempt should have run")) +} + +async fn run_actor_action_direct( + client: reqwest::Client, + endpoint: String, + actor_id: String, + action: &'static str, + action_timeout: Duration, +) -> Result { + let action_result = tokio::time::timeout( + action_timeout, + wait_for_json_action_direct(&client, &endpoint, &actor_id, action), + ) + .await; + let body = match action_result { + Ok(Ok(body)) => body, + Ok(Err(err)) if is_actor_unavailable(&err) => { + return Ok(ActionAttempt::Unavailable { + actor_id, + reason: format!("{err:#}"), + }); + } + Ok(Err(err)) => { + return Err(err) + .with_context(|| format!("run sqlite fuzz {action} for actor {actor_id}")); + } + Err(_) => { + return Ok(ActionAttempt::Unavailable { + actor_id, + reason: format!("action timed out after {action_timeout:?}"), + }); + } + }; + let output = action_output(&body)?; + assert_eq!( + output.get("integrity").and_then(JsonValue::as_str), + Some("ok") + ); + assert_eq!( + output.get("quick_check").and_then(JsonValue::as_str), + Some("ok") + ); + Ok(ActionAttempt::Ok) +} + +async fn wait_for_json_action_direct( + client: &reqwest::Client, + endpoint: &str, + actor_id: &str, + action: &str, +) -> Result { + let deadline = Instant::now() + Duration::from_secs(30); + let mut last_error = None; + while Instant::now() < deadline { + match send_json_action_direct(client, endpoint, actor_id, action).await { + Ok(body) => return Ok(body), + Err(err) => { + let message = format!("{err:#}"); + if !message.contains("actor_ready_timeout") + && !message.contains("Service Unavailable") + && !message.contains("error sending request") + && !message.contains("connection closed") + && !message.contains("Connection reset") + && !message.contains("dropped_reply") + && !message.contains("Actor is not ready") + { + return Err(err).context("actor action returned non-readiness error"); + } + last_error = Some(err); + tokio::time::sleep(Duration::from_millis(250)).await; + } + } + } + + match last_error { + Some(err) => Err(err).context("timed out waiting for actor action"), + None => bail!("timed out waiting for actor action"), + } +} + +async fn send_json_action_direct( + client: &reqwest::Client, + endpoint: &str, + actor_id: &str, + action: &str, +) -> Result { + let response = client + .post(format!("{endpoint}/gateway/{actor_id}/action/{action}")) + .header("x-rivet-encoding", "json") + .header("content-type", "application/json") + .body(r#"{"args":[]}"#) + .send() + .await + .context("send actor action")?; + let status = response.status(); + let body = response + .text() + .await + .context("read actor action response")?; + if !status.is_success() { + bail!("actor action failed with {status}: {body}"); + } + Ok(body) +} + +fn is_actor_unavailable(err: &anyhow::Error) -> bool { + let message = format!("{err:#}"); + message.contains("actor_ready_timeout") + || message.contains("Service Unavailable") + || message.contains("error sending request") + || message.contains("connection closed") + || message.contains("Connection reset") + || message.contains("dropped_reply") + || message.contains("Actor is not ready") +} + +fn sqlite_fuzz_registry() -> CoreRegistry { + let mut registry = CoreRegistry::new(); + registry.register(ACTOR_NAME, sqlite_fuzz_factory()); + registry +} + +fn sqlite_fuzz_factory() -> ActorFactory { + let config = ActorConfig { + has_database: true, + remote_sqlite: false, + sleep_timeout: Duration::from_millis(100), + sleep_grace_period: Duration::from_millis(500), + sleep_grace_period_overridden: true, + ..ActorConfig::default() + }; + let save_every_steps = env_usize( + "SQLITE_CORRUPTION_FUZZ_SAVE_EVERY_STEPS", + DEFAULT_SAVE_EVERY_STEPS, + ) as i64; + ActorFactory::new(config, move |start| { + Box::pin(async move { + let ctx = start.ctx; + let mut step = read_step(&ctx.state()); + let mut events = start.events; + while let Some(event) = events.recv().await { + match event { + ActorEvent::Action { + name, + args: _, + conn: _, + reply, + } => match name.as_str() { + "step" => { + step += 1; + let result = run_sqlite_step(&ctx, step).await; + if result.is_ok() + && save_every_steps != 0 && step % save_every_steps == 0 + { + ctx.request_save(RequestSaveOpts::default()); + } + reply.send(result.map(|summary| encode_json(&summary))); + } + "check" => { + reply.send( + run_sqlite_check(&ctx) + .await + .map(|summary| encode_json(&summary)), + ); + } + name => { + reply.send(Err(anyhow::anyhow!("unknown action `{name}`"))); + } + }, + ActorEvent::SerializeState { reason, reply } => match reason { + SerializeStateReason::Save | SerializeStateReason::Inspector => { + reply.send(Ok(vec![StateDelta::ActorState(encode_json(&json!({ + "step": step, + })))])); + } + }, + ActorEvent::RunGracefulCleanup { reason: _, reply } => { + reply.send(Ok(())); + } + ActorEvent::HttpRequest { request: _, reply } => { + reply.send(Err(anyhow::anyhow!("http requests are not handled"))); + } + ActorEvent::QueueSend { + name: _, + body: _, + conn: _, + request: _, + wait: _, + timeout_ms: _, + reply, + } => { + reply.send(Err(anyhow::anyhow!("queue sends are not handled"))); + } + ActorEvent::WebSocketOpen { + ws: _, + conn: _, + request: _, + reply, + } => { + reply.send(Err(anyhow::anyhow!("websockets are not handled"))); + } + ActorEvent::ConnectionPreflight { + conn: _, + params: _, + request: _, + reply, + } => { + reply.send(Ok(())); + } + ActorEvent::ConnectionOpen { reply, .. } => { + reply.send(Ok(())); + } + ActorEvent::ConnectionClosed { conn: _ } => {} + ActorEvent::SubscribeRequest { + conn: _, + event_name: _, + reply, + } => { + reply.send(Err(anyhow::anyhow!("subscriptions are not handled"))); + } + ActorEvent::DisconnectConn { conn_id: _, reply } => { + reply.send(Ok(())); + } + ActorEvent::WorkflowHistoryRequested { reply } => { + reply.send(Ok(None)); + } + ActorEvent::WorkflowReplayRequested { entry_id: _, reply } => { + reply.send(Ok(None)); + } + } + } + + Ok(()) + }) + }) +} + +async fn run_sqlite_step(ctx: &rivetkit_core::ActorContext, step: i64) -> Result { + let plan = StepPlan::new(ctx.actor_id(), step); + ensure_schema(ctx).await?; + + if plan.schema_churn { + ctx.db_run("DROP INDEX IF EXISTS idx_fuzz_rows_note", None) + .await?; + ctx.db_run( + "CREATE INDEX IF NOT EXISTS idx_fuzz_rows_note + ON fuzz_rows(note, id)", + None, + ) + .await?; + } + + ctx.db_run("BEGIN IMMEDIATE", None).await?; + let transaction_result = run_write_transaction(ctx, step, &plan).await; + if transaction_result.is_err() { + let _ = ctx.db_run("ROLLBACK", None).await; + } + transaction_result?; + ctx.db_run("COMMIT", None).await?; + + if plan.reindex { + ctx.db_run("REINDEX", None).await?; + } + if plan.analyze { + ctx.db_run("ANALYZE", None).await?; + } + if plan.vacuum { + ctx.db_run("VACUUM", None).await?; + } + + run_read_checks(ctx, step, &plan).await?; + + let integrity = assert_pragma_ok(ctx, "PRAGMA integrity_check", "integrity_check").await?; + let quick_check = assert_pragma_ok(ctx, "PRAGMA quick_check", "quick_check").await?; + if plan.close_after { + ctx.sql().close().await?; + } + if plan.sleep_after { + ctx.sleep()?; + } + + Ok(json!({ + "step": step, + "op": plan.op, + "row_id": plan.row_id, + "payload_size": plan.payload_size, + "integrity": integrity, + "quick_check": quick_check, + "sleep_after": plan.sleep_after, + })) +} + +async fn run_sqlite_check(ctx: &rivetkit_core::ActorContext) -> Result { + let integrity = assert_pragma_ok(ctx, "PRAGMA integrity_check", "integrity_check").await?; + let quick_check = assert_pragma_ok(ctx, "PRAGMA quick_check", "quick_check").await?; + ctx.sql().close().await?; + Ok(json!({ + "integrity": integrity, + "quick_check": quick_check, + })) +} + +async fn ensure_schema(ctx: &rivetkit_core::ActorContext) -> Result<()> { + ctx.db_run( + "CREATE TABLE IF NOT EXISTS fuzz_rows ( + id INTEGER PRIMARY KEY, + bucket INTEGER NOT NULL, + revision INTEGER NOT NULL, + payload BLOB NOT NULL, + note TEXT NOT NULL + )", + None, + ) + .await?; + ctx.db_run( + "CREATE TABLE IF NOT EXISTS fuzz_log ( + step INTEGER PRIMARY KEY, + op INTEGER NOT NULL, + row_id INTEGER NOT NULL, + payload_size INTEGER NOT NULL + )", + None, + ) + .await?; + ctx.db_run( + "CREATE INDEX IF NOT EXISTS idx_fuzz_rows_bucket_payload + ON fuzz_rows(bucket, note)", + None, + ) + .await?; + ctx.db_run( + "CREATE INDEX IF NOT EXISTS idx_fuzz_rows_revision + ON fuzz_rows(revision)", + None, + ) + .await?; + Ok(()) +} + +async fn run_write_transaction( + ctx: &rivetkit_core::ActorContext, + step: i64, + plan: &StepPlan, +) -> Result<()> { + for offset in 0..plan.burst_rows { + let row_id = (plan.row_id + offset as i64 * 17) % 4099; + let bucket = (plan.bucket + offset as i64) % 31; + let payload_size = plan.payload_size + offset as i64 * 97; + ctx.db_run( + &format!( + "INSERT INTO fuzz_rows (id, bucket, revision, payload, note) + VALUES ({row_id}, {bucket}, {step}, zeroblob({payload_size}), 'note-{step}-{row_id}') + ON CONFLICT(id) DO UPDATE SET + bucket = excluded.bucket, + revision = excluded.revision, + payload = excluded.payload, + note = excluded.note" + ), + None, + ) + .await?; + } + + match plan.op { + 0 => { + ctx.db_run( + &format!( + "UPDATE fuzz_rows + SET payload = zeroblob({}), revision = {step}, note = note || ':u{step}' + WHERE id % 23 = {}", + plan.payload_size / 2 + 64, + step % 23 + ), + None, + ) + .await?; + } + 1 => { + ctx.db_run( + &format!( + "DELETE FROM fuzz_rows + WHERE id % 37 = {} AND id <> {}", + step % 37, + plan.row_id + ), + None, + ) + .await?; + } + 2 => { + ctx.db_run( + &format!( + "INSERT INTO fuzz_rows (id, bucket, revision, payload, note) + SELECT id + 5000, bucket, {step}, zeroblob({}), note || ':copy' + FROM fuzz_rows + WHERE bucket = {} + ORDER BY id + LIMIT 3 + ON CONFLICT(id) DO UPDATE SET + revision = excluded.revision, + payload = excluded.payload, + note = excluded.note", + plan.payload_size + 128, + plan.bucket + ), + None, + ) + .await?; + } + 3 => { + ctx.db_run( + &format!( + "UPDATE fuzz_rows + SET bucket = (bucket + 7) % 31, revision = {step} + WHERE id BETWEEN {} AND {}", + plan.row_id.saturating_sub(11), + plan.row_id + 11 + ), + None, + ) + .await?; + } + 4 => { + ctx.db_execute( + &format!( + "SELECT id, note FROM fuzz_rows + WHERE bucket = {} + ORDER BY revision DESC, id + LIMIT 6", + plan.bucket + ), + None, + ) + .await?; + } + 5 => { + ctx.db_run( + &format!( + "UPDATE fuzz_rows + SET payload = zeroblob({}), revision = {step} + WHERE bucket BETWEEN {} AND {}", + plan.payload_size + 256, + plan.bucket.saturating_sub(1), + plan.bucket + 1 + ), + None, + ) + .await?; + } + _ => unreachable!("plan op should be modulo 6"), + } + + ctx.db_run( + &format!( + "INSERT INTO fuzz_log (step, op, row_id, payload_size) + VALUES ({step}, {}, {}, {}) + ON CONFLICT(step) DO UPDATE SET + op = excluded.op, + row_id = excluded.row_id, + payload_size = excluded.payload_size", + plan.op, plan.row_id, plan.payload_size + ), + None, + ) + .await?; + + Ok(()) +} + +async fn run_read_checks( + ctx: &rivetkit_core::ActorContext, + step: i64, + plan: &StepPlan, +) -> Result<()> { + ctx.db_query( + &format!( + "SELECT COUNT(*) AS count FROM fuzz_rows WHERE bucket = {}", + plan.bucket + ), + None, + ) + .await?; + ctx.db_query( + &format!( + "SELECT id, note FROM fuzz_rows INDEXED BY idx_fuzz_rows_bucket_payload + WHERE bucket BETWEEN {} AND {} + ORDER BY note, id + LIMIT 16", + plan.bucket.saturating_sub(2), + plan.bucket + 2 + ), + None, + ) + .await?; + ctx.db_query( + &format!( + "SELECT id, length(payload) AS payload_len + FROM fuzz_rows + WHERE revision BETWEEN {} AND {} + ORDER BY revision DESC, id + LIMIT 12", + step.saturating_sub(20), + step + 20 + ), + None, + ) + .await?; + ctx.db_query( + "SELECT l.step, r.id + FROM fuzz_log l + LEFT JOIN fuzz_rows r ON r.id = l.row_id + ORDER BY l.step DESC + LIMIT 8", + None, + ) + .await?; + Ok(()) +} + +async fn assert_pragma_ok( + ctx: &rivetkit_core::ActorContext, + sql: &str, + column: &str, +) -> Result<&'static str> { + let rows = ctx.db_query(sql, None).await?; + let value: JsonValue = + ciborium::from_reader(Cursor::new(rows)).context("decode pragma rows from cbor")?; + let status = value + .as_array() + .and_then(|rows| rows.first()) + .and_then(|row| row.get(column)) + .and_then(JsonValue::as_str) + .with_context(|| format!("read {column} status"))?; + if status != "ok" { + bail!("{sql} failed: {status}"); + } + Ok("ok") +} + +struct StepPlan { + op: u64, + row_id: i64, + bucket: i64, + payload_size: i64, + burst_rows: usize, + schema_churn: bool, + reindex: bool, + analyze: bool, + vacuum: bool, + close_after: bool, + sleep_after: bool, +} + +impl StepPlan { + fn new(actor_id: &str, step: i64) -> Self { + let seed = hash64(actor_id.as_bytes()) ^ ((step as u64).wrapping_mul(0x9e3779b97f4a7c15)); + let mixed = mix64(seed); + let payload_multiplier = env_usize( + "SQLITE_CORRUPTION_FUZZ_PAYLOAD_MULTIPLIER", + DEFAULT_PAYLOAD_MULTIPLIER, + ) + .max(1) as i64; + let payload_size = (256 + (mixed % 16_384) as i64) * payload_multiplier; + + Self { + op: mixed % 6, + row_id: ((mixed >> 8) % 4099) as i64, + bucket: ((mixed >> 19) % 31) as i64, + payload_size, + burst_rows: 1 + ((mixed >> 27) % 5) as usize, + schema_churn: step % 11 == 0, + reindex: step % 17 == 0, + analyze: step % 19 == 0, + vacuum: step % 37 == 0, + close_after: step % 3 == 0 || mixed & 0x20 != 0, + sleep_after: step % 13 == 0, + } + } +} + +fn env_usize(name: &str, default: usize) -> usize { + env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +fn fuzz_run_id() -> String { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + format!("{}-{now}", std::process::id()) +} + +fn hash64(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +fn mix64(mut value: u64) -> u64 { + value ^= value >> 30; + value = value.wrapping_mul(0xbf58476d1ce4e5b9); + value ^= value >> 27; + value = value.wrapping_mul(0x94d049bb133111eb); + value ^ (value >> 31) +} + +fn action_output(body: &str) -> Result { + let value: JsonValue = serde_json::from_str(body).context("decode action response")?; + Ok(value.get("output").cloned().unwrap_or(JsonValue::Null)) +} + +fn read_step(state: &[u8]) -> i64 { + if state.is_empty() { + return 0; + } + + let value: JsonValue = ciborium::from_reader(Cursor::new(state)).unwrap_or(JsonValue::Null); + value + .get("step") + .and_then(JsonValue::as_i64) + .unwrap_or_default() +} + +fn encode_json(value: &JsonValue) -> Vec { + let mut out = Vec::new(); + ciborium::into_writer(value, &mut out).expect("encode cbor json"); + out +} diff --git a/rivetkit-rust/packages/rivetkit-core/tests/metrics.rs b/rivetkit-rust/packages/rivetkit-core/tests/metrics.rs index 6a636edf35..2cc4303de1 100644 --- a/rivetkit-rust/packages/rivetkit-core/tests/metrics.rs +++ b/rivetkit-rust/packages/rivetkit-core/tests/metrics.rs @@ -71,6 +71,40 @@ mod moved_tests { assert_metric_value(&rendered, "rivet_actor_lifecycle_event_inbox_depth", "3"); } + #[test] + fn actor_connection_close_metrics_render() { + let metrics = ActorMetrics::new("actor-conn", Some(9), "counter/main", "envoy-1"); + + metrics.record_connection_closed("ws_close", Duration::from_millis(25)); + + let rendered = render_global_metrics(); + let closed_line = rendered + .lines() + .find(|line| { + metric_line_for_actor(line, "rivet_actor_connection_closed_total", "actor-conn:9") + && line.contains("reason=\"ws_close\"") + }) + .expect("connection close counter should render"); + assert!( + closed_line.ends_with('1'), + "connection close counter should increment: {closed_line}" + ); + let lifetime_line = rendered + .lines() + .find(|line| { + metric_line_for_actor( + line, + "rivet_actor_connection_lifetime_seconds_count", + "actor-conn:9", + ) + }) + .expect("connection lifetime histogram should render"); + assert!( + lifetime_line.ends_with('1'), + "connection lifetime histogram should observe: {lifetime_line}" + ); + } + #[test] fn actor_active_metric_is_retained_after_drop() { let metrics = ActorMetrics::new("actor-retention", Some(7), "counter/main", "envoy-1"); diff --git a/rivetkit-rust/packages/rivetkit-core/tests/registry_http.rs b/rivetkit-rust/packages/rivetkit-core/tests/registry_http.rs index 23b54ac5d6..23e0c365af 100644 --- a/rivetkit-rust/packages/rivetkit-core/tests/registry_http.rs +++ b/rivetkit-rust/packages/rivetkit-core/tests/registry_http.rs @@ -8,8 +8,7 @@ mod moved_tests { HttpResponseEncoding, authorization_bearer_token, authorization_bearer_token_map, framework_action_error_response, framework_anyhow_error_response_with_actor, is_actor_request_path, message_boundary_error_response, - message_boundary_error_response_with_actor, normalize_actor_request_path, - request_encoding, + message_boundary_error_response_with_actor, normalize_actor_request_path, request_encoding, workflow_dispatch_result, }; use crate::actor::action::ActionDispatchError; diff --git a/rivetkit-rust/packages/rivetkit-core/tests/sqlite.rs b/rivetkit-rust/packages/rivetkit-core/tests/sqlite.rs index 174c327657..7e2038de67 100644 --- a/rivetkit-rust/packages/rivetkit-core/tests/sqlite.rs +++ b/rivetkit-rust/packages/rivetkit-core/tests/sqlite.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use std::sync::atomic::AtomicBool; use std::sync::Mutex as StdMutex; +use std::sync::atomic::AtomicBool; use super::*; use depot_client_types::{HEAD_FENCE_MISMATCH_CODE, HEAD_FENCE_MISMATCH_GROUP}; @@ -318,7 +318,10 @@ async fn remote_execute_logs_operation_context_at_source() { let result = db .execute( "SELECT ?", - Some(vec![BindParam::Integer(1), BindParam::Text("two".to_owned())]), + Some(vec![ + BindParam::Integer(1), + BindParam::Text("two".to_owned()), + ]), ) .await; diff --git a/rivetkit-rust/packages/rivetkit-core/tests/state.rs b/rivetkit-rust/packages/rivetkit-core/tests/state.rs index 400fe53d14..abf648e2e6 100644 --- a/rivetkit-rust/packages/rivetkit-core/tests/state.rs +++ b/rivetkit-rust/packages/rivetkit-core/tests/state.rs @@ -178,10 +178,7 @@ mod moved_tests { #[tokio::test] async fn request_save_coalesces_and_escalates_to_immediate() { - let state = ActorContext::new_for_state_tests( - new_in_memory(), - ActorConfig::default(), - ); + let state = ActorContext::new_for_state_tests(new_in_memory(), ActorConfig::default()); let (events_tx, mut events_rx) = mpsc::unbounded_channel(); state.configure_lifecycle_events(Some(events_tx)); diff --git a/rivetkit-typescript/packages/rivetkit-napi/index.d.ts b/rivetkit-typescript/packages/rivetkit-napi/index.d.ts index 2cba08a1c7..d719c8e89b 100644 --- a/rivetkit-typescript/packages/rivetkit-napi/index.d.ts +++ b/rivetkit-typescript/packages/rivetkit-napi/index.d.ts @@ -316,6 +316,10 @@ export declare class CoreRegistry { */ shutdown(): Promise diagnostics(): Promise + actorStopThresholdMs(): Promise + health(): Promise + metadata(): JsRegistryRouteResponse + metrics(): Promise handleServerlessRequest(req: JsServerlessRequest, onStreamEvent: (...args: any[]) => any, cancelToken: CancellationToken, config: JsServeConfig): Promise } export declare class Schedule { @@ -327,3 +331,12 @@ export declare class WebSocket { close(code?: number | undefined | null, reason?: string | undefined | null): Promise setEventCallback(callback: (...args: any[]) => any): void } +export declare function jsSetEventloopLagQuantile(quantile: string, seconds: number): void +export declare function jsSetEventloopUtilization(value: number): void +export declare function jsSetEventloopHeartbeatTsMs(epochMs: number): void +export declare function jsAddProcessCpuSeconds(mode: string, seconds: number): void +export declare function jsSetProcessResidentMemoryBytes(bytes: number): void +export declare function jsSetHeapBytes(state: string, bytes: number): void +export declare function jsObserveGcDuration(kind: string, seconds: number): void +export declare function jsSetActiveHandles(count: number): void +export declare function jsSetActiveRequests(count: number): void diff --git a/rivetkit-typescript/packages/rivetkit-napi/index.js b/rivetkit-typescript/packages/rivetkit-napi/index.js index d5c3c616dc..4c4d3de6fe 100644 --- a/rivetkit-typescript/packages/rivetkit-napi/index.js +++ b/rivetkit-typescript/packages/rivetkit-napi/index.js @@ -310,7 +310,7 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { ActorContext, NapiActorFactory, CancellationToken, ConnHandle, JsNativeDatabase, Kv, Queue, QueueMessage, CoreRegistry, Schedule, WebSocket } = nativeBinding +const { ActorContext, NapiActorFactory, CancellationToken, ConnHandle, JsNativeDatabase, Kv, Queue, QueueMessage, CoreRegistry, Schedule, WebSocket, jsSetEventloopLagQuantile, jsSetEventloopUtilization, jsSetEventloopHeartbeatTsMs, jsAddProcessCpuSeconds, jsSetProcessResidentMemoryBytes, jsSetHeapBytes, jsObserveGcDuration, jsSetActiveHandles, jsSetActiveRequests } = nativeBinding module.exports.ActorContext = ActorContext module.exports.NapiActorFactory = NapiActorFactory @@ -323,3 +323,12 @@ module.exports.QueueMessage = QueueMessage module.exports.CoreRegistry = CoreRegistry module.exports.Schedule = Schedule module.exports.WebSocket = WebSocket +module.exports.jsSetEventloopLagQuantile = jsSetEventloopLagQuantile +module.exports.jsSetEventloopUtilization = jsSetEventloopUtilization +module.exports.jsSetEventloopHeartbeatTsMs = jsSetEventloopHeartbeatTsMs +module.exports.jsAddProcessCpuSeconds = jsAddProcessCpuSeconds +module.exports.jsSetProcessResidentMemoryBytes = jsSetProcessResidentMemoryBytes +module.exports.jsSetHeapBytes = jsSetHeapBytes +module.exports.jsObserveGcDuration = jsObserveGcDuration +module.exports.jsSetActiveHandles = jsSetActiveHandles +module.exports.jsSetActiveRequests = jsSetActiveRequests diff --git a/rivetkit-typescript/packages/rivetkit-napi/src/lib.rs b/rivetkit-typescript/packages/rivetkit-napi/src/lib.rs index 51e164eab6..c54e0ae652 100644 --- a/rivetkit-typescript/packages/rivetkit-napi/src/lib.rs +++ b/rivetkit-typescript/packages/rivetkit-napi/src/lib.rs @@ -7,6 +7,7 @@ pub mod kv; pub mod napi_actor_events; pub mod queue; pub mod registry; +pub mod runtime_metrics; pub mod schedule; pub mod types; pub mod websocket; diff --git a/rivetkit-typescript/packages/rivetkit-napi/src/napi_actor_events.rs b/rivetkit-typescript/packages/rivetkit-napi/src/napi_actor_events.rs index 4553e11d32..d9e28c25c5 100644 --- a/rivetkit-typescript/packages/rivetkit-napi/src/napi_actor_events.rs +++ b/rivetkit-typescript/packages/rivetkit-napi/src/napi_actor_events.rs @@ -1,5 +1,5 @@ -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use anyhow::Result; diff --git a/rivetkit-typescript/packages/rivetkit-napi/src/registry.rs b/rivetkit-typescript/packages/rivetkit-napi/src/registry.rs index 5f720d3232..6187d24a04 100644 --- a/rivetkit-typescript/packages/rivetkit-napi/src/registry.rs +++ b/rivetkit-typescript/packages/rivetkit-napi/src/registry.rs @@ -241,6 +241,30 @@ impl CoreRegistry { Ok(()) } + #[napi(js_name = "actorStopThresholdMs")] + pub async fn actor_stop_threshold_ms(&self) -> napi::Result> { + let (active_envoy, serverless_runtime) = { + let guard = self.state.lock().await; + match &*guard { + RegistryState::Serving => (self.serving_envoy.lock().clone(), None), + RegistryState::Serverless(runtime) => (None, Some(runtime.clone())), + RegistryState::Registering(_) + | RegistryState::BuildingServerless + | RegistryState::ShuttingDown + | RegistryState::ShutDown => (None, None), + } + }; + + if let Some(runtime) = serverless_runtime { + return Ok(runtime.active_envoy_actor_stop_threshold_ms().await); + } + + match active_envoy { + Some(envoy) => Ok(envoy.actor_stop_threshold_ms().await), + None => Ok(None), + } + } + #[napi] pub async fn diagnostics(&self) -> napi::Result { let guard = self.state.lock().await; @@ -276,6 +300,98 @@ impl CoreRegistry { Ok(diagnostics) } + #[napi] + pub async fn health(&self) -> napi::Result { + let version = self.route_package_version(); + let serverless_runtime = { + let guard = self.state.lock().await; + match &*guard { + RegistryState::Registering(_) => { + return Ok(health_response(503, "not_started", &version)); + } + RegistryState::BuildingServerless => { + return Ok(health_response(503, "starting", &version)); + } + RegistryState::Serving => match self + .serving_envoy + .lock() + .as_ref() + .map(CoreEnvoyHandle::status) + { + Some(envoy) => { + return Ok(health_response( + if envoy.ping_healthy { 200 } else { 503 }, + if envoy.ping_healthy { "ok" } else { "engine_ping_stale" }, + &version, + )); + } + None => return Ok(health_response(503, "starting", &version)), + }, + RegistryState::Serverless(runtime) => runtime.clone(), + RegistryState::ShuttingDown => { + return Ok(health_response(503, "shutting_down", &version)); + } + RegistryState::ShutDown => { + return Ok(health_response(503, "shut_down", &version)); + } + } + }; + + let response = match serverless_runtime.active_envoy_status().await { + Some(envoy) => health_response( + if envoy.ping_healthy { 200 } else { 503 }, + if envoy.ping_healthy { "ok" } else { "engine_ping_stale" }, + &version, + ), + None => health_response(503, "engine_ping_stale", &version), + }; + Ok(response) + } + + #[napi] + pub fn metadata(&self) -> napi::Result { + self.route_metadata.lock().clone().ok_or_else(|| { + napi_anyhow_error( + NapiInvalidState { + state: "metadata_unavailable".to_owned(), + reason: "registry metadata is not available until the registry has started" + .to_owned(), + } + .build(), + ) + }) + } + + #[napi] + pub async fn metrics(&self) -> napi::Result { + let (active_envoy, serverless_runtime) = { + let guard = self.state.lock().await; + match &*guard { + RegistryState::Serving => ( + self.serving_envoy.lock().as_ref().map(CoreEnvoyHandle::status), + None, + ), + RegistryState::Serverless(runtime) => (None, Some(runtime.clone())), + RegistryState::Registering(_) + | RegistryState::BuildingServerless + | RegistryState::ShuttingDown + | RegistryState::ShutDown => (None, None), + } + }; + let serverless_envoy = match serverless_runtime { + Some(runtime) => runtime.active_envoy_status().await, + None => None, + }; + let envoy_status = active_envoy.as_ref().or(serverless_envoy.as_ref()); + let metrics = rivetkit_core::metrics_endpoint::render_prometheus_metrics(envoy_status) + .map_err(napi_anyhow_error)?; + Ok(JsRegistryRouteResponse { + status: 200, + headers: HashMap::from([("content-type".to_owned(), metrics.content_type)]), + body: Buffer::from(metrics.body), + }) + } + #[napi(ts_return_type = "Promise")] pub fn handle_serverless_request( &self, diff --git a/rivetkit-typescript/packages/rivetkit-napi/src/runtime_metrics.rs b/rivetkit-typescript/packages/rivetkit-napi/src/runtime_metrics.rs new file mode 100644 index 0000000000..8c7f5eeaed --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit-napi/src/runtime_metrics.rs @@ -0,0 +1,47 @@ +use napi_derive::napi; +use rivetkit_core::runtime_metrics; + +#[napi] +pub fn js_set_eventloop_lag_quantile(quantile: String, seconds: f64) { + runtime_metrics::set_eventloop_lag_quantile(&quantile, seconds); +} + +#[napi] +pub fn js_set_eventloop_utilization(value: f64) { + runtime_metrics::set_eventloop_utilization(value); +} + +#[napi] +pub fn js_set_eventloop_heartbeat_ts_ms(epoch_ms: i64) { + runtime_metrics::set_eventloop_heartbeat_ts_ms(epoch_ms); +} + +#[napi] +pub fn js_add_process_cpu_seconds(mode: String, seconds: f64) { + runtime_metrics::add_process_cpu_seconds(&mode, seconds); +} + +#[napi] +pub fn js_set_process_resident_memory_bytes(bytes: i64) { + runtime_metrics::set_process_resident_memory_bytes(bytes); +} + +#[napi] +pub fn js_set_heap_bytes(state: String, bytes: i64) { + runtime_metrics::set_heap_bytes(&state, bytes); +} + +#[napi] +pub fn js_observe_gc_duration(kind: String, seconds: f64) { + runtime_metrics::observe_gc_duration(&kind, seconds); +} + +#[napi] +pub fn js_set_active_handles(count: i64) { + runtime_metrics::set_active_handles(count); +} + +#[napi] +pub fn js_set_active_requests(count: i64) { + runtime_metrics::set_active_requests(count); +} diff --git a/rivetkit-typescript/packages/rivetkit-wasm/src/lib.rs b/rivetkit-typescript/packages/rivetkit-wasm/src/lib.rs index a1ea940dd6..f3f1207b62 100644 --- a/rivetkit-typescript/packages/rivetkit-wasm/src/lib.rs +++ b/rivetkit-typescript/packages/rivetkit-wasm/src/lib.rs @@ -7,18 +7,15 @@ use std::time::Duration; use anyhow::{Result, anyhow}; use js_sys::{Array, Function, Object, Promise, Reflect, Uint8Array}; -use rivet_error::{ - ActorSpecifier, RivetError as RivetTransportError, RivetErrorKind, -}; +use rivet_error::{ActorSpecifier, RivetError as RivetTransportError, RivetErrorKind}; use rivetkit_core::error::public_error_status_code; use rivetkit_core::inspector::InspectorAuth; use rivetkit_core::{ ActorConfig, ActorConfigInput, ActorEvent, ActorFactory as CoreActorFactory, ActorStart, - ActorWorkKind, - BindParam, ColumnValue, CoreRegistry as NativeCoreRegistry, CoreServerlessRuntime, - EnqueueAndWaitOpts, KeepAwakeRegion, ListOpts, QueueMessage, QueueNextBatchOpts, - QueueSendResult, QueueSendStatus, QueueTryNextBatchOpts, QueueWaitOpts, Request, - RequestSaveOpts, Response, RuntimeSpawner, SerializeStateReason, ServeConfig, + ActorWorkKind, BindParam, ColumnValue, CoreRegistry as NativeCoreRegistry, + CoreServerlessRuntime, EnqueueAndWaitOpts, KeepAwakeRegion, ListOpts, QueueMessage, + QueueNextBatchOpts, QueueSendResult, QueueSendStatus, QueueTryNextBatchOpts, QueueWaitOpts, + Request, RequestSaveOpts, Response, RuntimeSpawner, SerializeStateReason, ServeConfig, ServerlessRequest, StateDelta, WebSocket, WebSocketCallbackRegion, WsMessage, }; use tokio::sync::oneshot; @@ -2865,9 +2862,7 @@ mod tests { } fn transport_message(error: &anyhow::Error) -> String { - transport_error(error) - .message() - .to_owned() + transport_error(error).message().to_owned() } #[test] diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/engine/actor-driver.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/engine/actor-driver.ts index bb7eabd6ed..60b65c6940 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/engine/actor-driver.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/engine/actor-driver.ts @@ -78,7 +78,7 @@ import { import { logger } from "./log"; const ENVOY_SSE_PING_INTERVAL = 1000; -const ENVOY_STOP_WAIT_MS = 15_000; +const FALLBACK_ENVOY_STOP_WAIT_MS = 15_000; const INITIAL_SLEEP_TIMEOUT_MS = 250; const REMOTE_ACK_HOOK_QUERY_PARAM = "__rivetkitAckHook"; @@ -847,6 +847,7 @@ export class EngineActorDriver implements ActorDriver { return; } this.#isShuttingDown = true; + const envoyStopWaitMs = this.#envoyStopWaitMs(); logger().info({ msg: "stopping engine actor driver", immediate }); if (!immediate) { @@ -861,7 +862,7 @@ export class EngineActorDriver implements ActorDriver { this.startSleep(actorId); } - const actorSleepDeadline = Date.now() + ENVOY_STOP_WAIT_MS; + const actorSleepDeadline = Date.now() + envoyStopWaitMs; while (this.#actors.size > 0 && Date.now() < actorSleepDeadline) { await new Promise((resolve) => setTimeout(resolve, 50)); } @@ -870,7 +871,7 @@ export class EngineActorDriver implements ActorDriver { logger().warn({ msg: "timed out waiting for actors to stop before envoy drain", remainingActors: this.#actors.size, - waitMs: ENVOY_STOP_WAIT_MS, + waitMs: envoyStopWaitMs, }); // Snapshot so concurrent removals from `stopActor` do not // invalidate the iterator. @@ -912,13 +913,13 @@ export class EngineActorDriver implements ActorDriver { const stopped = await Promise.race([ this.#envoyStopped.promise.then(() => true), new Promise((resolve) => - setTimeout(() => resolve(false), ENVOY_STOP_WAIT_MS), + setTimeout(() => resolve(false), envoyStopWaitMs), ), ]); if (!stopped) { logger().warn({ msg: "timed out waiting for envoy shutdown", - waitMs: ENVOY_STOP_WAIT_MS, + waitMs: envoyStopWaitMs, }); } @@ -929,6 +930,16 @@ export class EngineActorDriver implements ActorDriver { await this.#envoy.started(); } + #envoyStopWaitMs(): number { + const actorStopThreshold = Number( + this.#envoy.getProtocolMetadata()?.actorStopThreshold, + ); + if (Number.isFinite(actorStopThreshold) && actorStopThreshold > 0) { + return actorStopThreshold; + } + return FALLBACK_ENVOY_STOP_WAIT_MS; + } + async #bindHibernatableConnectSocket( binding: HibernatableConnectBinding, isRestoringHibernatable: boolean, diff --git a/rivetkit-typescript/packages/rivetkit/src/registry/config/index.ts b/rivetkit-typescript/packages/rivetkit/src/registry/config/index.ts index 50a4f12081..27da4a7f09 100644 --- a/rivetkit-typescript/packages/rivetkit/src/registry/config/index.ts +++ b/rivetkit-typescript/packages/rivetkit/src/registry/config/index.ts @@ -264,8 +264,8 @@ export const RegistryConfigSchema = z .object({ /** * Wait this many milliseconds for the serve promise to resolve - * after calling `CoreRegistry::shutdown()`. Defaults to 30s, - * matching Kubernetes `terminationGracePeriodSeconds`. + * after calling `CoreRegistry::shutdown()`. Defaults to the + * engine-provided actor stop threshold once the envoy connects. * * Must be >= rivetkit-core's drain timeout (20s) + margin. */ @@ -273,8 +273,7 @@ export const RegistryConfigSchema = z .number() .int() .min(1_000) - .optional() - .default(30_000), + .optional(), /** * If true, rivetkit will not install SIGINT/SIGTERM handlers. * Use when the host application owns signal policy and will @@ -284,7 +283,6 @@ export const RegistryConfigSchema = z }) .optional() .default(() => ({ - gracePeriodMs: 30_000, disableSignalHandlers: false, })), }) diff --git a/rivetkit-typescript/packages/rivetkit/src/registry/index.ts b/rivetkit-typescript/packages/rivetkit/src/registry/index.ts index 7d2b61af42..971e2669bf 100644 --- a/rivetkit-typescript/packages/rivetkit/src/registry/index.ts +++ b/rivetkit-typescript/packages/rivetkit/src/registry/index.ts @@ -15,6 +15,22 @@ import type { RuntimeServerlessResponseHead } from "./runtime"; type ShutdownSignal = "SIGINT" | "SIGTERM"; +function signalExitCode(signal: ShutdownSignal): number { + switch (signal) { + case "SIGINT": + return 130; + case "SIGTERM": + return 143; + } +} + +function finishShutdownSignal(signal: ShutdownSignal): void { + if (process.pid === 1) { + process.exit(signalExitCode(signal)); + } + process.kill(process.pid, signal); +} + export type FetchHandler = ( request: Request, ...args: any @@ -396,10 +412,11 @@ export class Registry { ): void { if (this.#shutdownInFlight !== null) { // Second delivery of the same (or another) shutdown signal. - // Remove our handler only (preserving any user-installed listeners) - // and re-raise so Node proceeds with its default exit path. + // Remove our handler only, preserving any user-installed listeners. + // PID 1 must exit directly because re-raised default signals can be + // swallowed by the container signal path. this.#removeSignalHandlers(); - process.kill(process.pid, signal); + finishShutdownSignal(signal); return; } this.#shutdownInFlight = this.#runShutdown( @@ -416,11 +433,13 @@ export class Registry { config: RegistryConfig, configuredRegistryPromise: ReturnType, ): Promise { - const gracePeriodMs = config.shutdown?.gracePeriodMs ?? 30_000; + const gracePeriodMs = + config.shutdown?.gracePeriodMs ?? + (await this.#actorStopThresholdMs(configuredRegistryPromise)) ?? + 30 * 60 * 1000; // Race the entire drain sequence (both modes + serve promise) against - // a single grace ceiling. Without this, each mode's Rust-side drain - // (20s) could stack sequentially and blow past gracePeriodMs before - // we re-raise the signal. + // a single grace ceiling. By default, this uses the engine-provided + // actor stop threshold, matching Pegboard's hard cutoff for actors. const drain = async () => { // Shut down every live `CoreRegistry` we know about. Mode A // (`start()`) and Mode B (`handler()`) each build a separate @@ -474,7 +493,30 @@ export class Registry { ), ]); this.#removeSignalHandlers(); - process.kill(process.pid, signal); + finishShutdownSignal(signal); + } + + async #actorStopThresholdMs( + configuredRegistryPromise: ReturnType, + ): Promise { + try { + const { runtime, registry } = await configuredRegistryPromise; + const thresholdMs = + await runtime.registryActorStopThresholdMs?.(registry); + if ( + thresholdMs !== undefined && + Number.isFinite(thresholdMs) && + thresholdMs > 0 + ) { + return thresholdMs; + } + } catch (err) { + logger().warn( + { err }, + "failed to read actor stop threshold for shutdown grace", + ); + } + return undefined; } #removeSignalHandlers(): void { diff --git a/rivetkit-typescript/packages/rivetkit/src/registry/napi-runtime.ts b/rivetkit-typescript/packages/rivetkit/src/registry/napi-runtime.ts index f1b8890a34..0f19700839 100644 --- a/rivetkit-typescript/packages/rivetkit/src/registry/napi-runtime.ts +++ b/rivetkit-typescript/packages/rivetkit/src/registry/napi-runtime.ts @@ -212,6 +212,48 @@ export class NapiCoreRuntime implements CoreRuntime { }; } + async registryActorStopThresholdMs( + registry: RegistryHandle, + ): Promise { + return ( + (await asNativeRegistry(registry).actorStopThresholdMs()) ?? + undefined + ); + } + + async registryHealth( + registry: RegistryHandle, + ): Promise { + const response = await asNativeRegistry(registry).health(); + return { + status: response.status, + headers: response.headers, + body: response.body, + }; + } + + async registryMetadata( + registry: RegistryHandle, + ): Promise { + const response = asNativeRegistry(registry).metadata(); + return { + status: response.status, + headers: response.headers, + body: response.body, + }; + } + + async registryMetrics( + registry: RegistryHandle, + ): Promise { + const response = await asNativeRegistry(registry).metrics(); + return { + status: response.status, + headers: response.headers, + body: response.body, + }; + } + async handleServerlessRequest( registry: RegistryHandle, req: RuntimeServerlessRequest, diff --git a/rivetkit-typescript/packages/rivetkit/src/registry/native.ts b/rivetkit-typescript/packages/rivetkit/src/registry/native.ts index 3059e3f37d..9291839d79 100644 --- a/rivetkit-typescript/packages/rivetkit/src/registry/native.ts +++ b/rivetkit-typescript/packages/rivetkit/src/registry/native.ts @@ -421,7 +421,20 @@ async function cleanupNativeSleepRuntimeState( runtime: CoreRuntime, ctx: ActorContextHandle, ): Promise { - await runtime.actorWaitForTrackedShutdownWork(ctx); + const waitStarted = Date.now(); + const drained = await runtime.actorWaitForTrackedShutdownWork(ctx); + const waitMs = Date.now() - waitStarted; + if (drained) { + logger().debug({ + msg: "sleep cleanup: tracked shutdown work drained", + waitMs, + }); + } else { + logger().warn({ + msg: "sleep cleanup: shutdown deadline reached before tracked work drained; closing DB anyway", + waitMs, + }); + } await closeNativeDatabaseClient(runtime, ctx); await closeNativeSqlDatabase(runtime, ctx); clearNativeRuntimeState(runtime, ctx); @@ -2791,22 +2804,41 @@ export class ActorContextHandleAdapter { } keepAwake(promise: Promise): Promise { + const startedAt = Date.now(); + logger().debug({ + msg: "keepAwake registered", + at: startedAt, + }); const trackedPromise = Promise.resolve(promise) - .catch((error) => { - logger().warn({ - msg: "keepAwake promise rejected", - error: stringifyError(error), - }); - }) + .then( + () => { + logger().debug({ + msg: "keepAwake promise resolved", + durationMs: Date.now() - startedAt, + }); + }, + (error) => { + logger().warn({ + msg: "keepAwake promise rejected", + durationMs: Date.now() - startedAt, + error: stringifyError(error), + }); + }, + ) .then(() => null); try { callNativeSync(() => this.#runtime.actorKeepAwake(this.#ctx, trackedPromise), ); } catch (error) { - if (!isClosedTaskRegistrationError(error)) { - throw error; + if (isClosedTaskRegistrationError(error)) { + logger().warn({ + msg: "keepAwake registration dropped (teardown already started); promise will not delay grace", + error: stringifyError(error), + }); + return promise; } + throw error; } return promise; } @@ -4654,6 +4686,14 @@ export async function buildConfiguredRegistry(config: RegistryConfig): Promise<{ serveConfig: RuntimeServeConfig; }> { const runtime = await loadConfiguredRuntime(config); + if (runtime.kind === "napi") { + // Start Node.js runtime health metrics collection (event loop lag, + // GC, heap, CPU, libuv handles). Only available on the native NAPI + // runtime; wasm/edge hosts do not expose perf_hooks/v8 the same + // way and have no Rust-side prometheus collectors loaded. + const { startProcessMetrics } = await import("./process-metrics"); + startProcessMetrics(); + } return buildRegistryWithRuntime( normalizeRuntimeConfig(config, runtime), runtime, diff --git a/rivetkit-typescript/packages/rivetkit/src/registry/process-metrics.ts b/rivetkit-typescript/packages/rivetkit/src/registry/process-metrics.ts new file mode 100644 index 0000000000..27452e7a87 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/registry/process-metrics.ts @@ -0,0 +1,169 @@ +/** + * Node.js runtime health metrics. + * + * Collects JS-internal data (event loop lag, GC, heap, libuv handles, + * event loop utilization, CPU) using Node built-ins (`node:perf_hooks`, + * `process`, `node:v8`, `PerformanceObserver`) and pushes them across NAPI + * into Rust-side prometheus collectors registered with + * `rivet_metrics::REGISTRY` so they appear on the existing `/metrics` + * endpoint. + * + * All data collection happens here in TypeScript. The NAPI bridge is pure + * type marshalling and the Rust side only registers + stores the metrics. + */ +import { monitorEventLoopDelay, performance, PerformanceObserver } from "node:perf_hooks"; +import { getHeapStatistics } from "node:v8"; +import * as napi from "@rivetkit/rivetkit-napi"; + +const SCRAPE_INTERVAL_MS = 5_000; +const HEARTBEAT_INTERVAL_MS = 100; +const EVENTLOOP_DELAY_RESOLUTION_MS = 20; +const NS_PER_SECOND = 1e9; +const US_PER_SECOND = 1e6; + +// V8 GC kind bitfield from Node's perf_hooks documentation. A `gc` performance +// entry's `kind` field is one of these values. +const GC_KIND_NAMES: Record = { + 1: "minor", + 2: "major", + 4: "incremental", + 8: "weakcb", +}; + +interface ProcessMetricsState { + scrapeInterval: NodeJS.Timeout; + heartbeatInterval: NodeJS.Timeout; + gcObserver: PerformanceObserver; + eventLoopHistogram: ReturnType; + lastCpuUsage: NodeJS.CpuUsage; + lastEventLoopUtilization: ReturnType; +} + +let state: ProcessMetricsState | undefined; + +export function startProcessMetrics(): void { + if (state) { + return; + } + + const eventLoopHistogram = monitorEventLoopDelay({ + resolution: EVENTLOOP_DELAY_RESOLUTION_MS, + }); + eventLoopHistogram.enable(); + + const gcObserver = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + const kind = + (entry as PerformanceEntry & { detail?: { kind?: number }; kind?: number }).detail + ?.kind ?? + (entry as PerformanceEntry & { kind?: number }).kind; + if (typeof kind !== "number") continue; + const kindName = GC_KIND_NAMES[kind]; + if (!kindName) continue; + // `entry.duration` is in milliseconds; convert to seconds. + napi.jsObserveGcDuration(kindName, entry.duration / 1000); + } + }); + gcObserver.observe({ entryTypes: ["gc"], buffered: false }); + + const lastCpuUsage = process.cpuUsage(); + const lastEventLoopUtilization = performance.eventLoopUtilization(); + + const heartbeatInterval = setInterval(() => { + napi.jsSetEventloopHeartbeatTsMs(Date.now()); + }, HEARTBEAT_INTERVAL_MS); + heartbeatInterval.unref(); + + const scrapeInterval = setInterval(() => { + try { + collectAndPush(); + } catch { + // Collection errors must never bring down the process; metrics + // are best-effort. + } + }, SCRAPE_INTERVAL_MS); + scrapeInterval.unref(); + + state = { + scrapeInterval, + heartbeatInterval, + gcObserver, + eventLoopHistogram, + lastCpuUsage, + lastEventLoopUtilization, + }; + + // Emit one snapshot immediately so freshly-scraped instances have data. + napi.jsSetEventloopHeartbeatTsMs(Date.now()); + try { + collectAndPush(); + } catch { + // As above; best-effort. + } +} + +export function stopProcessMetrics(): void { + if (!state) { + return; + } + clearInterval(state.scrapeInterval); + clearInterval(state.heartbeatInterval); + state.gcObserver.disconnect(); + state.eventLoopHistogram.disable(); + state = undefined; +} + +function collectAndPush(): void { + if (!state) return; + + // Event loop delay quantiles. `monitorEventLoopDelay()` reports values in + // nanoseconds; convert to seconds. Reset after reading so the next window + // reflects only the new interval. + const hist = state.eventLoopHistogram; + napi.jsSetEventloopLagQuantile("p50", hist.percentile(50) / NS_PER_SECOND); + napi.jsSetEventloopLagQuantile("p90", hist.percentile(90) / NS_PER_SECOND); + napi.jsSetEventloopLagQuantile("p99", hist.percentile(99) / NS_PER_SECOND); + napi.jsSetEventloopLagQuantile("max", hist.max / NS_PER_SECOND); + hist.reset(); + + // Event loop utilization delta over the scrape window. + const nextElu = performance.eventLoopUtilization(); + const eluDelta = performance.eventLoopUtilization(nextElu, state.lastEventLoopUtilization); + state.lastEventLoopUtilization = nextElu; + napi.jsSetEventloopUtilization(eluDelta.utilization); + + // CPU usage delta. `process.cpuUsage()` returns microseconds. + const nextCpu = process.cpuUsage(); + const userDeltaUs = nextCpu.user - state.lastCpuUsage.user; + const systemDeltaUs = nextCpu.system - state.lastCpuUsage.system; + state.lastCpuUsage = nextCpu; + if (userDeltaUs > 0) { + napi.jsAddProcessCpuSeconds("user", userDeltaUs / US_PER_SECOND); + } + if (systemDeltaUs > 0) { + napi.jsAddProcessCpuSeconds("system", systemDeltaUs / US_PER_SECOND); + } + + // Memory + heap. + const mem = process.memoryUsage(); + napi.jsSetProcessResidentMemoryBytes(mem.rss); + napi.jsSetHeapBytes("used", mem.heapUsed); + napi.jsSetHeapBytes("total", mem.heapTotal); + const heapLimit = getHeapStatistics().heap_size_limit; + napi.jsSetHeapBytes("limit", heapLimit); + + // libuv active handles + requests. These are unstable Node internals + // guarded behind underscore-prefixed names; if a future Node release + // removes them the try/catch above keeps the rest of the collection + // alive. + const proc = process as unknown as { + _getActiveHandles?: () => unknown[]; + _getActiveRequests?: () => unknown[]; + }; + if (typeof proc._getActiveHandles === "function") { + napi.jsSetActiveHandles(proc._getActiveHandles().length); + } + if (typeof proc._getActiveRequests === "function") { + napi.jsSetActiveRequests(proc._getActiveRequests().length); + } +} diff --git a/rivetkit-typescript/packages/rivetkit/src/registry/runtime.ts b/rivetkit-typescript/packages/rivetkit/src/registry/runtime.ts index 0e4875e966..11461d266f 100644 --- a/rivetkit-typescript/packages/rivetkit/src/registry/runtime.ts +++ b/rivetkit-typescript/packages/rivetkit/src/registry/runtime.ts @@ -309,6 +309,9 @@ export interface CoreRuntime { config: RuntimeServeConfig, ): Promise; shutdownRegistry(registry: RegistryHandle): Promise; + registryActorStopThresholdMs?( + registry: RegistryHandle, + ): Promise; handleServerlessRequest( registry: RegistryHandle, req: RuntimeServerlessRequest, diff --git a/rivetkit-typescript/packages/rivetkit/tests/driver/engine-restart-serverless.md b/rivetkit-typescript/packages/rivetkit/tests/driver/engine-restart-serverless.md new file mode 100644 index 0000000000..e51d3631a6 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/tests/driver/engine-restart-serverless.md @@ -0,0 +1,276 @@ +# Engine Restart Serverless Investigation + +## Scope + +This documents the local reproduction work for serverless Rivet Actor access after restarting `rivet-engine`. + +The harness runs: + +- `tests/driver/engine-restart-serverless.ts` +- `tests/fixtures/engine-restart-serverless-runtime.ts` + +The runtime is a normal Node process, not Vitest. It listens on a port with `serverless.basePath = "/api/rivet"`. The engine is started as a real `rivet-engine` binary and configured with a serverless runner config pointing at that runtime. + +## Main Finding + +The bug is not SQLite-specific. + +After `rivet-engine` restarts, there is a short window where gateway traffic to already-existing warmed actors can hang. This affects: + +- action calls through the client +- HTTP actor requests through `/gateway/{actor}/request/...` +- raw actor WebSockets through `/gateway/{actor}/websocket/...` + +New actor keys work immediately while warmed existing keys can hang, so this looks like stale or not-yet-settled gateway/serverless actor routing state for existing actors. + +## Timing Window + +Timing is measured from the moment engine `/health` returns after restart. + +Observed local window: + +- Unsafe: `0-2400ms` +- Flaky edge: roughly `2500-3250ms` +- Local minimum to trust: `3000ms` +- Conservative diagnostic delay: `5000ms` + +This is not a crisp timer. The edge moves run-to-run. + +## Baseline Actor Action Results + +These probes use the same existing actor key and a new key after engine restart. + +No heartbeat, immediate probe: + +- same-handle `getCount`: timed out +- same-handle `tick`: timed out +- fresh handle with same key `getCount`: timed out +- new key `tick`: succeeded + +This reproduced the "actor bricked" symptom: + +```text +bricked actor symptom reproduced. mode=idle failedPostRestartProbes=3 before=0 +``` + +No heartbeat, delayed probe: + +- same existing key succeeded +- new key succeeded + +This means the actor is not permanently broken in the local harness. It is unreachable through the existing-key gateway path during the post-restart race window. + +## Heartbeat Results + +The runtime supports `RIVETKIT_HEARTBEAT_MODE=none|sqlite|kv`. + +Immediate post-restart probes still reproduced the brick with: + +- no heartbeat +- SQLite heartbeat +- KV heartbeat + +Both SQLite and KV actor-originated heartbeat traffic could recover after restart while gateway/client traffic to the same warmed existing actor key still hung. + +Conclusion: heartbeat success does not prove gateway readiness, and SQLite is not special. + +## HTTP Gateway Health Sweep + +The actor exposes an `onRequest` health endpoint at: + +```text +/gateway/{actor}/request/health +``` + +The sweep warms one existing actor key per delay before restart, then probes each key once after engine `/health` returns. This avoids an early failed probe poisoning later delay measurements. + +Coarse sweep: + +```text +0ms: timeout +250ms: timeout +500ms: timeout +1000ms: timeout +2000ms: timeout +3000ms: 200 OK +5000ms: 200 OK +8000ms: 200 OK +12000ms: 200 OK +``` + +Narrow sweeps: + +```text +2000ms: timeout +2250ms: timeout +2500ms: timeout in one run, success in another +2750ms: success in one run +3000ms: success +3250ms: success +``` + +Low repeat: + +```text +1000ms: timeout +1500ms: timeout +1800ms: timeout +2000ms: timeout +2200ms: timeout +2400ms: timeout +``` + +HTTP conclusion: gateway HTTP actor requests become reliably usable around `3s` locally, with `5s` as the safer diagnostic delay. + +## WebSocket Ping/Pong Sweep + +The actor exposes `onWebSocket` ping/pong at: + +```text +/gateway/{actor}/websocket/ping +``` + +The client opens the WebSocket with Rivet gateway subprotocols: + +```text +rivet +rivet_encoding.bare +``` + +Then it sends: + +```json +{"type":"ping","sentAt":...} +``` + +And waits for: + +```json +{"type":"pong","sentAt":...} +``` + +Coarse sweep: + +```text +0ms: timeout +250ms: timeout +500ms: timeout +1000ms: timeout +2000ms: timeout +3000ms: pong +5000ms: pong +8000ms: pong +12000ms: pong +``` + +Narrow sweeps: + +```text +2000ms: timeout +2250ms: timeout +2500ms: timeout +2750ms: timeout in one run +3000ms: timeout in one run, success in another +3250ms: pong +``` + +Edge repeat: + +```text +2800ms: pong +3000ms: pong +3200ms: pong +3400ms: pong +3600ms: pong +``` + +Low repeat: + +```text +2200ms: timeout +2400ms: timeout +2600ms: timeout +2800ms: pong +``` + +WebSocket conclusion: raw actor WebSocket ping/pong has the same post-restart readiness window as HTTP gateway traffic. It is not action-specific. + +## Commit During Restart + +The harness can coordinate an actor action that starts a SQLite transaction, signals the driver, then attempts `COMMIT` after the engine is stopped. + +Immediate post-restart probes after this failed commit reproduced the same shape: + +- failed in-flight operation +- same existing key probes timed out +- new key succeeded + +Delayed post-restart probes passed. + +This is consistent with the gateway/serverless actor routing race, not durable SQLite session poisoning. + +## Important Corrections + +Earlier, it looked like a SQLite heartbeat caused later gateway probes to pass. That was wrong. A no-heartbeat delayed control also passed. + +The actual variable was delay after engine health, not SQLite activity. + +## Useful Logs + +Representative local logs: + +```text +/tmp/rivet-idle-none-immediate.log +/tmp/rivet-idle-kv-immediate.log +/tmp/rivet-idle-sqlite-immediate.log +/tmp/rivet-idle-none-after.log +/tmp/rivet-idle-kv-after.log +/tmp/rivet-idle-sqlite-after.log +/tmp/rivet-commit-none-immediate.log +/tmp/rivet-commit-none-after.log +/tmp/rivet-gateway-health-sweep.log +/tmp/rivet-gateway-health-sweep-narrow.log +/tmp/rivet-gateway-health-sweep-edge.log +/tmp/rivet-gateway-health-sweep-low.log +/tmp/rivet-gateway-websocket-sweep.log +/tmp/rivet-gateway-websocket-sweep-narrow.log +/tmp/rivet-gateway-websocket-sweep-edge.log +/tmp/rivet-gateway-websocket-sweep-low.log +``` + +## Commands + +Action/client repro: + +```bash +RIVETKIT_ENGINE_RESTART_MODE=idle \ +RIVETKIT_HEARTBEAT_MODE=none \ +RIVETKIT_POST_RESTART_PROBE_TIMING=immediate \ +node --import tsx tests/driver/engine-restart-serverless.ts +``` + +HTTP health sweep: + +```bash +RIVETKIT_ENGINE_RESTART_MODE=idle \ +RIVETKIT_HEARTBEAT_MODE=none \ +RIVETKIT_GATEWAY_HEALTH_DELAYS_MS=0,250,500,1000,2000,3000,5000,8000,12000 \ +node --import tsx tests/driver/engine-restart-serverless.ts +``` + +WebSocket ping/pong sweep: + +```bash +RIVETKIT_ENGINE_RESTART_MODE=idle \ +RIVETKIT_HEARTBEAT_MODE=none \ +RIVETKIT_GATEWAY_WEBSOCKET_DELAYS_MS=0,250,500,1000,2000,3000,5000,8000,12000 \ +node --import tsx tests/driver/engine-restart-serverless.ts +``` + +## Current Theory + +Engine `/health` returning does not mean gateway/serverless routing for previously warmed existing actors is fully settled. + +Requests in the first few seconds can attach to stale or incomplete actor routing/request state. Those requests hang until the caller timeout. A new key succeeds because it follows a fresh actor allocation path instead of the stale existing-key path. + +The likely fix area is the gateway/serverless/pegboard-envoy readiness and stale actor state path after engine restart, especially around existing actor generation, in-flight wake, stopped state, and pending request routing. diff --git a/rivetkit-typescript/packages/rivetkit/tests/driver/engine-restart-serverless.ts b/rivetkit-typescript/packages/rivetkit/tests/driver/engine-restart-serverless.ts new file mode 100644 index 0000000000..07b04c276a --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/tests/driver/engine-restart-serverless.ts @@ -0,0 +1,1327 @@ +import { type ChildProcess, spawn } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { getEnginePath } from "@rivetkit/engine-cli"; +import getPort from "get-port"; +import { createClient } from "../../src/client/mod"; + +const TEST_DIR = join(dirname(fileURLToPath(import.meta.url)), ".."); +const PACKAGE_DIR = dirname(TEST_DIR); +const REPO_ENGINE_BINARY = process.env.ENGINE_BINARY ?? join( + PACKAGE_DIR, + "../../../target/debug/rivet-engine", +); +const SERVERLESS_RUNTIME_PATH = join( + TEST_DIR, + "fixtures/engine-restart-serverless-runtime.ts", +); +const TOKEN = "dev"; +const HOST = "127.0.0.1"; +const ENGINE_START_TIMEOUT_MS = 90_000; +const SERVERLESS_START_TIMEOUT_MS = 30_000; +const INITIAL_COUNTER_READY_TIMEOUT_MS = 90_000; +const POST_RESTART_HEARTBEAT_OBSERVATION_MS = Number( + process.env.RIVETKIT_POST_RESTART_WAIT_MS ?? "12000", +); +const POST_RESTART_PROBE_TIMEOUT_MS = 20_000; +const GATEWAY_HEALTH_TIMEOUT_MS = Number( + process.env.RIVETKIT_GATEWAY_HEALTH_TIMEOUT_MS ?? "5000", +); +const COMMIT_FAILURE_TIMEOUT_MS = 45_000; +const RESTART_MODE = process.env.RIVETKIT_ENGINE_RESTART_MODE ?? "commit"; +const HEARTBEAT_MODE = process.env.RIVETKIT_HEARTBEAT_MODE ?? "sqlite"; +const POST_RESTART_PROBE_TIMING = + process.env.RIVETKIT_POST_RESTART_PROBE_TIMING ?? "after-heartbeat"; +const GATEWAY_HEALTH_DELAYS_MS = parseDelayList( + process.env.RIVETKIT_GATEWAY_HEALTH_DELAYS_MS, +); +const GATEWAY_WEBSOCKET_DELAYS_MS = parseDelayList( + process.env.RIVETKIT_GATEWAY_WEBSOCKET_DELAYS_MS, +); + +if (!["none", "sqlite", "kv"].includes(HEARTBEAT_MODE)) { + throw new Error("RIVETKIT_HEARTBEAT_MODE must be one of: none, sqlite, kv"); +} +if (!["immediate", "after-heartbeat"].includes(POST_RESTART_PROBE_TIMING)) { + throw new Error( + "RIVETKIT_POST_RESTART_PROBE_TIMING must be one of: immediate, after-heartbeat", + ); +} + +interface RuntimeLogs { + stdout: string; + stderr: string; +} + +interface HeartbeatStats { + ticks: number; + sqlOk: number; + sqlErr: number; + kvOk: number; + kvErr: number; + onWake: number; + onSleep: number; + abort: number; + rollbackErr: number; + lastOkCount: number | undefined; + lastError: string | undefined; +} + +interface CommitSignalServer { + url: string; + waitForSignal: Promise; + close: () => Promise; +} + +interface GatewayHealthTarget { + delayMs: number; + key: string; + url: string; +} + +interface GatewayWebSocketTarget { + delayMs: number; + key: string; + url: string; +} + +class OwnedEngine { + readonly dbRoot = mkdtempSync(join(tmpdir(), "rivetkit-engine-restart-")); + readonly endpoint: string; + readonly peerUrl: string; + readonly configPath = join(this.dbRoot, "config.json"); + readonly #guardPort: number; + readonly #apiPeerPort: number; + readonly #metricsPort: number; + #child: ChildProcess | undefined; + #logs: RuntimeLogs = { stdout: "", stderr: "" }; + + private constructor( + guardPort: number, + apiPeerPort: number, + metricsPort: number, + ) { + this.#guardPort = guardPort; + this.#apiPeerPort = apiPeerPort; + this.#metricsPort = metricsPort; + this.endpoint = `http://${HOST}:${guardPort}`; + this.peerUrl = `http://${HOST}:${apiPeerPort}`; + this.#writeConfig(); + } + + static async start(): Promise { + const guardPort = await getPort({ host: HOST }); + const apiPeerPort = await getPort({ host: HOST, exclude: [guardPort] }); + const metricsPort = await getPort({ + host: HOST, + exclude: [guardPort, apiPeerPort], + }); + const engine = new OwnedEngine(guardPort, apiPeerPort, metricsPort); + await engine.startProcess(); + return engine; + } + + async startProcess(): Promise { + if (isProcessRunning(this.#child)) { + return; + } + + this.#logs = { stdout: "", stderr: "" }; + const child = spawn( + resolveEngineBinaryPath(), + ["start", "--config", this.configPath], + { + env: { + ...process.env, + RIVET__GUARD__HOST: HOST, + RIVET__GUARD__PORT: this.#guardPort.toString(), + RIVET__API_PEER__HOST: HOST, + RIVET__API_PEER__PORT: this.#apiPeerPort.toString(), + RIVET__METRICS__HOST: HOST, + RIVET__METRICS__PORT: this.#metricsPort.toString(), + RIVET__FILE_SYSTEM__PATH: join(this.dbRoot, "db"), + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + this.#child = child; + child.stdout?.on("data", (chunk) => { + const text = chunk.toString(); + this.#logs.stdout += text; + if (process.env.DRIVER_ENGINE_LOGS === "1") { + process.stderr.write(`[RESTART_ENG.OUT] ${text}`); + } + }); + child.stderr?.on("data", (chunk) => { + const text = chunk.toString(); + this.#logs.stderr += text; + if (process.env.DRIVER_ENGINE_LOGS === "1") { + process.stderr.write(`[RESTART_ENG.ERR] ${text}`); + } + }); + await waitForEngineHealth( + child, + this.#logs, + this.endpoint, + ENGINE_START_TIMEOUT_MS, + ); + console.log(`engine listening at ${this.endpoint} (${this.dbRoot})`); + } + + async stopProcess(signal: NodeJS.Signals = "SIGTERM"): Promise { + const child = this.#child; + if (!isProcessRunning(child)) { + return; + } + + await stopChildProcess(child, signal); + } + + async cleanup(): Promise { + await this.stopProcess(); + rmSync(this.dbRoot, { force: true, recursive: true }); + } + + #writeConfig(): void { + writeFileSync( + this.configPath, + JSON.stringify({ + topology: { + datacenter_label: 1, + datacenters: { + default: { + datacenter_label: 1, + is_leader: true, + public_url: this.endpoint, + peer_url: this.peerUrl, + }, + }, + }, + }), + ); + } +} + +async function stopChildProcess( + child: ChildProcess, + signal: NodeJS.Signals = "SIGTERM", +): Promise { + if (!isProcessRunning(child)) { + return; + } + + await new Promise((resolve) => { + let forceKill: NodeJS.Timeout | undefined; + const finish = () => { + if (forceKill) { + clearTimeout(forceKill); + } + resolve(); + }; + child.once("exit", finish); + child.kill(signal); + forceKill = setTimeout(() => { + if (isProcessRunning(child)) { + child.kill("SIGKILL"); + } + }, 5_000); + }); +} + +class ServerlessRuntime { + readonly url: string; + readonly #port: number; + #child: ChildProcess | undefined; + #logs: RuntimeLogs = { stdout: "", stderr: "" }; + + private constructor(port: number) { + this.#port = port; + this.url = `http://${HOST}:${port}/api/rivet`; + } + + static async start(input: { + endpoint: string; + namespace: string; + poolName: string; + }): Promise { + const port = await getPort({ host: HOST }); + const runtime = new ServerlessRuntime(port); + await runtime.startProcess(input); + return runtime; + } + + getOutput(): string { + return childOutput(this.#logs); + } + + async startProcess(input: { + endpoint: string; + namespace: string; + poolName: string; + }): Promise { + const child = spawn( + process.execPath, + ["--import", "tsx", SERVERLESS_RUNTIME_PATH], + { + cwd: PACKAGE_DIR, + env: { + ...process.env, + RIVET_TOKEN: TOKEN, + RIVET_NAMESPACE: input.namespace, + RIVETKIT_TEST_ENDPOINT: input.endpoint, + RIVETKIT_HEARTBEAT_MODE: HEARTBEAT_MODE, + RIVETKIT_TEST_HOST: HOST, + RIVETKIT_TEST_POOL_NAME: input.poolName, + RIVETKIT_TEST_PORT: this.#port.toString(), + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + this.#child = child; + child.stdout?.on("data", (chunk) => { + const text = chunk.toString(); + this.#logs.stdout += text; + if (process.env.DRIVER_RUNTIME_LOGS === "1") { + process.stderr.write(`[SERVERLESS_RT.OUT] ${text}`); + } + }); + child.stderr?.on("data", (chunk) => { + const text = chunk.toString(); + this.#logs.stderr += text; + if (process.env.DRIVER_RUNTIME_LOGS === "1") { + process.stderr.write(`[SERVERLESS_RT.ERR] ${text}`); + } + }); + await waitForHttpOk({ + url: `${this.url}/health`, + child, + logs: this.#logs, + timeoutMs: SERVERLESS_START_TIMEOUT_MS, + }); + console.log(`serverless runtime listening at ${this.url}`); + } + + async cleanup(): Promise { + const child = this.#child; + if (!isProcessRunning(child)) { + return; + } + + await stopChildProcess(child, "SIGTERM"); + } +} + +async function main() { + const namespace = `restart-${crypto.randomUUID()}`; + const poolName = `serverless-restart-${crypto.randomUUID()}`; + const actorKey = `sqlite-counter-${crypto.randomUUID()}`; + const engine = await OwnedEngine.start(); + let runtime: ServerlessRuntime | undefined; + let client: ReturnType | undefined; + let signalServer: CommitSignalServer | undefined; + + try { + await createNamespace(engine.endpoint, namespace); + runtime = await ServerlessRuntime.start({ + endpoint: engine.endpoint, + namespace, + poolName, + }); + await upsertServerlessRunnerConfig({ + endpoint: engine.endpoint, + namespace, + poolName, + serverlessUrl: runtime.url, + }); + await waitForRunnerConfigReady({ + endpoint: engine.endpoint, + namespace, + poolName, + }); + + client = createClient({ + endpoint: engine.endpoint, + namespace, + poolName, + token: TOKEN, + encoding: "bare", + disableMetadataLookup: true, + }); + + const actorHandle = client.sqliteCounter.getOrCreate([actorKey]); + const heartbeatWarmupStartedAt = Date.now(); + const countBeforeRestart = (await actorHandle.getCount()) as number; + console.log( + `restart scenario configured. restartMode=${RESTART_MODE} heartbeatMode=${HEARTBEAT_MODE} postRestartProbeTiming=${POST_RESTART_PROBE_TIMING}`, + ); + if (HEARTBEAT_MODE !== "none") { + await waitFor( + () => + heartbeatSuccessCount( + getHeartbeatStats( + runtime?.getOutput() ?? "", + heartbeatWarmupStartedAt, + ), + ) >= 2, + INITIAL_COUNTER_READY_TIMEOUT_MS, + () => { + const stats = getHeartbeatStats( + runtime?.getOutput() ?? "", + heartbeatWarmupStartedAt, + ); + return `actor heartbeat did not start. ${formatHeartbeatStats(stats)}`; + }, + ); + } + const warmupStats = getHeartbeatStats( + runtime.getOutput(), + heartbeatWarmupStartedAt, + ); + console.log( + `actor-originated heartbeat warmup finished. before=${countBeforeRestart} ${formatHeartbeatStats(warmupStats)}`, + ); + const gatewayHealthTargets = + GATEWAY_HEALTH_DELAYS_MS.length > 0 + ? await prepareGatewayHealthTargets({ + client, + baseKey: actorKey, + delaysMs: GATEWAY_HEALTH_DELAYS_MS, + }) + : []; + const gatewayWebSocketTargets = + GATEWAY_WEBSOCKET_DELAYS_MS.length > 0 + ? await prepareGatewayWebSocketTargets({ + client, + baseKey: actorKey, + delaysMs: GATEWAY_WEBSOCKET_DELAYS_MS, + }) + : []; + if (RESTART_MODE === "idle") { + console.log( + `restarting engine while actor is idle at count=${countBeforeRestart}`, + ); + await sleep(1_000); + const restartStartedAt = Date.now(); + await engine.stopProcess("SIGTERM"); + console.log("restarting engine"); + await engine.startProcess(); + const engineRestartedAt = Date.now(); + + await runPostRestartSequence({ + runtime, + client, + actorHandle, + actorKey, + countBeforeRestart, + mode: "idle", + restartStartedAt, + engineRestartedAt, + gatewayHealthTargets, + gatewayWebSocketTargets, + }); + return; + } + + if (RESTART_MODE !== "commit") { + throw new Error( + `unsupported RIVETKIT_ENGINE_RESTART_MODE: ${RESTART_MODE}`, + ); + } + + console.log( + `starting coordinated commit failure at count=${countBeforeRestart}`, + ); + signalServer = await createCommitSignalServer(); + const commitAttemptStartedAt = Date.now(); + const commitAttempt = withTimeout( + actorHandle.commitDuringEngineRestart({ + signalUrl: signalServer.url, + delayBeforeCommitMs: 500, + payloadBytes: 8192, + }) as Promise, + COMMIT_FAILURE_TIMEOUT_MS, + "commit attempt did not finish after engine restart signal", + ).then( + (value) => ({ ok: true as const, value }), + (error) => ({ ok: false as const, error }), + ); + + await signalServer.waitForSignal; + console.log("actor reached pre-commit point; sending engine SIGTERM"); + const restartStartedAt = Date.now(); + await engine.stopProcess("SIGTERM"); + + const commitResult = await commitAttempt; + if (commitResult.ok) { + console.warn( + `commit unexpectedly succeeded after engine shutdown in ${Date.now() - commitAttemptStartedAt}ms`, + ); + } else { + console.log( + `commit failed after engine shutdown in ${Date.now() - commitAttemptStartedAt}ms: ${stringifyError(commitResult.error)}`, + ); + } + + console.log("restarting engine"); + await engine.startProcess(); + const engineRestartedAt = Date.now(); + + await runPostRestartSequence({ + runtime, + client, + actorHandle, + actorKey, + countBeforeRestart, + mode: "commit", + restartStartedAt, + engineRestartedAt, + gatewayHealthTargets, + gatewayWebSocketTargets, + }); + } finally { + await signalServer?.close(); + await client?.dispose(); + await runtime?.cleanup(); + await engine.cleanup(); + } +} + +function resolveEngineBinaryPath(): string { + if (existsSync(REPO_ENGINE_BINARY)) { + return REPO_ENGINE_BINARY; + } + + return getEnginePath(); +} + +function childOutput(logs: RuntimeLogs): string { + return [logs.stdout, logs.stderr].filter(Boolean).join("\n"); +} + +function isProcessRunning( + child: ChildProcess | undefined, +): child is ChildProcess { + return ( + child !== undefined && + child.exitCode === null && + child.signalCode === null + ); +} + +async function waitForEngineHealth( + child: ChildProcess, + logs: RuntimeLogs, + endpoint: string, + timeoutMs: number, +): Promise { + await waitForHttpOk({ + child, + logs, + timeoutMs, + url: `${endpoint}/health`, + }); +} + +async function waitForHttpOk(input: { + child?: ChildProcess; + logs: RuntimeLogs; + timeoutMs: number; + url: string; +}): Promise { + const deadline = Date.now() + input.timeoutMs; + + while (Date.now() < deadline) { + if (input.child && !isProcessRunning(input.child)) { + throw new Error( + `process exited before health check passed:\n${childOutput(input.logs)}`, + ); + } + + try { + const response = await fetch(input.url); + if (response.ok) { + return; + } + } catch { } + + await sleep(500); + } + + throw new Error( + `timed out waiting for health at ${input.url}:\n${childOutput(input.logs)}`, + ); +} + +async function createNamespace( + endpoint: string, + namespace: string, +): Promise { + const response = await fetch(`${endpoint}/namespaces`, { + method: "POST", + headers: { + Authorization: `Bearer ${TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + name: namespace, + display_name: `Engine restart ${namespace}`, + }), + }); + + if (!response.ok) { + throw new Error( + `failed to create namespace ${namespace}: ${response.status} ${await response.text()}`, + ); + } +} + +async function getDatacenter( + endpoint: string, + namespace: string, +): Promise { + const response = await fetch( + `${endpoint}/datacenters?namespace=${encodeURIComponent(namespace)}`, + { + headers: { + Authorization: `Bearer ${TOKEN}`, + }, + }, + ); + + if (!response.ok) { + throw new Error( + `failed to list datacenters: ${response.status} ${await response.text()}`, + ); + } + + const body = (await response.json()) as { + datacenters: Array<{ name: string }>; + }; + const datacenter = body.datacenters[0]?.name; + if (!datacenter) { + throw new Error("engine returned no datacenters"); + } + + return datacenter; +} + +async function upsertServerlessRunnerConfig(input: { + endpoint: string; + namespace: string; + poolName: string; + serverlessUrl: string; +}): Promise { + const datacenter = await getDatacenter(input.endpoint, input.namespace); + const response = await fetch( + `${input.endpoint}/runner-configs/${encodeURIComponent(input.poolName)}?namespace=${encodeURIComponent(input.namespace)}`, + { + method: "PUT", + headers: { + Authorization: `Bearer ${TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + datacenters: { + [datacenter]: { + serverless: { + url: input.serverlessUrl, + headers: { + "x-rivet-token": TOKEN, + }, + request_lifespan: 3600, + drain_grace_period: 5, + metadata_poll_interval: 1000, + max_runners: 10, + min_runners: 0, + runners_margin: 0, + slots_per_runner: 1, + }, + metadata: {}, + drain_on_version_upgrade: true, + }, + }, + }), + }, + ); + + if (!response.ok) { + throw new Error( + `failed to upsert serverless runner config: ${response.status} ${await response.text()}`, + ); + } +} + +async function waitForRunnerConfigReady(input: { + endpoint: string; + namespace: string; + poolName: string; +}): Promise { + const deadline = Date.now() + 30_000; + let bodyText = ""; + + while (Date.now() < deadline) { + const response = await fetch( + `${input.endpoint}/runner-configs?namespace=${encodeURIComponent(input.namespace)}&runner_name=${encodeURIComponent(input.poolName)}`, + { + headers: { + Authorization: `Bearer ${TOKEN}`, + }, + }, + ); + bodyText = await response.text(); + if (response.ok) { + const body = JSON.parse(bodyText) as { + runner_configs?: Record< + string, + { + datacenters?: Record< + string, + { + protocol_version?: number | null; + } + >; + } + >; + }; + const config = body.runner_configs?.[input.poolName]; + const datacenters = Object.values(config?.datacenters ?? {}); + if ( + datacenters.length > 0 && + datacenters.every( + (datacenter) => datacenter.protocol_version != null, + ) + ) { + return; + } + } + + await sleep(250); + } + + throw new Error(`serverless runner config was not ready: ${bodyText}`); +} + +async function createCommitSignalServer(): Promise { + const port = await getPort({ host: HOST }); + let resolveSignal!: () => void; + const waitForSignal = new Promise((resolve) => { + resolveSignal = resolve; + }); + const server: Server = createServer((request, response) => { + if (request.method === "POST" && request.url === "/before-commit") { + resolveSignal(); + response.writeHead(204); + response.end(); + return; + } + + response.writeHead(404); + response.end(); + }); + + await new Promise((resolve) => { + server.listen(port, HOST, resolve); + }); + + return { + url: `http://${HOST}:${port}/before-commit`, + waitForSignal, + close: () => + new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }), + }; +} + +async function probeActor( + name: string, + operation: () => Promise, +): Promise<{ + name: string; + elapsedMs: number; + result: unknown; +}> { + const startedAt = Date.now(); + const result = await withTimeout( + operation(), + POST_RESTART_PROBE_TIMEOUT_MS, + `${name} did not complete after engine restart`, + ); + + return { + name, + elapsedMs: Date.now() - startedAt, + result, + }; +} + +async function prepareGatewayHealthTargets(input: { + client: ReturnType; + baseKey: string; + delaysMs: number[]; +}): Promise { + const targets: GatewayHealthTarget[] = []; + for (const delayMs of input.delaysMs) { + const key = `${input.baseKey}-gateway-health-${delayMs}`; + const handle = input.client.sqliteCounter.getOrCreate([key]); + await handle.getCount(); + const url = buildActorRequestUrl(await handle.getGatewayUrl(), "health"); + const response = await fetch(url, { + signal: AbortSignal.timeout(GATEWAY_HEALTH_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error( + `gateway health preflight failed for delay ${delayMs}: ${response.status} ${await response.text()}`, + ); + } + targets.push({ + delayMs, + key, + url, + }); + } + + console.log( + `gateway health targets warmed. delaysMs=${input.delaysMs.join(",")}`, + ); + return targets; +} + +async function runGatewayHealthDelaySweep(input: { + targets: GatewayHealthTarget[]; + engineRestartedAt: number; + mode: string; +}): Promise { + const startedAt = Date.now(); + console.log( + `gateway health delay sweep starting. mode=${input.mode} delaysMs=${input.targets.map((target) => target.delayMs).join(",")}`, + ); + + const results = await Promise.all( + input.targets.map(async (target) => { + const sleepMs = Math.max(0, input.engineRestartedAt + target.delayMs - Date.now()); + if (sleepMs > 0) { + await sleep(sleepMs); + } + + const probeStartedAt = Date.now(); + try { + const response = await fetch(target.url, { + signal: AbortSignal.timeout(GATEWAY_HEALTH_TIMEOUT_MS), + }); + const body = await response.text(); + return { + ...target, + ok: response.ok, + status: response.status, + elapsedMs: Date.now() - probeStartedAt, + startOffsetMs: probeStartedAt - input.engineRestartedAt, + body, + }; + } catch (error) { + return { + ...target, + ok: false, + status: "error", + elapsedMs: Date.now() - probeStartedAt, + startOffsetMs: probeStartedAt - input.engineRestartedAt, + body: stringifyError(error), + }; + } + }), + ); + + for (const result of results.sort((a, b) => a.delayMs - b.delayMs)) { + const body = result.body.length > 240 ? `${result.body.slice(0, 240)}...` : result.body; + console.log( + `gateway-health delayMs=${result.delayMs} startOffsetMs=${result.startOffsetMs} ok=${result.ok} status=${result.status} elapsedMs=${result.elapsedMs} key=${result.key} body=${JSON.stringify(body)}`, + ); + } + + const firstOk = results + .filter((result) => result.ok) + .sort((a, b) => a.delayMs - b.delayMs)[0]; + if (firstOk) { + console.log( + `gateway health first success. mode=${input.mode} delayMs=${firstOk.delayMs} totalSweepMs=${Date.now() - startedAt}`, + ); + } else { + console.log( + `gateway health no successes. mode=${input.mode} totalSweepMs=${Date.now() - startedAt}`, + ); + } +} + +function buildActorRequestUrl(gatewayUrl: string, path: string): string { + const url = new URL(gatewayUrl); + const normalizedPath = path.replace(/^\/+/, ""); + url.pathname = `${url.pathname.replace(/\/$/, "")}/request/${normalizedPath}`; + return url.toString(); +} + +async function prepareGatewayWebSocketTargets(input: { + client: ReturnType; + baseKey: string; + delaysMs: number[]; +}): Promise { + const targets: GatewayWebSocketTarget[] = []; + for (const delayMs of input.delaysMs) { + const key = `${input.baseKey}-gateway-ws-${delayMs}`; + const handle = input.client.sqliteCounter.getOrCreate([key]); + await handle.getCount(); + const url = buildActorWebSocketUrl(await handle.getGatewayUrl(), "ping"); + await openWebSocketPingPong(url); + targets.push({ + delayMs, + key, + url, + }); + } + + console.log( + `gateway websocket targets warmed. delaysMs=${input.delaysMs.join(",")}`, + ); + return targets; +} + +async function runGatewayWebSocketDelaySweep(input: { + targets: GatewayWebSocketTarget[]; + engineRestartedAt: number; + mode: string; +}): Promise { + const startedAt = Date.now(); + console.log( + `gateway websocket delay sweep starting. mode=${input.mode} delaysMs=${input.targets.map((target) => target.delayMs).join(",")}`, + ); + + const results = await Promise.all( + input.targets.map(async (target) => { + const sleepMs = Math.max(0, input.engineRestartedAt + target.delayMs - Date.now()); + if (sleepMs > 0) { + await sleep(sleepMs); + } + + const probeStartedAt = Date.now(); + try { + const message = await openWebSocketPingPong(target.url); + return { + ...target, + ok: true, + elapsedMs: Date.now() - probeStartedAt, + startOffsetMs: probeStartedAt - input.engineRestartedAt, + body: message, + }; + } catch (error) { + return { + ...target, + ok: false, + elapsedMs: Date.now() - probeStartedAt, + startOffsetMs: probeStartedAt - input.engineRestartedAt, + body: stringifyError(error), + }; + } + }), + ); + + for (const result of results.sort((a, b) => a.delayMs - b.delayMs)) { + const body = result.body.length > 240 ? `${result.body.slice(0, 240)}...` : result.body; + console.log( + `gateway-websocket delayMs=${result.delayMs} startOffsetMs=${result.startOffsetMs} ok=${result.ok} elapsedMs=${result.elapsedMs} key=${result.key} body=${JSON.stringify(body)}`, + ); + } + + const firstOk = results + .filter((result) => result.ok) + .sort((a, b) => a.delayMs - b.delayMs)[0]; + if (firstOk) { + console.log( + `gateway websocket first success. mode=${input.mode} delayMs=${firstOk.delayMs} totalSweepMs=${Date.now() - startedAt}`, + ); + } else { + console.log( + `gateway websocket no successes. mode=${input.mode} totalSweepMs=${Date.now() - startedAt}`, + ); + } +} + +function buildActorWebSocketUrl(gatewayUrl: string, path: string): string { + const url = new URL(gatewayUrl); + const normalizedPath = path.replace(/^\/+/, ""); + url.pathname = `${url.pathname.replace(/\/$/, "")}/websocket/${normalizedPath}`; + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + return url.toString(); +} + +function openWebSocketPingPong(url: string): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const websocket = new WebSocket(url, ["rivet", "rivet_encoding.bare"]); + const timeout = setTimeout(() => { + finish(new Error("websocket ping/pong timed out")); + }, GATEWAY_HEALTH_TIMEOUT_MS); + + const finish = (result: string | Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + try { + websocket.close(); + } catch { } + if (result instanceof Error) { + reject(result); + } else { + resolve(result); + } + }; + + websocket.addEventListener("open", () => { + websocket.send(JSON.stringify({ type: "ping", sentAt: Date.now() })); + }); + websocket.addEventListener("message", (event) => { + const data = typeof event.data === "string" ? event.data : String(event.data); + try { + const message = JSON.parse(data) as { type?: string }; + if (message.type === "pong") { + finish(data); + } + } catch { + finish(new Error(`invalid websocket message: ${data}`)); + } + }); + websocket.addEventListener("error", () => { + finish(new Error("websocket error")); + }); + websocket.addEventListener("close", (event) => { + if (!settled) { + finish( + new Error( + `websocket closed before pong: code=${event.code} reason=${event.reason}`, + ), + ); + } + }); + }); +} + +async function runPostRestartSequence(input: { + runtime: ServerlessRuntime; + client: ReturnType; + actorHandle: ReturnType< + ReturnType["sqliteCounter"]["getOrCreate"] + >; + actorKey: string; + countBeforeRestart: number; + mode: string; + restartStartedAt: number; + engineRestartedAt: number; + gatewayHealthTargets: GatewayHealthTarget[]; + gatewayWebSocketTargets: GatewayWebSocketTarget[]; +}): Promise { + if (input.gatewayWebSocketTargets.length > 0) { + await runGatewayWebSocketDelaySweep({ + targets: input.gatewayWebSocketTargets, + engineRestartedAt: input.engineRestartedAt, + mode: input.mode, + }); + return; + } + + if (input.gatewayHealthTargets.length > 0) { + await runGatewayHealthDelaySweep({ + targets: input.gatewayHealthTargets, + engineRestartedAt: input.engineRestartedAt, + mode: input.mode, + }); + return; + } + + if (POST_RESTART_PROBE_TIMING === "immediate") { + await runPostRestartProbes(input); + await observePostRestartHeartbeat(input); + return; + } + + await observePostRestartHeartbeat(input); + await runPostRestartProbes(input); +} + +async function runPostRestartProbes(input: { + client: ReturnType; + actorHandle: ReturnType< + ReturnType["sqliteCounter"]["getOrCreate"] + >; + actorKey: string; + countBeforeRestart: number; + mode: string; +}): Promise { + const probeResults = await Promise.allSettled([ + probeActor("same-handle-getCount", () => input.actorHandle.getCount()), + probeActor("same-handle-tick", () => input.actorHandle.tick(8192)), + probeActor("fresh-handle-getCount", () => + input.client.sqliteCounter.getOrCreate([input.actorKey]).getCount(), + ), + probeActor("new-key-tick", () => + input.client.sqliteCounter + .getOrCreate([`post-restart-${crypto.randomUUID()}`]) + .tick(8192), + ), + ]); + let postRestartProbeFailures = 0; + for (const probeResult of probeResults) { + if (probeResult.status === "fulfilled") { + console.log( + `${probeResult.value.name} post-restart probe ok elapsedMs=${probeResult.value.elapsedMs} result=${JSON.stringify(probeResult.value.result)}`, + ); + } else { + postRestartProbeFailures += 1; + console.warn( + `post-restart probe failed: ${stringifyError(probeResult.reason)}`, + ); + } + } + + if (postRestartProbeFailures > 0) { + console.log( + `bricked actor symptom reproduced. mode=${input.mode} failedPostRestartProbes=${postRestartProbeFailures} before=${input.countBeforeRestart}`, + ); + } else { + console.log( + `serverless restart scenario passed without bricking. mode=${input.mode} before=${input.countBeforeRestart}`, + ); + } +} + +async function observePostRestartHeartbeat(input: { + runtime: ServerlessRuntime; + mode: string; + restartStartedAt: number; + engineRestartedAt: number; +}): Promise { + await sleep(POST_RESTART_HEARTBEAT_OBSERVATION_MS); + + const output = input.runtime.getOutput(); + const duringRestart = getHeartbeatStats(output, input.restartStartedAt); + const afterEngineRestarted = getHeartbeatStats(output, input.engineRestartedAt); + + console.log( + `heartbeat observation since restart signal. mode=${input.mode} ${formatHeartbeatStats(duringRestart)}`, + ); + console.log( + `heartbeat observation after engine healthy. mode=${input.mode} ${formatHeartbeatStats(afterEngineRestarted)}`, + ); + + if (HEARTBEAT_MODE === "none") { + console.log( + `no actor-originated heartbeat configured after engine restart. mode=${input.mode}`, + ); + } else if (heartbeatSuccessCount(afterEngineRestarted) > 0) { + console.log( + `actor-originated ${HEARTBEAT_MODE} survived engine restart. mode=${input.mode}`, + ); + } else if (heartbeatErrorCount(afterEngineRestarted) > 0) { + console.log( + `actor-originated ${HEARTBEAT_MODE} is failing after engine restart. mode=${input.mode} lastError=${afterEngineRestarted.lastError}`, + ); + } else if (duringRestart.onSleep > 0 || duringRestart.abort > 0) { + console.log( + `actor heartbeat stopped because actor shutdown ran during restart. mode=${input.mode}`, + ); + } else { + console.log( + `actor heartbeat produced no post-restart ${HEARTBEAT_MODE} signal. mode=${input.mode}`, + ); + } +} + +function getHeartbeatStats(output: string, sinceTs: number): HeartbeatStats { + const stats: HeartbeatStats = { + ticks: 0, + sqlOk: 0, + sqlErr: 0, + kvOk: 0, + kvErr: 0, + onWake: 0, + onSleep: 0, + abort: 0, + rollbackErr: 0, + lastOkCount: undefined, + lastError: undefined, + }; + + for (const line of output.split(/\r?\n/)) { + if (!line.startsWith("{")) { + continue; + } + + let event: { + event?: string; + ts?: number; + count?: number; + error?: string; + }; + try { + event = JSON.parse(line); + } catch { + continue; + } + + if (!event.event || typeof event.ts !== "number" || event.ts < sinceTs) { + continue; + } + + switch (event.event) { + case "heartbeat_tick": + stats.ticks += 1; + break; + case "heartbeat_sql_ok": + stats.sqlOk += 1; + if (typeof event.count === "number") { + stats.lastOkCount = event.count; + } + break; + case "heartbeat_sql_err": + stats.sqlErr += 1; + stats.lastError = event.error; + break; + case "heartbeat_kv_ok": + stats.kvOk += 1; + if (typeof event.count === "number") { + stats.lastOkCount = event.count; + } + break; + case "heartbeat_kv_err": + stats.kvErr += 1; + stats.lastError = event.error; + break; + case "heartbeat_on_wake": + stats.onWake += 1; + break; + case "heartbeat_on_sleep": + stats.onSleep += 1; + break; + case "heartbeat_abort": + stats.abort += 1; + break; + case "heartbeat_rollback_err": + stats.rollbackErr += 1; + stats.lastError = event.error; + break; + default: + break; + } + } + + return stats; +} + +function formatHeartbeatStats(stats: HeartbeatStats): string { + return [ + `ticks=${stats.ticks}`, + `sqlOk=${stats.sqlOk}`, + `sqlErr=${stats.sqlErr}`, + `kvOk=${stats.kvOk}`, + `kvErr=${stats.kvErr}`, + `onWake=${stats.onWake}`, + `onSleep=${stats.onSleep}`, + `abort=${stats.abort}`, + `rollbackErr=${stats.rollbackErr}`, + `lastOkCount=${stats.lastOkCount ?? "none"}`, + `lastError=${stats.lastError ?? "none"}`, + ].join(" "); +} + +function heartbeatSuccessCount(stats: HeartbeatStats): number { + if (HEARTBEAT_MODE === "sqlite") { + return stats.sqlOk; + } + if (HEARTBEAT_MODE === "kv") { + return stats.kvOk; + } + return 0; +} + +function heartbeatErrorCount(stats: HeartbeatStats): number { + if (HEARTBEAT_MODE === "sqlite") { + return stats.sqlErr; + } + if (HEARTBEAT_MODE === "kv") { + return stats.kvErr; + } + return 0; +} + +function parseDelayList(value: string | undefined): number[] { + if (!value) { + return []; + } + + return value + .split(",") + .map((part) => Number(part.trim())) + .filter((delayMs) => Number.isFinite(delayMs) && delayMs >= 0) + .sort((a, b) => a - b); +} + +async function waitFor( + predicate: () => boolean, + timeoutMs: number, + makeError: () => string, +): Promise { + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + if (predicate()) { + return; + } + await sleep(250); + } + + throw new Error(makeError()); +} + +function stringifyError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function withTimeout( + promise: Promise, + timeoutMs: number, + message: string, +): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error(message)); + }, timeoutMs); + + promise.then( + (value) => { + clearTimeout(timeout); + resolve(value); + }, + (error) => { + clearTimeout(timeout); + reject(error); + }, + ); + }); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/rivetkit-typescript/packages/rivetkit/tests/fixtures/engine-restart-serverless-runtime.ts b/rivetkit-typescript/packages/rivetkit/tests/fixtures/engine-restart-serverless-runtime.ts new file mode 100644 index 0000000000..fa3e2a9372 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/tests/fixtures/engine-restart-serverless-runtime.ts @@ -0,0 +1,472 @@ +import { serve } from "@hono/node-server"; +import { actor, setup } from "rivetkit"; + +const host = process.env.RIVETKIT_TEST_HOST ?? "127.0.0.1"; +const port = Number(process.env.RIVETKIT_TEST_PORT); +const endpoint = process.env.RIVETKIT_TEST_ENDPOINT; +const namespace = process.env.RIVET_NAMESPACE; +const token = process.env.RIVET_TOKEN ?? "dev"; +const poolName = process.env.RIVETKIT_TEST_POOL_NAME; +const heartbeatMode = process.env.RIVETKIT_HEARTBEAT_MODE ?? "sqlite"; + +if (!Number.isInteger(port) || port <= 0) { + throw new Error("RIVETKIT_TEST_PORT must be a positive integer"); +} +if (!endpoint) { + throw new Error("RIVETKIT_TEST_ENDPOINT is required"); +} +if (!namespace) { + throw new Error("RIVET_NAMESPACE is required"); +} +if (!poolName) { + throw new Error("RIVETKIT_TEST_POOL_NAME is required"); +} +if (!["none", "sqlite", "kv"].includes(heartbeatMode)) { + throw new Error("RIVETKIT_HEARTBEAT_MODE must be one of: none, sqlite, kv"); +} + +interface SqliteDatabase { + run(sql: string, params?: unknown[]): Promise; + query( + sql: string, + params?: unknown[], + ): Promise<{ + rows: unknown[][]; + }>; +} + +interface HeartbeatVars { + heartbeatTimer?: ReturnType; + heartbeatInFlight?: boolean; + heartbeatSeq?: number; +} + +const rawSqlDatabaseProvider = { + createClient: async () => ({ + execute: async () => [], + close: async () => {}, + }), + onMigrate: async () => {}, +}; + +async function ensureTables(database: SqliteDatabase) { + await database.run(` + CREATE TABLE IF NOT EXISTS restart_counter ( + id INTEGER PRIMARY KEY CHECK (id = 1), + count INTEGER NOT NULL + ) + `); + await database.run(` + CREATE TABLE IF NOT EXISTS restart_counter_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + count INTEGER NOT NULL, + payload TEXT NOT NULL, + created_at INTEGER NOT NULL + ) + `); + await database.run(` + CREATE TABLE IF NOT EXISTS restart_heartbeat ( + id INTEGER PRIMARY KEY CHECK (id = 1), + count INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `); +} + +function stringifyError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function logRuntimeEvent(event: string, fields: Record) { + console.log( + JSON.stringify({ + event, + ts: Date.now(), + ...fields, + }), + ); +} + +async function runHeartbeatSql(database: SqliteDatabase): Promise { + await ensureTables(database); + await database.run("BEGIN"); + try { + await database.run( + ` + INSERT INTO restart_heartbeat (id, count, updated_at) + VALUES (1, 1, ?) + ON CONFLICT(id) DO UPDATE SET + count = count + 1, + updated_at = excluded.updated_at + `, + [Date.now()], + ); + await database.run("COMMIT"); + + const rows = await database.query( + "SELECT count FROM restart_heartbeat WHERE id = ?", + [1], + ); + return Number(rows.rows[0]?.[0] ?? 0); + } catch (error) { + try { + await database.run("ROLLBACK"); + } catch (rollbackError) { + logRuntimeEvent("heartbeat_rollback_err", { + error: stringifyError(rollbackError), + }); + } + throw error; + } +} + +async function runHeartbeatKv( + kv: { + get(key: string): Promise; + put(key: string, value: string): Promise; + }, +): Promise { + const current = Number((await kv.get("heartbeat_count")) ?? "0"); + const next = current + 1; + await kv.put("heartbeat_count", String(next)); + return next; +} + +const sqliteCounter = actor({ + state: {}, + vars: {} as HeartbeatVars, + db: rawSqlDatabaseProvider, + onRequest: (ctx, request) => { + const url = new URL(request.url); + if (url.pathname !== "/health") { + return new Response("not found", { status: 404 }); + } + + logRuntimeEvent("gateway_health_request", { + actorId: ctx.actorId, + key: ctx.key, + }); + return new Response( + JSON.stringify({ + ok: true, + actorId: ctx.actorId, + key: ctx.key, + }), + { + headers: { "Content-Type": "application/json" }, + }, + ); + }, + onWebSocket: (ctx, websocket) => { + logRuntimeEvent("gateway_websocket_open", { + actorId: ctx.actorId, + key: ctx.key, + path: ctx.request ? new URL(ctx.request.url).pathname : "unknown", + }); + websocket.addEventListener("message", (event: { data: unknown }) => { + const data = typeof event.data === "string" ? event.data : String(event.data); + try { + const message = JSON.parse(data) as { type?: string; sentAt?: number }; + if (message.type === "ping") { + websocket.send( + JSON.stringify({ + type: "pong", + sentAt: message.sentAt, + actorId: ctx.actorId, + key: ctx.key, + }), + ); + return; + } + } catch {} + + websocket.send(data); + }); + websocket.addEventListener("close", () => { + logRuntimeEvent("gateway_websocket_close", { + actorId: ctx.actorId, + key: ctx.key, + }); + }); + }, + onWake: async (ctx) => { + const vars = ctx.vars as HeartbeatVars; + if (vars.heartbeatTimer) { + return; + } + + const database = ctx.sql as SqliteDatabase; + vars.heartbeatSeq = 0; + logRuntimeEvent("heartbeat_on_wake", { + actorId: ctx.actorId, + key: ctx.key, + mode: heartbeatMode, + }); + if (heartbeatMode === "none") { + return; + } + + const tick = async () => { + if (ctx.abortSignal.aborted || vars.heartbeatInFlight) { + return; + } + + vars.heartbeatInFlight = true; + const seq = (vars.heartbeatSeq ?? 0) + 1; + vars.heartbeatSeq = seq; + logRuntimeEvent("heartbeat_tick", { + actorId: ctx.actorId, + key: ctx.key, + seq, + mode: heartbeatMode, + }); + + try { + if (heartbeatMode === "sqlite") { + const count = await runHeartbeatSql(database); + logRuntimeEvent("heartbeat_sql_ok", { + actorId: ctx.actorId, + key: ctx.key, + seq, + count, + }); + } else { + const count = await runHeartbeatKv(ctx.kv); + logRuntimeEvent("heartbeat_kv_ok", { + actorId: ctx.actorId, + key: ctx.key, + seq, + count, + }); + } + } catch (error) { + logRuntimeEvent( + heartbeatMode === "sqlite" ? "heartbeat_sql_err" : "heartbeat_kv_err", + { + actorId: ctx.actorId, + key: ctx.key, + seq, + error: stringifyError(error), + }, + ); + } finally { + vars.heartbeatInFlight = false; + } + }; + + vars.heartbeatTimer = setInterval(() => { + void tick(); + }, 1_000); + + ctx.abortSignal.addEventListener( + "abort", + () => { + if (vars.heartbeatTimer) { + clearInterval(vars.heartbeatTimer); + vars.heartbeatTimer = undefined; + } + logRuntimeEvent("heartbeat_abort", { + actorId: ctx.actorId, + key: ctx.key, + mode: heartbeatMode, + }); + }, + { once: true }, + ); + + await tick(); + }, + onSleep: (ctx) => { + const vars = ctx.vars as HeartbeatVars; + if (vars.heartbeatTimer) { + clearInterval(vars.heartbeatTimer); + vars.heartbeatTimer = undefined; + } + logRuntimeEvent("heartbeat_on_sleep", { + actorId: ctx.actorId, + key: ctx.key, + mode: heartbeatMode, + }); + }, + actions: { + tick: async (ctx, payloadBytes = 4096) => { + const database = ctx.sql as SqliteDatabase; + const payload = "x".repeat(Math.max(0, Math.trunc(payloadBytes))); + const now = Date.now(); + + await ensureTables(database); + await database.run("BEGIN"); + try { + await database.run( + ` + INSERT INTO restart_counter (id, count) + VALUES (1, 1) + ON CONFLICT(id) DO UPDATE SET count = count + 1 + `, + ); + const counterRows = await database.query( + "SELECT count FROM restart_counter WHERE id = ?", + [1], + ); + const count = Number(counterRows.rows[0]?.[0] ?? 0); + await database.run( + ` + INSERT INTO restart_counter_events (count, payload, created_at) + VALUES (?, ?, ?) + `, + [count, payload, now], + ); + await database.run( + ` + DELETE FROM restart_counter_events + WHERE id IN ( + SELECT id FROM restart_counter_events + ORDER BY id ASC + LIMIT max((SELECT COUNT(*) FROM restart_counter_events) - 200, 0) + ) + `, + ); + await database.run("COMMIT"); + + const eventRows = await database.query( + "SELECT COUNT(*) AS events FROM restart_counter_events", + ); + + return { + count, + events: Number(eventRows.rows[0]?.[0] ?? 0), + }; + } catch (error) { + await database.run("ROLLBACK"); + throw error; + } + }, + getCount: async (ctx) => { + const database = ctx.sql as SqliteDatabase; + await ensureTables(database); + const rows = await database.query( + "SELECT count FROM restart_counter WHERE id = ?", + [1], + ); + return Number(rows.rows[0]?.[0] ?? 0); + }, + commitDuringEngineRestart: async ( + ctx, + input: { + signalUrl: string; + delayBeforeCommitMs?: number; + payloadBytes?: number; + }, + ) => { + const database = ctx.sql as SqliteDatabase; + const payload = "x".repeat( + Math.max(0, Math.trunc(input.payloadBytes ?? 8192)), + ); + + await ensureTables(database); + await database.run("BEGIN"); + await database.run( + ` + INSERT INTO restart_counter (id, count) + VALUES (1, 1) + ON CONFLICT(id) DO UPDATE SET count = count + 1 + `, + ); + await database.run( + ` + INSERT INTO restart_counter_events (count, payload, created_at) + VALUES ((SELECT count FROM restart_counter WHERE id = 1), ?, ?) + `, + [payload, Date.now()], + ); + + await fetch(input.signalUrl, { method: "POST" }); + await sleep( + Math.max(0, Math.trunc(input.delayBeforeCommitMs ?? 500)), + ); + + const commitStartedAt = Date.now(); + try { + await database.run("COMMIT"); + return { + ok: true, + commitDurationMs: Date.now() - commitStartedAt, + }; + } catch (commitError) { + let rollbackErrorMessage: string | undefined; + try { + await database.run("ROLLBACK"); + } catch (rollbackError) { + rollbackErrorMessage = stringifyError(rollbackError); + } + + console.error( + JSON.stringify({ + event: "commitDuringEngineRestartFailed", + commitDurationMs: Date.now() - commitStartedAt, + commitError: stringifyError(commitError), + rollbackError: rollbackErrorMessage, + }), + ); + + throw new Error( + `commit failed: ${stringifyError(commitError)}; rollback failed: ${rollbackErrorMessage ?? "no"}`, + ); + } + }, + }, + options: { + sleepTimeout: 300_000, + }, +}); + +const registry = setup({ + use: { + sqliteCounter, + }, + runtime: "native", + sqlite: "remote", + endpoint, + namespace, + token, + envoy: { + poolName, + }, + serverless: { + basePath: "/api/rivet", + }, + noWelcome: true, + test: { + enabled: true, + sqliteBackend: "remote", + }, +}); + +const server = serve( + { + fetch: (request) => registry.handler(request), + hostname: host, + port, + }, + () => { + console.log( + JSON.stringify({ + event: "listening", + url: `http://${host}:${port}/api/rivet`, + }), + ); + }, +); + +function shutdown() { + server.close(() => { + process.exit(0); + }); +} + +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown);