From 0c04da2b522c9ade99459e0d9b8488a5363c8384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Veyssi=C3=A8re?= Date: Mon, 27 Jul 2026 16:43:45 -0500 Subject: [PATCH] =?UTF-8?q?v0.3.0=20=E2=80=94=20three=20verbs=20on=20Redis?= =?UTF-8?q?=20Streams?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the single-verb model (Pool + Scheduler + emit) with Moleculer's semantic surface carried on Redis Streams: call, dispatch and emit, with a message envelope, cascading deadlines and typed errors. Pool and Scheduler merge into one Service, because the moment a handler calls another action its process is both executant and caller — the split forced one domain to live in two files. Adds per-message ttl and delay, request correlation (rid/level/caller/meta), pydantic params validation, dispatch task records, and runtime switches that pause a service or registration without a deploy. Fixes two live 0.2 bugs found on the way: idle workers reconnected every few seconds because redis-py's default socket_timeout is exactly the runtime's XREADGROUP BLOCK window, and the connection pool was never sized to the topology. Also hardens the worker loop, which previously let any failure outside a handler's own guard kill the process. Breaking: the public API and the Redis keyspace both change, so 0.2 and 0.3 cannot interoperate. See CHANGELOG.md. 153 tests, 96% coverage, green on fakeredis and real Redis. Co-Authored-By: Claude --- .github/workflows/ci.yml | 32 +- .gitignore | 3 + CHANGELOG.md | 95 +++++ README.md | 220 +++++----- doc/concepts.md | 125 ++++++ doc/index.md | 56 +++ doc/internals.md | 229 +++++++++++ doc/limits.md | 122 ++++++ doc/messages.md | 179 ++++++++ doc/operations.md | 141 +++++++ examples/events/main.py | 70 ++++ examples/jobs/main.py | 78 ++++ examples/minimal/main.py | 33 +- examples/orders/pipeline.py | 37 +- examples/playwright/main.py | 17 +- examples/playwright/pipeline.py | 40 +- examples/poisson/main.py | 58 +-- examples/rpc/main.py | 93 +++++ pyproject.toml | 15 +- runtime/__init__.py | 64 ++- runtime/_version.py | 2 +- runtime/app.py | 415 +++++++++++++------ runtime/broker.py | 182 ++++++-- runtime/context.py | 383 +++++++++++++++-- runtime/duration.py | 54 +++ runtime/envelope.py | 143 +++++++ runtime/errors.py | 128 ++++++ runtime/node.py | 119 ++++++ runtime/pool.py | 155 ------- runtime/redis_io.py | 267 +++++++++--- runtime/scheduler.py | 162 -------- runtime/service.py | 530 ++++++++++++++++++++++++ runtime/switches.py | 128 ++++++ runtime/types.py | 15 +- tests/conftest.py | 34 +- tests/helpers.py | 26 ++ tests/test_api.py | 148 ------- tests/test_app.py | 366 +++++++++++++++++ tests/test_context.py | 39 -- tests/test_duration.py | 47 +++ tests/test_envelope.py | 132 ++++++ tests/test_errors.py | 66 +++ tests/test_example_minimal.py | 24 -- tests/test_example_orders.py | 34 -- tests/test_example_poisson.py | 32 -- tests/test_examples.py | 99 +++++ tests/test_redis_io.py | 347 +++++++++++++--- tests/test_scheduler.py | 206 ---------- tests/test_serve.py | 262 ------------ tests/test_service.py | 200 +++++++++ tests/test_verbs.py | 709 ++++++++++++++++++++++++++++++++ tests/test_worker.py | 480 ++++++++++++++++----- uv.lock | 198 +++++++++ 53 files changed, 6139 insertions(+), 1700 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 doc/concepts.md create mode 100644 doc/index.md create mode 100644 doc/internals.md create mode 100644 doc/limits.md create mode 100644 doc/messages.md create mode 100644 doc/operations.md create mode 100644 examples/events/main.py create mode 100644 examples/jobs/main.py create mode 100644 examples/rpc/main.py create mode 100644 runtime/duration.py create mode 100644 runtime/envelope.py create mode 100644 runtime/errors.py create mode 100644 runtime/node.py delete mode 100644 runtime/pool.py delete mode 100644 runtime/scheduler.py create mode 100644 runtime/service.py create mode 100644 runtime/switches.py delete mode 100644 tests/test_api.py create mode 100644 tests/test_app.py delete mode 100644 tests/test_context.py create mode 100644 tests/test_duration.py create mode 100644 tests/test_envelope.py create mode 100644 tests/test_errors.py delete mode 100644 tests/test_example_minimal.py delete mode 100644 tests/test_example_orders.py delete mode 100644 tests/test_example_poisson.py create mode 100644 tests/test_examples.py delete mode 100644 tests/test_scheduler.py delete mode 100644 tests/test_serve.py create mode 100644 tests/test_service.py create mode 100644 tests/test_verbs.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d5a389..d8753c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,20 @@ jobs: fail-fast: false matrix: python-version: ["3.12", "3.13", "3.14"] + # Both backends are exercised on every leg. They are not interchangeable: + # a real XREADGROUP BLOCK is woken by an arriving message and fakeredis's is + # not, and a real Redis is shared across the whole run while fakeredis is + # fresh per test. Bugs hide on each side of that difference. + services: + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 steps: - uses: actions/checkout@v6 - uses: astral-sh/setup-uv@v7 @@ -35,4 +49,20 @@ jobs: enable-cache: true python-version: ${{ matrix.python-version }} - run: uv sync --extra dev - - run: uv run python -m pytest + + - name: tests (fakeredis) + env: + RUNTIME_TEST_BACKEND: fake + run: uv run python -m pytest + + # Coverage is measured on this leg rather than the fakeredis one: a real + # Redis reaches strictly more paths (blocking reads that are genuinely + # woken, the timeout-to-cancellation conversion). The floor is a ratchet + # against a large untested addition, not a target — read the missing lines, + # which are mostly failure and configuration paths. + - name: tests (real Redis, with coverage) + env: + RUNTIME_TEST_BACKEND: real + run: >- + uv run python -m pytest + --cov=runtime --cov-report=term-missing --cov-fail-under=90 diff --git a/.gitignore b/.gitignore index 3074d11..0068eee 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ htmlcov/ context.md plan.md runtime_api.py +# Proposal for a metrics seam that does not exist yet; shipping it as +# documentation would describe an API nobody can call. +doc/observability.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..977d1f7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,95 @@ +# Changelog + +## 0.3.0 + +A clean break. The public API is replaced, and the Redis keyspace changes, so +0.2 and 0.3 processes **cannot** talk to each other and messages queued by 0.2 +are not read by 0.3. Drain before upgrading, or run the two on separate +`namespace`s. + +### Added + +**Three verbs instead of one.** `emit` used to be the only way to send anything. + +| | Cardinality | Reply | +|---|---|---| +| `await ctx.call(action, **params)` | exactly one executant | the return value, awaited | +| `await ctx.dispatch(action, **params)` | exactly one executant | a task id | +| `await ctx.emit(event, **params)` | every subscriber | none | + +- **Request/reply.** An action's return value crosses back to the caller; errors + are typed, serialized, and re-raised in the calling process. +- **Per-message `ttl` and `delay`.** A `ttl` becomes an absolute deadline that + travels with the message; a worker claiming an expired one drops it without + running the handler. A child `call`/`dispatch` inherits the remaining budget + and can only narrow it. +- **Correlation.** Every message carries a request id, call depth, provenance and + user `meta`, propagated across all three verbs, reachable as `ctx.request_id`, + `ctx.level`, `ctx.caller`, `ctx.meta`. +- **Params validation.** `@service.action(name, params=SomeModel)` validates + before the handler runs; the handler receives the model instance. +- **Task records.** `dispatch` returns an id; `ctx.task_status(id)` reports + `pending | done | failed | expired`. +- **Runtime switches.** Pause and resume a service, action, subscription or + producer without a deploy, from `runtime.switches` or `ctx.set_enabled`. + Pausing stops consumption; queued work drains on resume. +- **`App` knobs:** `default_ttl`, `max_depth`, `stream_maxlen`, `task_ttl`, + `claim_block`, `sweeper`, `switch_interval`. + +### Changed (breaking) + +| 0.2 | 0.3 | +|---|---| +| `Pool(...)` and `Scheduler(...)` | one `Service(...)` holding both | +| `@pool.flow(consumes=X)` | `@service.action(X)` — or `@service.event(X)` for fan-out | +| `ctx.emit(X, **p)` to reach one consumer | `ctx.dispatch(X, **p)` | +| `Event` type alias | `Params` | +| `Handler` | `ActionHandler` / `EventHandler` | +| `broker.stream_name(t)` | `broker.action_stream(n)` / `event_stream(n)` | +| stream `flow:{type}` | `act:{name}` (group `workers`) / `evt:{name}` (group per subscriber) | + +- **`ctx.emit` changed meaning** — from "deliver to the one consumer" to "fan out + to every subscriber". A site that should now be `dispatch` and is left as + `emit` writes to a stream with no subscribers, and the message is silently + lost. There is no registry, so this cannot be detected for you. Check every + `emit` call site when migrating. +- **`max_slots` is required** for any service that consumes, and refused at boot + otherwise. Producer-only services need none. +- **Every message is acked**, including one whose handler raised. With no + reclaim, leaving it un-acked meant lost anyway plus a pending entry nothing + would ever claim. +- **A new event subscriber starts at the tail**, not the head — deploying one no + longer replays the retained stream into it. +- **The `Broker` protocol grew** from 5 methods to cover replies, task records, + the delay set and the switch registry. Custom backends must implement them. +- Handlers keep their `(ctx, params)` shape; only the decorator name changes. +- **pydantic is now a dependency** (`>=2.12`). + +### Fixed + +- **Idle workers reconnected to Redis every few seconds, forever.** redis-py's + default `socket_timeout` is exactly the runtime's default `XREADGROUP BLOCK` + window, so every idle read raced its own client timeout — and the resulting + error was handled as an ordinary empty poll, hiding it. `connect()` now sets + the socket timeout above the longest blocking command. +- **The connection pool was not sized to the topology.** redis-py caps a pool at + 100 while every blocked worker holds one for its whole claim window; a large + enough deployment hit the ceiling as a crash. It is now computed from the + resolved slot count. +- A failure anywhere outside a handler's own guard — a reply that would not + serialize, a blip on an ack — killed the worker task and with it the whole + process. One bad message no longer stops the worker. +- Lifespans could be torn down while handlers were still using their resources, + because the shutdown path caught only `CancelledError`. + +### Known limits + +No reclaim, no retries, no dead-letter, no heartbeat, no graceful drain, no +service registry. Each omission and what it costs is documented in +[`doc/limits.md`](https://github.com/archipellabs/runtime/blob/main/doc/limits.md) +— read it before deploying. + +## 0.2.0 + +First public release. `Pool` + `Scheduler` over Redis Streams, one verb (`emit`), +a flow per event type. diff --git a/README.md b/README.md index 587b440..fd6946b 100644 --- a/README.md +++ b/README.md @@ -1,182 +1,158 @@ # archipellabs-runtime -An async runtime for running lots of tasks at once — Playwright sessions, API -calls, SSE streams, timed waits — without losing control of how many run together. +An async runtime for distributing work over Redis — Playwright sessions, API +calls, SSE streams, timed waits — without losing control of how many run at once. These tasks spend most of their time *waiting* (on a network call, a browser, a timer), so you want many of them going at once. Two easy approaches both break: -- **A new task for every event:** a sudden spike starts thousands at once and falls - over — too many browsers, too many open connections. -- **One event at a time:** safe, but a single slow call (say a 2-second request) - holds up everything behind it. All that waiting happens one after another instead - of together. +- **A new task per event:** a spike starts thousands at once and falls over — too + many browsers, too many open connections. +- **One at a time:** safe, but a single slow call holds up everything behind it. + All that waiting happens in sequence instead of together. -A **pool** is the middle ground: a fixed number of workers (`max_slots`, say 20) -sharing the work. Up to 20 events run together — enough to overlap the waiting — and -never more, so nothing gets overwhelmed. Each pool has its own size, so a slow batch -of browser sessions can't hog the workers your API calls need. +A **service** is the middle ground: a fixed budget of workers (`max_slots`, say +20) sharing the work. Up to 20 run together — enough to overlap the waiting — and +never more. Each service has its own budget, so a slow batch of browser sessions +cannot hog the workers your API calls need. -You pick each pool's size from whatever its work is bound by — a RAM budget for -Playwright browsers, a rate limit for an HTTP API, a connection cap for a database. - -Work is driven by **events** over a Redis stream: **producers** emit events on a -schedule, **consumers** handle them concurrently. The two sides are decoupled — -they share only an event name, never a reference — so you can run them together in -one process or scale them apart. +Services talk to each other with three verbs and nothing else. No registry, no +service discovery, no configuration tying them together: a producer in one process +and a consumer in another agree because they share a string. > Distribution name `archipellabs-runtime`; it imports as `runtime`. ## Install ```sh -pip install archipellabs-runtime # requires Python 3.12+ and a Redis server -``` - -```python -from runtime import App, Pool, Scheduler +pip install archipellabs-runtime # requires Python 3.12+ and a Redis server ``` ## Quickstart -A consumer (`Pool` + a flow) and a producer (`Scheduler` + `@every`), wired into -one `App`: - ```python import os -from runtime import App, Pool, Scheduler +from runtime import App, Service -# ── consumer: a Pool of flows ────────────────────────────────────────────── -orders = Pool("orders", max_slots=20) +pricing = Service("pricing", max_slots=20) -@orders.flow(consumes="order.placed") -async def fulfill(ctx, event): - print(f"fulfilling order {event['id']}") - # ... do the work; called once per event, should be idempotent ... +@pricing.action("pricing.quote") # one executant, returns a value +async def quote(ctx, params): + return {"total": round(19.99 * params["qty"], 2)} -# ── producer: a Scheduler that emits on an interval ──────────────────────── -load = Scheduler("load") +@pricing.event("order.placed") # every subscriber gets a copy +async def note_order(ctx, params): + print(f"order {params['id']}") -@load.every("500ms") # emit twice a second -async def place_orders(ctx): - await ctx.emit("order.placed", id=os.urandom(4).hex()) +@pricing.every("500ms") # runs on a schedule +async def tick(ctx): + total = await ctx.call("pricing.quote", qty=3) + print(f"quote: {total}") -# ── wire it up and run ───────────────────────────────────────────────────── -app = App(redis="redis://localhost:6379/0") -app.include(orders) -app.include(load) -app.start() # blocking; Ctrl-C to stop +app = App(redis=os.environ.get("REDIS_URL", "redis://localhost:6379/0")) +app.include(pricing) +app.start() # blocking; logs the topology first ``` -`app.start()` connects to Redis, spawns the consumer workers and the producer -loop, and runs until interrupted. You'll need a Redis on `:6379` — for local dev, -the quickest is: +## The three verbs -```sh -docker run --rm -p 6379:6379 redis +| | Cardinality | Reply | Caller | +|---|---|---|---| +| `await ctx.call(action, **params)` | exactly one executant | the return value, awaited | coupled in time | +| `await ctx.dispatch(action, **params)` | exactly one executant | a task id | not coupled | +| `await ctx.emit(event, **params)` | every subscriber | none | not coupled | + +Every message can carry a `ttl` — a deadline that travels with it, and that any +`call` or `dispatch` the handler makes inherits and can only narrow — and a +`delay`. Failures cross the wire as typed errors and are re-raised in the caller. +Actions can declare a pydantic model for their params and get validation before +the handler runs. + +```python +await ctx.call("pricing.quote", ttl="2s", qty=3) +await ctx.dispatch("report.build", delay="30s", month="2026-07") +await ctx.emit("order.placed", id="o1") ``` -Runnable examples live in [`examples/`](examples): +### Prior art -- **`minimal/`** — the smallest app (one pool, one flow, one producer; no deps). -- **`poisson/`** — a pool that is *both* consumer and producer: a scheduler opens a - window every 10s, the `generator` pool turns each window into a Poisson-sized - burst of events for the `workers` pool (no deps). -- **`orders/`** — a `lifespan` resource on *both* sides (a store on the consumer, - a catalog client on the producer). `main.py` spawns the `producer` and - `consumer` as two processes (the split deployment); no external Python deps. -- **`playwright/`** — a browser-driven load sim (needs the `playwright` extra). +The semantics are lifted from **[Moleculer](https://moleculer.services)** — actions, +`call`, a propagated context, typed errors, distributed timeouts. What is not taken +is its protocol: no registry, no heartbeats, no discovery, because Redis Streams +already do that. `dispatch` is the one addition, where Moleculer folds decoupled +work into a balanced `emit`. -## Shared resources: the lifespan +[doc/index.md](https://github.com/archipellabs/runtime/blob/main/doc/index.md) has +the argument; [doc/limits.md](https://github.com/archipellabs/runtime/blob/main/doc/limits.md) +has what it costs. -A `Pool` (or `Scheduler`) can hold a resource opened once at boot and shared by -everything in it — a browser process, an HTTP client, a database pool. Provide a -`lifespan`: an async context manager `(config) -> resources`. Whatever it yields -is reachable as `ctx.resources`. +## Shared resources + +A service's `lifespan` opens what its handlers share — a browser, a DB pool, an +API client — once at boot, and closes it on shutdown. ```python from contextlib import asynccontextmanager -from runtime import Pool @asynccontextmanager -async def browser(config): - pw = await launch_browser(headless=config["headless"]) +async def store(config): + db = await connect(config["dsn"]) try: - yield {"browser": pw} # → ctx.resources["browser"] + yield {"db": db} # → ctx.resources["db"] finally: - await pw.close() # torn down on shutdown + await db.close() -shop = Pool("shop", max_slots=12, lifespan=browser) +warehouse = Service("warehouse", max_slots=8, lifespan=store) -@shop.flow(consumes="shopping.session") -async def shop_session(ctx, event): - page = await ctx.resources["browser"].new_context() - try: - ... - finally: - await page.close() # per-event teardown lives in the body +@warehouse.action("order.fulfil") +async def fulfil(ctx, params): + await ctx.resources["db"].fulfil(params["id"]) -# config is injected at wiring time: -app.include(shop, config={"headless": True}) +app.include(warehouse, config={"dsn": "postgres://…"}) ``` -## Core concepts - -- **Pool → flows (consumers).** A `Pool` groups *flows* — plain async - `(ctx, event)` handlers, each bound to one event type with - `@pool.flow(consumes=...)` — that share a lifespan and a concurrency budget - (`max_slots`). Workers compete for messages; the pool's semaphore caps how many - run at once. -- **Scheduler → producers.** The mirror of a `Pool`: *producers* are async `(ctx)` - bodies bound with `@scheduler.every(interval)` and called on that interval. - Intervals are durations — `"10min"`, `"1.5s"`, `"500ms"`, `"1h"`, or a number of - seconds. -- **Context.** Every handler and producer gets a `ctx`: `ctx.resources` (the - lifespan's shared resources), `ctx.config` (injected via - `App.include(config=...)`), and `await ctx.emit(event_type, **payload)` to push - an event. -- **Delivery.** Events flow through a Redis stream behind a small `Broker` protocol, - at-least-once — a crash can redeliver a message — so handlers should be - idempotent. - -Both decorators have an imperative twin for dynamic wiring: -`pool.register(handler, consumes=...)` and `scheduler.register(producer, interval=...)`. +## Documentation -## Deployment +Full documentation is in [`doc/`](https://github.com/archipellabs/runtime/blob/main/doc/index.md): -Because the two sides talk only through Redis, you can run everything in **one -process** (as in the quickstart) or **split** it across deployments — the same -`App`, with `include()` deciding what each entrypoint runs. +| | | +|---|---| +| [concepts](https://github.com/archipellabs/runtime/blob/main/doc/concepts.md) | services, the three verbs, choosing between them | +| [messages](https://github.com/archipellabs/runtime/blob/main/doc/messages.md) | the envelope, correlation, deadlines, errors, params | +| [internals](https://github.com/archipellabs/runtime/blob/main/doc/internals.md) | the keyspace, a message's life, what runs in a process | +| [operations](https://github.com/archipellabs/runtime/blob/main/doc/operations.md) | budgets, runtime switches, deployment, `App` knobs | +| [limits](https://github.com/archipellabs/runtime/blob/main/doc/limits.md) | what this deliberately does not do — **read before deploying** | -At scale, the simplest pattern is **one pool (or scheduler) per app**, gated by -`enabled`, and let your infra (Kubernetes, ECS, …) run and scale each independently. -Every entrypoint builds the same `App`; an env var picks its role: +Runnable examples in [`examples/`](https://github.com/archipellabs/runtime/tree/main/examples): -```python -role = os.environ["ROLE"] -app.include(load, enabled=role == "producer") # schedulers — 1 replica -app.include(browsers, enabled=role == "browsers") # heavy pool — N replicas -app.include(apis, enabled=role == "apis") # light pool — M replicas -app.start() -``` +- [`minimal`](https://github.com/archipellabs/runtime/tree/main/examples/minimal) — one service, one action, one producer +- [`rpc`](https://github.com/archipellabs/runtime/tree/main/examples/rpc) — `call`, typed errors, params validation, a ttl expiry +- [`events`](https://github.com/archipellabs/runtime/tree/main/examples/events) — fan-out to independent subscribers +- [`jobs`](https://github.com/archipellabs/runtime/tree/main/examples/jobs) — `dispatch` + `task_status`, a `delay`, and a runtime switch +- [`orders`](https://github.com/archipellabs/runtime/tree/main/examples/orders) — two processes, lifespans on both sides +- [`poisson`](https://github.com/archipellabs/runtime/tree/main/examples/poisson) — a service that consumes *and* produces +- [`playwright`](https://github.com/archipellabs/runtime/tree/main/examples/playwright) — a heavy, isolated cost profile -A disabled component is neither started nor armed, so one image + one env var per -deployment is all it takes — no per-role code, and each pool scales on its own. +Upgrading from 0.2? The API is replaced — see +[CHANGELOG.md](https://github.com/archipellabs/runtime/blob/main/CHANGELOG.md). -To run several environments against **one** Redis (e.g. dev and staging on the -same box), give each its own `namespace` — it prefixes every stream key so their -streams and consumer groups can't mix: +## Deployment -```python -app = App(redis="redis://localhost:6379/0", namespace=os.environ["ENV"]) # "dev", "staging", … -``` +The same App runs in one process, split by role, or scaled to N replicas with no +code change. `App(namespace=...)` isolates environments sharing one Redis; +`include(enabled=False)` leaves a service out of a process entirely, and runtime +[switches](https://github.com/archipellabs/runtime/blob/main/doc/operations.md#switches) +pause one that is mounted. ## Development ```sh uv sync --extra dev -uv run python -m pytest # falls back to fakeredis if no Redis is running -uv run mypy +uv run python -m pytest # auto-detects a backend +RUNTIME_TEST_BACKEND=fake uv run python -m pytest # CI runs both legs +RUNTIME_TEST_BACKEND=real uv run python -m pytest # needs a Redis on :6379 +uv run python -m pytest --cov=runtime --cov-report=term-missing +uv run mypy && uv run ruff check . ``` ## License diff --git a/doc/concepts.md b/doc/concepts.md new file mode 100644 index 0000000..63d9375 --- /dev/null +++ b/doc/concepts.md @@ -0,0 +1,125 @@ +# Concepts + +## Service + +One domain: the actions it exposes, the events it reacts to, its scheduled +producers, the resources they share, the budget they compete for. + +```python +catalog = Service("catalog", max_slots=1, lifespan=prestashop_clients) + +@catalog.action("catalog.sync") # others call or dispatch this +async def sync(ctx, params): ... + +@catalog.event("product.updated") # reacts to someone else's announcement +async def on_update(ctx, params): ... + +@catalog.every("5m") # scheduled +async def reconcile(ctx): ... + +@catalog.once(delay="30s") # once, after boot +async def warm_cache(ctx): ... +``` + +Consumers and producers share one object because a handler that calls another +action makes its process both executant and caller. A service is also the unit of +**isolation** (split by cost profile — Playwright, SSE and API calls do not belong +in one budget) and of the [master switch](operations.md#switches). + +**`max_slots`** — handlers running at once, across everything the service +consumes. Pick it from what the work is bound by: RAM for browser contexts, a rate +limit for an API, a connection cap for a DB. Required if the service consumes +anything, refused at boot otherwise. No default, because both directions are +wrong: too low silently serialises a workload, too high removes the bound the +number exists to enforce. Producer-only services need none. + +**`lifespan`** — an `@asynccontextmanager` taking the injected config, yielding a +mapping that becomes `ctx.resources`. Runs once at boot, torn down on shutdown. +Two scopes exist and no more: SERVICE (the lifespan) and HANDLE (a `try/finally` +in the handler body). + +```python +@asynccontextmanager +async def prestashop_clients(config): + client = await connect(config["dsn"]) + try: + yield {"db": client} # → ctx.resources["db"] + finally: + await client.close() +``` + +## The three verbs + +Available on every context, a handler's or a producer's. + +| | Cardinality | Reply | Caller | +|---|---|---|---| +| `await ctx.call(action, **params)` | exactly one executant | return value, awaited | coupled in time | +| `await ctx.dispatch(action, **params)` | exactly one executant | a task id | not coupled | +| `await ctx.emit(event, **params)` | every subscriber | none | not coupled | + +`call` and `dispatch` are the **same message**, same stream, same consumer group — +they differ only in whether a reply channel is named. `emit` differs in kind. + +Choosing, in order: + +1. **Does anyone need the result?** Yes → `call`. "Not really, but I'd like to know + it worked" → `dispatch` plus `ctx.task_status(id)`; `call` makes a slow + executant into your latency. +2. **Instruction or announcement?** "Sync the catalogue" has one correct executant + and running it twice concurrently is a bug — an action. "An order was placed" + is a fact that analytics, billing and search may each want — an event. + +> Both compile. An `emit` that should have been a `dispatch` writes to a stream +> with no subscribers and vanishes silently; without a registry the runtime cannot +> know nobody was listening. + +## Fan-out, precisely + +Each subscriber joins the event stream under **its own consumer group**, named +after the subscribing service by default — that is what gives each a copy. Within +a group exactly one worker wins, so scaling a subscriber to N replicas does not +process the event N times. + +```python +@analytics.event("order.placed") # group "analytics" +@billing.event("order.placed") # group "billing" +# → both react to every order, independently + +@audit.event("order.placed", group="billing") +# → competes with billing; only one of them sees each message +``` + +Sharing a group is occasionally right (splitting load) and a silent bug otherwise, +so it is opt-in; two handlers on one event in one service is refused at +registration — i.e. at import time, before the App is even built — with an error +naming the fix. The cross-service case is caught at boot instead. A new subscriber +starts **at the tail**: deploying one never dumps the retained stream into it. + +## Handlers + +```python +async def handler(ctx, params) -> Any: ... +``` + +`ctx.params` is the same object. Decorators return the function unchanged, so a +handler stays testable bare: `await sync(fake_ctx, {"sku": "barrel"})`. An action's +return value becomes the reply; an event handler's is ignored. Handlers must be +**idempotent** — delivery is at-least-once. + +## Context + +| | | +|---|---| +| `ctx.resources` | what the lifespan yielded | +| `ctx.config` | injected at `App.include(config=...)` | +| `ctx.params` | the body being handled | +| `ctx.meta` | user metadata that arrived, and propagates onward | +| `ctx.request_id` | shared by everything descended from one origin | +| `ctx.level` | depth in the call chain; 0 straight from a producer | +| `ctx.caller` | the action or event that produced this message | +| `ctx.remaining()` | seconds of budget left, or `None` | +| `ctx.is_enabled(name)`, `ctx.set_enabled(name, bool)` | [switches](operations.md#switches) | + +The correlation fields are what keep a chain of calls legible — see +[messages.md](messages.md). diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 0000000..a2e5baa --- /dev/null +++ b/doc/index.md @@ -0,0 +1,56 @@ +# archipellabs-runtime — documentation + +An async runtime for distributing work over Redis. You declare **services**; they +talk to each other with **three verbs**; the runtime handles the transport, +concurrency and correlation. + +```python +from runtime import App, Service + +pricing = Service("pricing", max_slots=4) + +@pricing.action("pricing.quote") +async def quote(ctx, params): + return {"total": 19.99 * params["qty"]} + +app = App(redis="redis://localhost:6379/0") +app.include(pricing) +app.start() +``` + +Anywhere else, in any process on the same Redis: + +```python +total = await ctx.call("pricing.quote", qty=3) +``` + +No registry, no service discovery, no configuration tying the two together. They +agree because they share a string. + +## The bet + +This design takes [Moleculer](https://moleculer.services)'s *semantics* — actions, +`call`, a propagated context, typed errors, distributed timeouts — and drops its +*protocol*: no node registry, no heartbeats, no discovery packets, no gossip. +Redis Streams already provide distribution, persistence, at-least-once delivery +and consumer groups. Reimplementing that in an application protocol is duplicated +machinery. + +What you get is about 2,000 lines you can read in an afternoon, with no protocol +to learn. What you give up is listed honestly in [limits.md](limits.md) — read it +before you deploy this. + +## Where to look + +| | | +|---|---| +| **[concepts.md](concepts.md)** | Services, the three verbs, and how to choose between them. Start here. | +| **[messages.md](messages.md)** | What travels on the wire: the envelope, correlation, deadlines, `ttl`, `delay`, typed errors, params validation. | +| **[internals.md](internals.md)** | How it actually works: the keyspace, a message's life, the coroutines inside a process. | +| **[operations.md](operations.md)** | Budgets, runtime switches, deployment shapes, and the knobs on `App`. | +| **[limits.md](limits.md)** | What this deliberately does not do, and what that costs you. | + +Runnable examples live in [`../examples`](../examples): `minimal` (dispatch), +`rpc` (call, typed errors, expiry), `events` (fan-out), `jobs` (task status, +delay, switches), `orders` (two processes, lifespans), `poisson` (a service that +consumes and produces), `playwright` (a heavy, isolated cost profile). diff --git a/doc/internals.md b/doc/internals.md new file mode 100644 index 0000000..699a964 --- /dev/null +++ b/doc/internals.md @@ -0,0 +1,229 @@ +# Internals + +## Keyspace + +Every key is prefixed by `App(namespace=...)`, so several environments can share +one Redis without their streams, groups, replies, tasks or switches mixing. + +``` +{ns}:act:{name} stream action requests — call and dispatch share it + group "workers" ONE group ⇒ one executant +{ns}:evt:{name} stream event notifications + group "{group}" one per subscriber, created at $ +{ns}:reply:{node} list replies to one process (LPUSH → BLPOP), EXPIRE'd +{ns}:task:{id} hash pending | done | failed | expired, EXPIRE'd +{ns}:due zset delayed messages, score = due epoch ms +{ns}:switches hash name → "0" (paused); ABSENT means running +``` + +Discovery is implicit in this naming. A producer in one process and a consumer in +another agree by sharing a string — there is no registry to keep, and no registry +to go stale. + +Naming lives in `broker.py` beside the `Broker` protocol, because it is +backend-agnostic routing rather than a Redis-ism. + +## Anatomy of a process + +One asyncio event loop. `App._serve` spawns, and supervises in one task list: + +``` +slot workers N per registration. claim → handle → report → ack. +reply pump one. BLPOP the reply channel, resolve waiting futures. +delay sweeper one. Move due messages onto their streams. +switch refresh one. HGETALL the registry into an in-process snapshot. +producers one per @every / @once. +``` + +The last four being ordinary tasks in the same list is deliberate. An +unsupervised reply pump that dies takes every subsequent `call` in the process +with it — silently, each one waiting out its own deadline. + +`asyncio.gather` catches **`BaseException`**, not just `CancelledError`. If it +caught only cancellation, one task raising would skip the drain and let +`AsyncExitStack` tear down lifespans while other handlers were still using +`ctx.resources`. + +## A message's life + +``` +ctx.dispatch("work", n=1) + │ build envelope (inherit rid/lvl/caller/meta/dl from the handled message) + │ guard max_depth + │ write task record: pending + ▼ +XADD {ns}:act:work meta= data= MAXLEN ~ N + ▼ +slot worker: XREADGROUP > (BLOCK claim_block) + ├─ undecodable? log, ACK ← never a poison pill + ├─ past its dl? report Expired, ACK — handler NOT entered + ├─ params invalid? report ParamsInvalid, ACK + ▼ +async with semaphore: ← wraps ONLY the handler call + result = await handler(ctx.for_message(envelope, body), body) + ▼ +report: LPUSH reply channel (if envelope.reply — i.e. it was a call) + write task record (if envelope.task — i.e. it was a dispatch) + log at ERROR (if it failed with nowhere to report) + ▼ +XACK +``` + +The semaphore wrapping only the handler matters: a burst of thousands of expired +messages after a long pause costs no permits, so shedding them cannot be mistaken +for a deadlock. + +### The ack rule: always ack + +Every message is acked, whatever happened to it. With no reclaim, leaving one +un-acked does **not** mean "retry later" — it means lost anyway, *plus* a pending +entry attached to a consumer name containing a pid that will never exist again, +which `MAXLEN` will eventually trim out from under. The outcome is reported first; +the ack follows. + +The genuine loss window is a crash between reporting and acking: the caller has +its answer and the message stays pending forever. Same order as the sweeper's. + +## The reply rendezvous + +A call cannot wait on the queue. The reply lands on a channel shared by every call +the process has outstanding, and something must match it to the one coroutine +waiting. That is `Node`: a `dict[envelope_id, Future]` plus one pump. + +``` +caller executant + register future[id] + XADD … reply=reply:{node} ──────▶ handler runs + await wait_for(future, remaining) LPUSH {ns}:reply:{node} + ◀────────────────────────────────────── (+ PEXPIRE) + pump BLPOPs it, resolves future[id] + finally: discard id +``` + +One channel per process, not one key per call. Correlation is in memory. + +The envelope carries the **logical** key, `reply:{node}` — the namespace is added +by the broker when it writes, like every other key. Nothing on the wire is tied to +one deployment's prefix. + +**Node id** is `hostname-pid-uuid8`. The uuid is not decoration: two Apps in one +process — which the examples and any integration test do — would otherwise share a +channel, each pump winning about half the pops and discarding the rest as unknown +ids. Half of all calls would time out for no visible reason. Pids are also reused +after a container restart. + +**A late reply is harmless.** The caller has already given up and removed its +entry, so the pump finds nothing waiting and drops it. + +**The pump's `finally` fails every pending call** with `Undeliverable`. This is +what makes a plain per-call `wait_for` sufficient instead of the sweeper coroutine +the design note called for: a timeout handles a slow executant, but only the pump +knows when the channel itself has gone away, and a caller should learn that at +once rather than at the end of its budget. + +## Delayed delivery + +Redis Streams have no native delayed delivery, so a sorted set holds messages +until due and a sweeper moves them. + +```python +ZADD {ns}:due "\n\n" +``` + +The member contains the **envelope id**, and that is load-bearing. Sorted-set +members are a *set*: a member derived from the body alone would make +`for _ in range(10): await ctx.emit("tick", delay=5)` store **one** message, with +each later emit silently overwriting the earlier one's due time. For a load +simulator that is the most likely thing anyone does, and it fails with no error at +all. There is a regression test. + +The sweep, every ~250 ms in every process: + +``` +ZRANGEBYSCORE due -inf now LIMIT 0 100 +pipeline: ZREM each ← the return value 1 says THIS process owns it +pipeline: XADD the winners +``` + +`ZREM` is the arbiter — no Lua, no lock. Exactly one process gets a `1` for a +given member, so N processes cooperate without coordinating. Both mutating stages +are pipelined, so a batch of 100 costs 3 round trips (the range read, then the +two pipelines) rather than 201. + +A full batch loops immediately instead of sleeping, so the batch size caps the +*idle* poll rate rather than throughput. The sleep is jittered so N processes do +not all wake on the same boundary. `App(sweeper=False)` opts a process out +entirely, for fleets large enough that a few dedicated sweepers beat thousands of +polls per second. + +**Accepted loss window:** a crash between `ZREM` and `XADD` drops the message. Lua +would close it; it would also cost the test suite its Lua-free `fakeredis`. + +## Switches + +`SwitchBoard` reads the whole registry in one `HGETALL` on a timer into an +in-process snapshot. Every check answers from memory, so nothing on the hot path +touches Redis — which is what makes `call`'s fail-fast free. + +`is_enabled(*names)` is simply "none of these is paused", so callers pass the whole +chain — service name and registration name — and the master switch needs no +precedence rules. + +A **generation counter** guards a real race: `refresh` awaits its read and *then* +assigns, so a local `set()` landing during that await would be silently clobbered +by the stale result. A refresh that started before a `set()` discards what it read. + +A failed refresh keeps the previous snapshot rather than pausing the world — +losing Redis should not stop a system that is otherwise able to run. + +Pausing is eventually consistent from two directions: one refresh interval to be +*seen*, plus one claim window, because a worker already parked in an +`XREADGROUP BLOCK` is woken by an arriving message and will handle it. Pausing +stops a worker taking **new** work; it cannot retract a read already in flight. + +## Connection settings + +Two defaults in redis-py are actively wrong here, so `connect()` sets both. + +**`socket_timeout` defaults to 5 s — exactly the default `XREADGROUP BLOCK` +window.** Every idle worker races its own read timeout and quietly reconnects +every few seconds, forever, with the resulting `TimeoutError` handled as a normal +empty poll. `connect(max_block_s=...)` puts the socket timeout above the longest +blocking command; `App(claim_block=...)` feeds it. + +**`max_connections` defaults to 100** while every blocked worker holds one for its +whole window. The App computes it from the topology — which is why the client is +opened *after* the topology walk, not in `__init__`. Otherwise a busy deployment +discovers the ceiling as a crash rather than a misconfiguration. + +`redis_io.py` also converts a cancellation that lands inside a blocking read back +into `CancelledError`: `async_timeout` turns it into a `TimeoutError`, making a +shutdown look exactly like an idle poll, and a worker would loop forever instead +of unwinding. + +## Boot validation + +`App._serve` refuses to start on anything checkable locally: + +- an action name claimed by two services — an action has one executant +- the same `(event, group)` twice — they would compete, not both run +- one name used as both an action and an event — different streams, so it "works", + and `call` and `emit` quietly reach different handlers +- a service name colliding with a registration name — one switch, two meanings +- a consuming service with no `max_slots` + +What no local check can catch: two *processes* claiming the same action name, or +subscribing under one group with different handlers. That is the price of having +no registry. + +## The Broker seam + +Everything that touches Redis goes through one Protocol — streams, replies, task +records, the due set, the switch registry. Several boot tests hand `App._serve` a +broker over `fakeredis` and a deliberately unusable URL; a subsystem that reached +past the seam for its own connection would drag a real Redis into tests that have +no business needing one. + +A SQL implementation is plausible for all of it: `SELECT … FOR UPDATE SKIP LOCKED` +for the streams, a table plus LISTEN/NOTIFY for replies, a `due_at` column for the +delay set, a row per task. diff --git a/doc/limits.md b/doc/limits.md new file mode 100644 index 0000000..9982005 --- /dev/null +++ b/doc/limits.md @@ -0,0 +1,122 @@ +# Limits + +What this runtime deliberately does not do, and what each omission costs. Read +this before deploying it somewhere that matters. + +The version is alpha. The goal so far has been to get the *patterns* and the API +right — reliability is a later, deliberate stage, not an oversight. + +## Not implemented + +### No reclaim + +Nothing recovers a message whose worker died mid-handle. There is no +`XAUTOCLAIM` loop, no visibility timeout, no pending-list recovery on boot. + +Because of that, every message is acked once its outcome is reported — leaving one +un-acked would not mean "retry later", it would mean lost anyway *plus* an +orphaned pending entry (see [internals](internals.md#the-ack-rule-always-ack)). + +**Cost:** a process killed mid-handler loses that message. At-least-once delivery +is real for redelivery-on-restart of *unclaimed* work, but a claimed-and-crashed +message is gone. + +### No retries, no dead-letter + +A handler that raises reports its failure and the message is done. Nothing tries +again; nothing is quarantined for inspection. + +**Cost:** retry policy is yours to build in the handler, or in a `@every` +reconciliation producer that re-derives what is missing. The latter is usually the +better shape anyway. + +### No heartbeat, no liveness + +A worker publishes nothing about itself. There is no way to ask what is running, +how loaded it is, or whether the thing you called is alive. + +**Cost:** a `call` to something that is down is indistinguishable from a `call` to +something slow. Both surface as `CallTimeout` at the end of the budget. + +### No registry + +Discovery is implicit in key naming. Nothing validates that an action you call +exists anywhere. + +**Cost:** `ctx.call("typo.in.name")` hangs until its `ttl` and then raises +`CallTimeout`. This is the single most likely thing to confuse someone new. +Boot-time validation catches collisions *within* a process, but nothing can see +across processes. + +### No graceful drain + +There is no SIGTERM handler. Shutdown cancels tasks and runs lifespan teardown; +in-flight handlers are cut. + +**Cost:** every redeploy interrupts running work, and with no reclaim it is not +picked up elsewhere. Keep handlers short and idempotent. + +## Sharp edges that are implemented + +### A handler that calls holds its slot + +`ctx.call` suspends inside the service semaphore. An action that calls **itself** +inside one service wedges deterministically in a single process once every permit +is held by a waiter; `A → B → A` across two services is dining philosophers. +Deadlines break both, and because a child inherits the parent's remaining budget, +the inner call is already expired when finally claimed and is dropped rather than +executed pointlessly. + +This is not fixed by releasing the permit across the wait, and that was a +considered decision: it would change `max_slots` from bounding in-flight +*messages* to bounding in-flight *work*, and a handler parked in `wait_for` still +holds whatever it opened — a browser context, a transaction. For the workloads +this runtime is for, that would quietly remove the memory bound `max_slots` exists +to enforce. + +What exists instead: `App(max_depth=8)` bounds the runaway case, and services are +the tool for the rest. Put the caller and the callee in **different services** — +which is the isolation argument the whole design rests on. + +`dispatch` and `emit` never block. `call` is the only verb that can wedge you. + +### `emit` to nobody is silent + +Writing to an event stream with no subscribers is legal, indistinguishable from +correct behaviour, and the runtime cannot detect it without a registry. An `emit` +that should have been a `dispatch` loses every message with no error anywhere. + +### Absolute deadlines assume synced clocks + +`dl` is a wall-clock value set on one host and compared on another. NTP steps and +VM resume can make an executant accept expired work or drop live work. Do not set +`default_ttl` below plausible skew. + +### The delay sweeper can lose a message + +A crash between `ZREM` and `XADD` drops it. Lua would make the move atomic, at the +cost of the test suite's Lua-free `fakeredis`. Delay also gives **no ordering +guarantee** — same-score messages sweep in lexicographic order. + +### Switches are eventually consistent + +One refresh interval to be seen, plus one claim window, because a worker already +parked in a blocking read will handle what that read returns. An operational +lever, not a safety mechanism. A master pause is also invisible to remote callers, +which will time out rather than fail fast. + +### Streams are trimmed approximately + +`XADD MAXLEN ~` is a bound, not a guarantee, and trimming is by *count*, not by +whether anyone has read it. A subscriber down longer than `stream_maxlen` messages +misses what was trimmed. Size it against your slowest consumer's realistic outage. + +## If you need the missing half + +The shapes are known and mostly independent: an `XAUTOCLAIM` reclaim loop with a +visibility timeout and heartbeat extension for long handlers; a dead-letter stream +after N attempts; SIGTERM → stop-dispatch → drain within a grace period; emit-side +dedup via `SET NX` on a deterministic bucket key; backpressure from `XLEN`; and +expiring per-process keys for fleet observability instead of a heartbeat protocol. + +None of it changes the public surface, which was the point of locking that first. diff --git a/doc/messages.md b/doc/messages.md new file mode 100644 index 0000000..b25633a --- /dev/null +++ b/doc/messages.md @@ -0,0 +1,179 @@ +# Messages + +## The envelope + +Every message is two stream fields: `meta` (the envelope) and `data` (the body, +untouched). They are kept apart so a deadline or due-time check never decodes a +body it may be about to discard. + +```python +class Envelope(BaseModel): + v: int = 3 # envelope version + id: str # message id + rid: str # request id — shared by one origin's descendants + lvl: int = 0 # call depth; gates max_depth + kind: Literal["call", "dispatch", "event"] + name: str + caller: str | None = None # the action/event whose handler sent this + reply: str | None = None # reply channel — set only for kind="call" + dl: int | None = None # absolute deadline, epoch MILLISECONDS + meta: dict = {} # user metadata; propagated + task: str | None = None # task id — set only for kind="dispatch" +``` + +All times in this runtime are epoch **milliseconds**, with no exceptions. Mixing +units across a wire format is a silent factor-of-1000 bug. + +> **Compatibility rule: every envelope field added after v0.3 must have a default.** +> `extra="ignore"` lets an old worker read a new producer's message; defaults are +> the other half. There is no dead-letter queue, so one required field added later +> turns every message from an un-redeployed producer into a poison pill nothing +> can quarantine. + +## Correlation + +Anything a handler sends inherits the envelope it is handling: + +| | | +|---|---| +| `rid` | copied — one id for the whole tree | +| `lvl` | `parent + 1` | +| `caller` | the parent's `name` | +| `meta` | shallow merge, child wins | +| `dl` | inherited, and can only narrow — **except for `emit`**, see below | + +The correlation half holds for all three verbs. A `dispatch` fired from inside a +`call` is still visibly part of the same request — which is the point: the audit +chain must not break the moment a flow turns asynchronous. + +A producer has no envelope of its own, so what it sends starts the tree: + +``` +producer (no envelope — nothing above it) + └─ call outer lvl=0 rid=abc caller=None + └─ call inner lvl=1 rid=abc caller="outer" +``` + +## Deadlines: `ttl` + +One knob per message. At send time it becomes an **absolute deadline** in the +envelope. A worker that claims an expired message drops it *without entering the +handler* and reports `Expired`. + +```python +await ctx.call("pricing.quote", ttl="2s", sku="x") +``` + +A child can only narrow what it was handed. This is the rule as applied to +`dispatch` and `emit`; `call` is a special case, immediately below. + +| parent `dl` | `ttl` | result | +|---|---|---| +| none | `t` | `now + t` | +| `d` | `t` | `min(d, now + t)` | +| `d` | `0` or omitted | `d` — inherited | +| none | `0` or omitted | none | + +`ttl=0` inherits rather than clearing, so one careless leaf cannot let a chain +outlive the request that started it. + +**`call` never reaches the bottom two rows.** An omitted `ttl` is replaced by +`App(default_ttl=...)` (10s) before the table is consulted, so a `call` under a +parent 10 minutes from its deadline gets `now + 10s`, not the parent's. And +`ttl=0` is rejected outright — an unbounded wait holds a worker slot forever. +Pass an explicit `ttl` when a call should be allowed to run as long as its parent. + +**`emit` does not inherit `dl`.** An event has no outcome channel, so an inherited +deadline would make a saturated subscriber drop notifications silently, under +exactly the load this runtime exists to generate. Events inherit correlation but +expire only if you say so. + +Deadlines are absolute wall-clock values compared on another host, so they assume +roughly synced clocks. Do not set `default_ttl` below plausible skew. + +## Delay + +```python +await ctx.dispatch("later", delay="30s") +``` + +The message is held in a sorted set and moved onto its stream once due. Its +**budget starts at the due time**, not at send time — otherwise +`call(ttl="5s", delay="10s")` would always arrive expired. + +Delay gives **no ordering guarantee**: messages due at the same moment are swept +in lexicographic order and get their stream ids then. See +[internals.md](internals.md#delayed-delivery) for the mechanism and its loss +window. + +## Params validation + +Declare a pydantic model and the runtime validates before the handler runs; the +handler receives the instance. + +```python +class Quote(BaseModel): + sku: str + qty: int = 1 + +@pricing.action("pricing.quote", params=Quote) +async def quote(ctx, params): # params: Quote + return 19.99 * params.qty +``` + +A body that fails becomes a `ParamsInvalid` — reported to the caller, then acked. +Never redelivered: the same bytes fail the same way. + +## Errors + +`ActionError` carries `code`, `message`, `data`. It is serialized into the reply +and rebuilt in the caller, so `await ctx.call(...)` raises something you can +branch on rather than a timeout or a stringified traceback. + +| code | meaning | +|---|---| +| `TIMEOUT` | the caller's budget ran out. Says nothing about whether the work ran | +| `EXPIRED` | claimed after its deadline; the handler definitely did **not** run | +| `PARAMS_INVALID` | failed the `params` model | +| `DISABLED` | the target is [paused](operations.md#switches); refused before queueing | +| `MAX_DEPTH` | the chain exceeded `App(max_depth=...)` | +| `UNDELIVERABLE` | the reply pump stopped; no reply can arrive | +| `REMOTE` | the handler raised something else, or a code this version does not know | +| `ERROR` | a bare `ActionError`, raised without subclassing | + +`ActionError` is the base, and only *subclasses* register themselves — so a bare +`ActionError` crosses as code `ERROR` and is rebuilt caller-side as a +`RemoteError` carrying that code, not as an `ActionError`. Subclass rather than +raising the base if you want the class back. + +Subclass for domain failures — the code is all it takes: + +```python +class OutOfStock(ActionError): + CODE = "OUT_OF_STOCK" + +raise OutOfStock("no stock", data={"sku": sku}) +``` + +A handler raising a plain exception still crosses, as `REMOTE`, with the original +class name kept in the message and in `data["exception"]` — so a caller sharing no +code with the executant can still tell a `ValueError` from a `KeyError`. An +unrecognised code degrades to `RemoteError` *carrying that code*, so branching on +`err.code` works even for classes you do not have. + +Moleculer also puts `type` and `retryable` on the wire. Both are omitted: nothing +retries here, and a wire field with no consumer is one you cannot change later. + +## Task records + +`dispatch` returns a task id; the outcome lands in a record with a TTL. + +```python +task_id = await ctx.dispatch("work", n=7) +await ctx.task_status(task_id) +# {"state": "done", "action": "work", "result": {...}, "error": None} +``` + +States are exactly `pending | done | failed | expired`. No `attempts`, no +`started_at`, no `progress` — those are reliability fields wearing an +observability hat, and none of them would mean anything until reclaim exists. diff --git a/doc/operations.md b/doc/operations.md new file mode 100644 index 0000000..823a4a9 --- /dev/null +++ b/doc/operations.md @@ -0,0 +1,141 @@ +# Operations + +## Wiring + +```python +app = App(redis="redis://localhost:6379/0", namespace="staging") +app.include(catalog, config={"dsn": ...}) +app.include(browser, enabled=settings.browser_enabled, max_slots=4) +app.start() # blocking; logs the resolved topology first +``` + +`App.start()` logs what it resolved before running — services, budgets and any +override, every registration with its stream and slot count (plus the group, for +events, since an action's is always `workers`), and every producer with its +schedule. Read it when something is not being consumed. + +### `App` knobs + +| | default | | +|---|---|---| +| `redis` | — | **required**; the connection URL | +| `namespace` | `""` | prefixes every key; isolates environments on one Redis | +| `default_ttl` | `10s` | budget for a `call` that names none | +| `max_depth` | `8` | how deep a call chain may go before `MaxDepth` | +| `stream_maxlen` | `10_000` | approximate cap on every stream | +| `task_ttl` | `1h` | how long a dispatch's record survives | +| `claim_block` | `5s` | how long a worker parks; also the granularity of a pause | +| `sweeper` | `True` | run the delay sweeper in this process | +| `switch_interval` | `2.0` | how often the switch registry is re-read | + +`stream_maxlen` is not optional in spirit: acking does not delete, and durable +fan-out means entries outlive all their readers. Without a cap every stream grows +forever. + +`default_ttl` is deliberately short. An unbounded call holds a worker slot +indefinitely, and a long default makes that failure look like a hang. + +### `enabled=` is not a switch + +`App.include(enabled=False)` is a **wiring** decision taken once: the service is +not constructed, its lifespan never runs, nothing it declares exists in this +process. That is how you stop a browser launching at all. + +A [switch](#switches) pauses something that *is* mounted and does **not** release +its resources. Conflating the two is the obvious way to be surprised that pausing +a browser service did not close the browser. + +## Switches + +Pause and resume parts of a running system without a deploy. Two levels: a service +name is a master over everything it registers; each action, event subscription and +producer also has its own. + +```python +# from outside — a short-lived connection, no App needed +from runtime import switches + +await switches.disable(url, "catalog.sync", namespace="staging") +await switches.enable(url, "catalog.sync", namespace="staging") +await switches.status(url, namespace="staging") # only what was explicitly set + +# from inside a handler or producer +await ctx.set_enabled("catalog.sync", False) +``` + +Absent means enabled, so nothing has to be seeded and a Redis with no switches +behaves exactly as if the feature did not exist. `redis-cli HSET {ns}:switches +catalog.sync 0` works too. + +What pausing does: + +| | | +|---|---| +| consumers | stop claiming — the stream grows and drains on resume | +| in-flight handlers | finish normally; nothing is interrupted | +| `dispatch`, `emit` | still queue — deferring work for a paused consumer is the point | +| `call` | raises `Disabled` at once rather than waiting out its budget | +| `@every` producers | skip the body, keep the cadence, so resuming is immediate | +| `@once` producers | wait — a one-shot has one chance to run, so it fires on resume rather than being lost | + +It is a throttle and a maintenance lever, never a way to discard work. It is +**eventually consistent** — one refresh interval plus one claim window — and +therefore not a safety mechanism. + +A master pause is invisible to *remote* callers: a caller knows an action's name, +not which service owns it, so `call` can only fail fast on the action's own +switch. Pausing the service makes remote calls time out instead. + +## Deployment shapes + +The same App runs in any of these with no code change, because producers and +consumers are coupled only by a name. + +**One process.** Everything included together. What the examples do. + +**Split by role.** One process includes the producers, another the consumers. +`examples/orders` does exactly this with two entrypoints over one pipeline module. + +**Scale a service.** Run N replicas of a consuming process. They join the same +consumer group and share the work; no configuration, no rebalancing. An action +still has exactly one executant per message, and an event subscriber still +processes each event once however many replicas exist. + +**Isolate a cost profile.** Give Playwright its own deployment. That is what a +service boundary is for — separate budgets mean a flood of browser sessions cannot +starve an API service. + +Caveats worth planning around: there is no graceful drain on SIGTERM, so in-flight +handlers are cut on redeploy; and with no reclaim, whatever they were holding is +not picked up by anyone else. Keep handlers short and idempotent, and see +[limits.md](limits.md). + +## Namespaces + +`App(namespace="dev")` prefixes every key. Two environments on one Redis will not +see each other's streams, groups, replies, tasks or switches. The test suite uses +a fresh namespace per test for exactly this reason — without it, one test pausing +an action leaves it paused for every test that follows on a shared Redis. + +## Running the tests + +```sh +uv sync --extra dev +uv run python -m pytest # auto-detects a backend +RUNTIME_TEST_BACKEND=fake uv run python -m pytest +RUNTIME_TEST_BACKEND=real uv run python -m pytest # needs a Redis on :6379 + +uv run python -m pytest --cov=runtime --cov-report=term-missing +``` + +Coverage sits around 96%. Read the missing lines rather than the number: what is +left is almost entirely failure and configuration paths, which is where the risk +concentrates — a broker raising mid-report, a lifespan failing at boot, the +settings in `connect()` that no test using an injected broker ever reaches. + +`auto` (the default) uses a real Redis when one is listening, else `fakeredis`. +CI runs **both** legs explicitly on every Python version, because the two differ +where it matters: a real `XREADGROUP BLOCK` is woken by an arriving message and +fakeredis's is not, and a real Redis is shared across the whole run while +fakeredis is fresh per test. Bugs hide on each side of that difference — pin the +backend rather than letting autodetection decide what you just tested. diff --git a/examples/events/main.py b/examples/events/main.py new file mode 100644 index 0000000..93963d4 --- /dev/null +++ b/examples/events/main.py @@ -0,0 +1,70 @@ +"""Events example — ``emit``: fan-out to every subscriber. + +The difference from ``dispatch`` is cardinality, not transport. A dispatched +action has exactly ONE executant however many replicas are running; an emitted +event reaches EVERY subscriber, each of which gets its own copy. + +That works because each subscriber joins under its own consumer group — by +default the subscribing service's name. Within a group the message still goes to +one worker, so scaling a subscriber to N replicas does not process the event N +times. Here `analytics` and `billing` both react to every order, independently: +if billing is down, analytics is unaffected and billing catches up when it +returns. + + python -m examples.events.main +""" + +import os +import uuid + +from runtime import App, Service + +analytics = Service("analytics", max_slots=4) + + +@analytics.event("order.placed") +async def index_order(ctx, params): + print(f"[analytics] indexed {params['id']}") + + +billing = Service("billing", max_slots=4) + + +@billing.event("order.placed") +async def invoice_order(ctx, params): + print(f"[billing] invoiced {params['id']}") + + +# A third subscriber that deliberately shares billing's group instead of taking +# its own. Two handlers in one group COMPETE for each message rather than both +# running — occasionally what you want (splitting load across processes), and a +# silent bug the rest of the time. It is opt-in for exactly that reason. +audit = Service("audit", max_slots=1) + + +@audit.event("order.placed", group="billing-shared") +async def record_order(ctx, params): + print(f"[audit] recorded {params['id']}") + + +storefront = Service("storefront") + + +@storefront.every("2s") +async def place_order(ctx): + order_id = uuid.uuid4().hex[:8] + print(f"[storefront] order {order_id} placed") + # No reply, no executant, and no error if nobody is listening — an event is + # an announcement, not an instruction. + await ctx.emit("order.placed", id=order_id) + + +def build_app() -> App: + app = App(redis=os.environ.get("REDIS_URL", "redis://localhost:6379/0")) + for service in (analytics, billing, audit, storefront): + app.include(service) + return app + + +if __name__ == "__main__": + build_app().start() diff --git a/examples/jobs/main.py b/examples/jobs/main.py new file mode 100644 index 0000000..07484aa --- /dev/null +++ b/examples/jobs/main.py @@ -0,0 +1,78 @@ +"""Jobs example — ``dispatch`` with a task record, a ``delay``, and a switch. + +The three things this shows are what you reach for when work is slow enough that +nobody should wait for it: + + 1. ``dispatch`` returns a task id, and ``ctx.task_status(id)`` follows the + outcome — so "fire and forget" does not have to mean "fire and never find + out"; + 2. ``delay`` schedules a message for later without holding anything open in the + meantime — no sleeping coroutine, no timer in your process, and it survives a + restart because the message is in Redis, not in memory; + 3. a **switch** pauses a consumer at runtime. Work keeps queueing and drains + when it is resumed — pausing is a throttle, never a way to discard work. + + python -m examples.jobs.main +""" + +import asyncio +import os + +from runtime import App, Service + +reports = Service("reports", max_slots=2) + + +@reports.action("report.build") +async def build(ctx, params): + await asyncio.sleep(0.2) # stand-in for real work + return {"month": params["month"], "rows": 1_234} + + +control = Service("control") + + +@control.once() +async def demo(ctx): + # ── 1. dispatch, then follow the outcome ──────────────────────────────── + task_id = await ctx.dispatch("report.build", month="2026-07") + print(f"[jobs] dispatched {task_id[:8]} — not waiting for it") + + while (record := await ctx.task_status(task_id))["state"] == "pending": + await asyncio.sleep(0.05) + print(f"[jobs] task finished: {record['state']} {record['result']}") + + # ── 2. delay: due in the future, held in Redis until then ─────────────── + await ctx.dispatch("report.build", delay="0.5s", month="2026-08") + print("[jobs] scheduled a build for 0.5s from now") + + # ── 3. pause the consumer; work queues rather than running ────────────── + await ctx.set_enabled("report.build", False) + # A worker already parked in a claim will still take what that claim returns, + # so give the pause a moment to be fully in effect before measuring it. + await asyncio.sleep(0.3) + + paused_task = await ctx.dispatch("report.build", month="2026-09") + await asyncio.sleep(0.5) + state = (await ctx.task_status(paused_task))["state"] + print(f"[jobs] while paused, the task is still {state!r} — queued, not lost") + + await ctx.set_enabled("report.build", True) + print("[jobs] resumed; the backlog drains") + + +def build_app() -> App: + app = App( + redis=os.environ.get("REDIS_URL", "redis://localhost:6379/0"), + # Small so the pause takes effect quickly enough to watch. Production + # defaults are 2s and 5s. + switch_interval=0.05, + claim_block="0.1s", + ) + app.include(reports) + app.include(control) + return app + + +if __name__ == "__main__": + build_app().start() diff --git a/examples/minimal/main.py b/examples/minimal/main.py index c170fb5..8116bc4 100644 --- a/examples/minimal/main.py +++ b/examples/minimal/main.py @@ -1,35 +1,38 @@ """Minimal example — the smallest possible app. -One pool (no shared resource) with one flow, and one scheduler with one producer, -all in a single process. No external dependencies. Run: +One service with one action and one producer, in a single process. No external +dependencies beyond a Redis. Run: python -m examples.minimal.main + +``dispatch`` hands the work to exactly one executant and does not wait. If you +want the result back, that is ``call`` — see ``examples/rpc``. If you want every +interested party to hear about it, that is ``emit`` — see ``examples/events``. """ import os -from runtime import App, Pool, Scheduler - -pool = Pool("hello", max_slots=5) # lifespan=None: no shared resource - +from runtime import App, Service -@pool.flow(consumes="greeting") -async def greet(ctx, event): - print(f"[minimal] hello {event['name']}") +# A service is one domain: what it consumes, what it produces, the resources they +# share, and the budget they compete for. max_slots is required because this one +# consumes — it is the ceiling on concurrent handlers. +hello = Service("hello", max_slots=5) -greeter = Scheduler("greeter") # lifespan=None: producer needs no resource +@hello.action("greeting") +async def greet(ctx, params): + print(f"[minimal] hello {params['name']}") -@greeter.every("1s") # id defaults to the function name -async def emit_greeting(ctx): - await ctx.emit("greeting", name="world") +@hello.every("1s") # id defaults to the function name +async def send_greeting(ctx): + await ctx.dispatch("greeting", name="world") def build_app() -> App: app = App(redis=os.environ.get("REDIS_URL", "redis://localhost:6379/0")) - app.include(pool) - app.include(greeter) + app.include(hello) return app diff --git a/examples/orders/pipeline.py b/examples/orders/pipeline.py index 0c86cad..5fed11f 100644 --- a/examples/orders/pipeline.py +++ b/examples/orders/pipeline.py @@ -1,19 +1,22 @@ -"""The orders pipeline — a warehouse Pool (consumers) and a load Scheduler -(producers), each with its own lifespan resource. +"""The orders pipeline — two services, each with its own lifespan resource. -The two sides share only the ``order.placed`` event type — never a reference. -main.py spawns them as two processes — producer.py (producers) and consumer.py -(consumers) — which in production would be separate deployments. +``warehouse`` consumes; ``load`` produces. They share only the ``order.fulfil`` +action name — never a reference — which is what lets main.py run them as two +separate processes, the way they would be deployed. + +Note that ``load`` has no ``max_slots``: it consumes nothing, so there is nothing +to bound. A service that registers an action or an event without a budget is +refused at boot rather than quietly given a default. """ import uuid from contextlib import asynccontextmanager from examples.orders.resources import connect_catalog, connect_store -from runtime import Pool, Scheduler +from runtime import Service -# ── consumers: a Pool whose lifespan opens a shared store, used by the flow ── +# ── consumers: a service whose lifespan opens a store shared by every slot ─── @asynccontextmanager async def store_lifespan(config): store = await connect_store(config["dsn"]) # opened once, shared by all slots @@ -23,16 +26,17 @@ async def store_lifespan(config): await store.close() -warehouse = Pool("warehouse", max_slots=8, lifespan=store_lifespan) +warehouse = Service("warehouse", max_slots=8, lifespan=store_lifespan) -@warehouse.flow(consumes="order.placed") -async def fulfill(ctx, event): - await ctx.resources["store"].fulfill(event["id"], event["sku"]) - print(f"[consumer] fulfilled order {event['id']} ({event['sku']})") +@warehouse.action("order.fulfil") +async def fulfil(ctx, params): + await ctx.resources["store"].fulfill(params["id"], params["sku"]) + print(f"[consumer] fulfilled order {params['id']} ({params['sku']})") + return {"order": params["id"], "status": "fulfilled"} -# ── producers: a Scheduler whose lifespan opens a shared catalog client ────── +# ── producers: a service whose lifespan opens a shared catalog client ──────── @asynccontextmanager async def catalog_lifespan(config): catalog = await connect_catalog() # opened once, shared by all producers @@ -42,12 +46,15 @@ async def catalog_lifespan(config): await catalog.close() -load = Scheduler("load", lifespan=catalog_lifespan) +load = Service("load", lifespan=catalog_lifespan) @load.every("500ms", id="place-orders") async def place_orders(ctx): sku = await ctx.resources["catalog"].low_stock_sku() # producer reads its resource order_id = uuid.uuid4().hex[:8] - await ctx.emit("order.placed", id=order_id, sku=sku) + # dispatch, not call: this side has no use for the result and should not be + # coupled to how long fulfilment takes. The outcome is still recorded — the + # returned task id can be handed to ctx.task_status(...) by anyone who cares. + await ctx.dispatch("order.fulfil", id=order_id, sku=sku) print(f"[producer] placed order {order_id} ({sku})") diff --git a/examples/playwright/main.py b/examples/playwright/main.py index cfcd8e4..ad64d80 100644 --- a/examples/playwright/main.py +++ b/examples/playwright/main.py @@ -1,30 +1,29 @@ -"""Single-process entrypoint — producer AND consumer in one process. +"""Entrypoint for the Playwright load sim. -Producers and consumers are connected only through Redis, so they can run as two -separate deployments (the production scaling pattern) OR together in a single -process, as here. Same App, you just include() both. Handy for local dev and -self-contained demos. +The service holds both the sessions and the schedule that starts them, so one +``include`` is the whole wiring. Splitting the load generator out to its own +deployment would mean a second service and no change to either handler — they are +coupled only by the action's name. Run: `python -m examples.playwright.main` """ import os -from examples.playwright.pipeline import browser_pool, load +from examples.playwright.pipeline import browser from runtime import App def build_app() -> App: app = App(redis=os.environ.get("REDIS_URL", "redis://localhost:6379/0")) app.include( - browser_pool, # consumer (the "how") - enabled=True, + browser, + enabled=True, # a WIRING decision — disabled means the browser never launches config={ "headless": True, "base_url": os.environ.get("TARGET_SITE", "http://localhost:8000"), }, ) - app.include(load, enabled=True) # scheduler (the "when") — same process return app diff --git a/examples/playwright/pipeline.py b/examples/playwright/pipeline.py index efa7be2..cf89c02 100644 --- a/examples/playwright/pipeline.py +++ b/examples/playwright/pipeline.py @@ -1,12 +1,18 @@ """Playwright load sim — simulated users on an e-commerce site. -A single Pool (`browser`, whose lifespan owns a shared browser process) and a -Scheduler (`shopping-load`) that drives sessions at a fixed rate. Run together via -main.py. Playwright is a heavy cost profile, so in production it gets its own -isolated deployment. +One service: its lifespan owns a shared browser process, its action drives one +session, and its producer schedules them. Playwright is a heavy cost profile, so +in production this gets its own deployment — which is exactly what a service +boundary is for. A flood of browser sessions cannot starve an API service's +budget, because the budgets are separate objects. + +``max_slots=12`` is a RAM ceiling, not a throughput target: twelve browser +contexts is what the box can hold. It is also why a handler that blocks on +``ctx.call`` keeps its slot — the context it opened is still resident, whatever +the coroutine is doing. Playwright is imported LAZILY inside the lifespan, so this module imports and -prints topology without it installed. To actually run the flow: +prints its topology without it installed. To actually run the action: pip install -e '.[playwright]' playwright install chromium @@ -15,10 +21,9 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from runtime import Pool, Scheduler +from runtime import Service -# ── consumer: a Pool whose lifespan owns a shared browser process ──────────── @asynccontextmanager async def browser_resource(config: dict) -> AsyncIterator[dict]: from playwright.async_api import async_playwright @@ -32,11 +37,13 @@ async def browser_resource(config: dict) -> AsyncIterator[dict]: await pw.stop() -browser_pool = Pool("browser", max_slots=12, lifespan=browser_resource) +browser = Service("browser", max_slots=12, lifespan=browser_resource) -@browser_pool.flow(consumes="shopping.session") -async def shopping(ctx, event): # HANDLE scope = one isolated user +@browser.action("shopping.session") +async def shopping(ctx, params): + # One isolated user. The service lifespan owns the browser process; per-session + # setup and teardown is a plain try/finally in the handler body. context = await ctx.resources["browser"].new_context() # fresh cookies/cart try: page = await context.new_page() @@ -46,10 +53,9 @@ async def shopping(ctx, event): # HANDLE scope = one isolated user await context.close() -# ── producer: a Scheduler driving sessions (no lifespan needed) ────────────── -load = Scheduler("shopping-load") - - -@load.every("1.5s", id="shopping-load") # ≈ 40/min -async def emit_session(ctx): - await ctx.emit("shopping.session") +@browser.every("1.5s", id="shopping-load") # ≈ 40/min +async def start_session(ctx): + # A ttl says this arrival is only worth simulating while it is fresh: if the + # browser service is saturated, a session queued minutes ago is stale traffic + # and is dropped at claim time rather than run late. + await ctx.dispatch("shopping.session", ttl="30s") diff --git a/examples/poisson/main.py b/examples/poisson/main.py index c1ba932..983edff 100644 --- a/examples/poisson/main.py +++ b/examples/poisson/main.py @@ -1,31 +1,36 @@ -"""Poisson example — a pool that is BOTH a consumer and a producer. +"""Poisson example — a service that consumes AND produces. -The point: a flow handler can itself ``emit``. There's nothing special about -producers — emitting is just a method on the context every handler already has. +The point: there is nothing special about a producer. Dispatching is a method on +the context every handler already has, so a handler can hand work onward exactly +the way a scheduled producer does. That is why services hold both — a domain that +only ever consumed would be unusual. -A scheduler opens a window every 10s. The ``generator`` pool *consumes* that -window event and, per window, *emits* a Poisson-distributed number of ``request`` -events; the ``workers`` pool consumes those. So ``generator`` sits in the middle — -consumer of ``window.open``, producer of ``request`` — wiring two pools together -through nothing but event names. +``generator`` opens a window every 10s, consumes its own window event, and per +window dispatches a Poisson-distributed number of ``request`` actions. ``workers`` +consumes those. The two services are wired together by nothing but a name. python -m examples.poisson.main Needs a Redis on :6379. Single process here; the same App could be split across -deployments (run the scheduler once, scale ``workers`` to N) with no code change. +deployments (run the generator once, scale ``workers`` to N) with no code change. """ import math import os import random -from runtime import App, Pool, Scheduler +from runtime import App, Service def _poisson(lam: float) -> int: """Sample a Poisson(lam) count with Knuth's algorithm — stdlib only, no numpy. This is the number of arrivals in one window (a Poisson process over a fixed - interval); spreading them in real time would use ``random.expovariate``.""" + interval); spreading them in real time would use ``random.expovariate``. + + Note the ceiling: exp(-lam) underflows to 0.0 past lam ≈ 745, so this + saturates rather than scaling. Fine for a demo, not for a rate you intend to + turn up. + """ target = math.exp(-lam) k, p = 0, 1.0 while p > target: @@ -34,40 +39,37 @@ def _poisson(lam: float) -> int: return k - 1 -# ── scheduler: open a window on a fixed interval ──────────────────────────── -clock = Scheduler("clock") +# ── one service: producer of windows, consumer of windows, producer of work ── +generator = Service("generator", max_slots=1) -@clock.every("10s") +@generator.every("10s") async def open_window(ctx): - await ctx.emit("window.open") - - -# ── pool #1: CONSUMER of windows, PRODUCER of requests ────────────────────── -generator = Pool("generator", max_slots=1) + await ctx.dispatch("window.open") -@generator.flow(consumes="window.open") -async def fan_out(ctx, event): +@generator.action("window.open") +async def fan_out(ctx, params): lam = ctx.config.get("lam", 8.0) # mean arrivals per window n = _poisson(lam) print(f"[generator] window → {n} arrival(s)") for seq in range(n): - await ctx.emit("request", seq=seq) # ← the pool acting as a producer + # A handler dispatching onward. Everything it sends carries this + # message's request id, so the whole fan-out stays one traceable request. + await ctx.dispatch("request", seq=seq) -# ── pool #2: CONSUMER of the generated requests ───────────────────────────── -workers = Pool("workers", max_slots=10) +# ── a second service, consumer of the generated work ──────────────────────── +workers = Service("workers", max_slots=10) -@workers.flow(consumes="request") -async def handle(ctx, event): - print(f"[workers] handling request {event['seq']}") +@workers.action("request") +async def handle(ctx, params): + print(f"[workers] handling request {params['seq']} (rid={ctx.request_id[:8]})") def build_app() -> App: app = App(redis=os.environ.get("REDIS_URL", "redis://localhost:6379/0")) - app.include(clock) app.include(generator, config={"lam": 8.0}) app.include(workers) return app diff --git a/examples/rpc/main.py b/examples/rpc/main.py new file mode 100644 index 0000000..0b6fc14 --- /dev/null +++ b/examples/rpc/main.py @@ -0,0 +1,93 @@ +"""RPC example — ``call``: request/reply across the queue. + +``await ctx.call(...)`` sends a request to exactly one executant and suspends +until the reply comes back. The executant may be in this process or on another +machine; nothing in the calling code changes either way, because the only thing +the two sides share is the action's name. + +Three things worth watching for in the output: + + 1. a value comes back through Redis and out of ``await``; + 2. an error raised in the handler is re-raised **in the caller**, with its type + and data intact — not a timeout, not a stringified traceback; + 3. a call that outruns its ``ttl`` fails with CallTimeout, and the executant + drops the request without running it because the deadline had already passed. + + python -m examples.rpc.main +""" + +import asyncio +import os + +from pydantic import BaseModel + +from runtime import ActionError, App, CallTimeout, ParamsInvalid, Service + + +class Quote(BaseModel): + """Declaring a params model means the runtime validates before the handler + runs, and a bad request comes back as a typed ParamsInvalid rather than + whatever KeyError the handler would have raised.""" + + sku: str + qty: int = 1 + + +class OutOfStock(ActionError): + """A domain failure that should reach the caller intact. Subclassing is all + it takes — the code is what rebuilds it on the other side.""" + + CODE = "OUT_OF_STOCK" + + +pricing = Service("pricing", max_slots=4) + + +@pricing.action("pricing.quote", params=Quote) +async def quote(ctx, params): + if params.sku == "SKU-unobtainium": + raise OutOfStock(f"{params.sku} is not available", data={"sku": params.sku}) + return {"sku": params.sku, "total": round(19.99 * params.qty, 2)} + + +@pricing.action("pricing.slow") +async def slow(ctx, params): + await asyncio.sleep(5) + return "you will never see this" + + +client = Service("client") + + +@client.once() +async def demo(ctx): + total = await ctx.call("pricing.quote", ttl="2s", sku="SKU-shoes", qty=3) + print(f"[rpc] quote came back: {total}") + + try: + await ctx.call("pricing.quote", ttl="2s", sku="SKU-unobtainium") + except OutOfStock as e: + print(f"[rpc] domain error, rebuilt caller-side: {e.code} {e.data}") + + try: + await ctx.call("pricing.quote", ttl="2s", sku="SKU-shoes", qty="three") + except ParamsInvalid: + print("[rpc] bad params rejected before the handler ran") + + try: + # The deadline travels with the request. When the worker finally picks it + # up it is already expired, so the handler is never entered. + await ctx.call("pricing.slow", ttl="0.5s") + except CallTimeout as e: + print(f"[rpc] gave up: {e.message}") + + +def build_app() -> App: + app = App(redis=os.environ.get("REDIS_URL", "redis://localhost:6379/0")) + app.include(pricing) + app.include(client) + return app + + +if __name__ == "__main__": + build_app().start() diff --git a/pyproject.toml b/pyproject.toml index 0c363c8..e8c80b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,17 @@ [project] name = "archipellabs-runtime" dynamic = ["version"] -description = "An async Redis runtime for load simulation and orchestration" +description = "Async Redis runtime: services that call, dispatch and emit — Moleculer semantics on Redis Streams" readme = "README.md" requires-python = ">=3.12" license = "MIT" license-files = ["LICENSE"] authors = [{ name = "Loïc Veyssière", email = "loic.veyssiere@archipellabs.com" }] -keywords = ["redis", "asyncio", "streams", "load-testing", "orchestration", "task-queue"] +keywords = [ + "redis", "asyncio", "streams", "task-queue", "orchestration", + "microservices", "rpc", "request-reply", "pubsub", "moleculer", + "job-queue", "load-testing", +] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -21,6 +25,9 @@ classifiers = [ ] dependencies = [ "redis>=5", + # 2.12 is the first release shipping cp314 wheels; the CI matrix includes 3.14, + # and an older pin makes that leg build pydantic-core from source or fail. + "pydantic>=2.12", ] [project.optional-dependencies] @@ -28,6 +35,7 @@ dev = [ "pytest>=7", "pytest-asyncio>=0.23", "pytest-timeout>=2.2", + "pytest-cov>=5", "fakeredis>=2.21", "mypy>=1.8", "ruff>=0.6", @@ -39,6 +47,8 @@ playwright = [ [project.urls] Homepage = "https://archipellabs.com" Repository = "https://github.com/archipellabs/runtime" +Documentation = "https://github.com/archipellabs/runtime/blob/main/doc/index.md" +Changelog = "https://github.com/archipellabs/runtime/blob/main/CHANGELOG.md" Issues = "https://github.com/archipellabs/runtime/issues" [build-system] @@ -72,3 +82,4 @@ line-length = 88 [tool.ruff.lint] select = ["E", "F", "I", "UP", "B"] ignore = ["E501"] # line length is the formatter's job; remaining long lines are intentional aligned comments + diff --git a/runtime/__init__.py b/runtime/__init__.py index 7d61bfe..b8030a3 100644 --- a/runtime/__init__.py +++ b/runtime/__init__.py @@ -1,37 +1,75 @@ """archipellabs-runtime — an async Redis runtime for load simulation and orchestration. Public API. -Two families of objects, connected only by an event name (through a Redis -stream): producers (a `Scheduler`'s `@every`/`@once` bodies) emit; consumers (a -`Pool`'s `@flow` handlers) handle. App.include() wires them; App.start() runs. +A `Service` declares what a domain does: actions others can `call` or `dispatch`, +events it subscribes to, producers that run on a schedule. `App.include()` wires +services; `App.start()` runs them. Producers and consumers are coupled only by a +name, so they can share a process or scale apart without knowing which. + +Three verbs, on every `Context`: + + await ctx.call(action, **params) one executant, a reply, you wait + await ctx.dispatch(action, **params) one executant, a task id, you do not + await ctx.emit(event, **params) every subscriber, no reply """ from runtime._version import __version__ from runtime.app import App -from runtime.broker import Broker -from runtime.context import Context, Handler, ProducerFn -from runtime.pool import FlowRegistration, Pool -from runtime.scheduler import EveryRegistration, OnceRegistration, Scheduler -from runtime.types import Config, Event, Lifespan, Payload, Resources +from runtime.broker import Broker, Delivery, Undecodable +from runtime.context import ActionHandler, Context, EventHandler, Limits, ProducerFn +from runtime.envelope import Envelope +from runtime.errors import ( + ActionError, + CallTimeout, + Disabled, + Expired, + MaxDepth, + ParamsInvalid, + RemoteError, + Undeliverable, +) +from runtime.service import ( + ActionRegistration, + EventRegistration, + EveryRegistration, + OnceRegistration, + Service, +) +from runtime.types import Config, Lifespan, Params, Payload, Resources __all__ = [ "__version__", # core "App", - "Pool", - "Scheduler", + "Service", "Context", - "Handler", + "ActionHandler", + "EventHandler", "ProducerFn", - "FlowRegistration", + "Limits", + # registrations — inspectable, testable without the runtime + "ActionRegistration", + "EventRegistration", "EveryRegistration", "OnceRegistration", # backend seam "Broker", + "Delivery", + "Undecodable", + "Envelope", + # errors + "ActionError", + "CallTimeout", + "Disabled", + "Expired", + "MaxDepth", + "ParamsInvalid", + "RemoteError", + "Undeliverable", # type vocabulary "Lifespan", "Payload", - "Event", + "Params", "Resources", "Config", ] diff --git a/runtime/_version.py b/runtime/_version.py index d3ec452..493f741 100644 --- a/runtime/_version.py +++ b/runtime/_version.py @@ -1 +1 @@ -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/runtime/app.py b/runtime/app.py index cccf380..30f5d58 100644 --- a/runtime/app.py +++ b/runtime/app.py @@ -1,38 +1,47 @@ -"""App — declarative root. include() wires Pools (consumers) AND Schedulers -(producers); start() is the explicit entry point (no CLI). +"""App — the declarative root. ``include()`` wires services; ``start()`` runs them. -start() logs the topology, then runs: it enters each container's lifespan (POOL -scope), creates the consumer group per flow, spawns the slot workers under a -shared pool semaphore, and starts each producer's emit loop — blocking until -interrupted. Graceful drain on SIGTERM is not implemented yet. +``start()`` logs the resolved topology, then serves: it validates the wiring, +opens a connection sized to what it is about to spawn, enters each service's +lifespan, creates the consumer groups, and starts everything — slot workers, +producers, the reply pump, the delay sweeper, the switch refresher — blocking +until interrupted. + +Those last three are ordinary tasks in the same list as the workers, deliberately. +A reply pump that dies unsupervised takes every subsequent ``ctx.call`` in the +process with it, silently, each one waiting out its own deadline. + +Graceful drain on SIGTERM is still not implemented. """ import asyncio import logging -import os -import socket +import random from contextlib import AbstractAsyncContextManager, AsyncExitStack from dataclasses import dataclass -from runtime.broker import Broker, stream_name -from runtime.pool import Pool, slot_worker +from runtime.broker import Broker +from runtime.context import Limits, RuntimeContext +from runtime.duration import to_ms +from runtime.envelope import now_ms +from runtime.node import Node from runtime.redis_io import connect -from runtime.scheduler import Scheduler, run_every, run_once +from runtime.service import BLOCK_MS, Service, run_every, run_once, slot_worker +from runtime.switches import DEFAULT_INTERVAL_S, SwitchBoard from runtime.types import Config, Lifespan, Resources log = logging.getLogger("runtime") - -@dataclass -class _PoolInclusion: - pool: Pool - max_slots: int | None - config: Config | None +_SWEEP_BATCH = 100 +_SWEEP_INTERVAL_S = 0.25 +_SPARE_CONNECTIONS = 8 +"""Headroom over the slot count: the pump, the sweeper, the switch refresher, and +room for a producer to issue a verb while every worker is blocked on a claim.""" @dataclass -class _SchedulerInclusion: - scheduler: Scheduler +class _Inclusion: + service: Service + max_slots: int | None config: Config | None @@ -44,9 +53,9 @@ def _ensure_logging() -> None: async def _enter_lifespan( stack: AsyncExitStack, name: str, lifespan: Lifespan, config: Config ) -> Resources: - """Call the lifespan and enter the async context manager it returns. Guards a - common mistake — forgetting `@asynccontextmanager` — with a clear, named error - at boot instead of a generic protocol failure deep in the stack.""" + """Call the lifespan and enter the context manager it returns. Guards a common + mistake — forgetting `@asynccontextmanager` — with a clear, named error at + boot instead of a generic protocol failure deep in the stack.""" cm = lifespan(config) if not isinstance(cm, AbstractAsyncContextManager): raise TypeError( @@ -56,206 +65,321 @@ async def _enter_lifespan( return await stack.enter_async_context(cm) -def _check_unique_consumes(pools: list[Pool]) -> None: - """One event type → one logical flow, across ALL pools (no logical fan-out). - Pool._validate guards within a pool; this guards across them.""" - owners: dict[str, str] = {} - for pool in pools: - for reg in pool._flows: - owner = f"{pool.name}/{reg.handler.__name__}" - if reg.consumes in owners: +def check_registrations(services: list[Service]) -> None: + """Every rule that can be checked without leaving the process. + + What cannot be checked here: two *processes* claiming the same action name, + or subscribing under the same group with different handlers. Nothing local + can see that — it is the price of having no service registry. + """ + action_owners: dict[str, str] = {} + event_owners: dict[tuple[str, str], str] = {} + event_names: dict[str, str] = {} + + for service in services: + for action in service._actions: + owner = f"{service.name}/{action.handler.__name__}" + if action.name in action_owners: raise ValueError( - f"event type {reg.consumes!r} consumed by both " - f"{owners[reg.consumes]} and {owner} — no logical fan-out" + f"action {action.name!r} is registered by both " + f"{action_owners[action.name]} and {owner} — an action has " + f"exactly one executant" ) - owners[reg.consumes] = owner + action_owners[action.name] = owner + + for event in service._events: + owner = f"{service.name}/{event.handler.__name__}" + key = (event.name, event.group) + if key in event_owners: + raise ValueError( + f"event {event.name!r} is consumed twice under group " + f"{event.group!r} ({event_owners[key]} and {owner}) — they " + f"would compete for each message rather than both running. " + f"Give one of them its own group=..." + ) + event_owners[key] = owner + event_names.setdefault(event.name, owner) + + both = set(action_owners) & set(event_names) + if both: + name = sorted(both)[0] + raise ValueError( + f"{name!r} is registered as both an action ({action_owners[name]}) " + f"and an event ({event_names[name]}). They live on different streams, " + f"so call() and emit() would silently reach different handlers" + ) + + # The master switch addresses a service by name, so a registration sharing + # that name would make one switch mean two things. + registrations = set(action_owners) | set(event_names) + for service in services: + registrations |= {r.id for r in service._every} | {r.id for r in service._once} + clash = {s.name for s in services} & registrations + if clash: + raise ValueError( + f"service name(s) {sorted(clash)} also name a registration — the " + f"master switch could not tell them apart" + ) + + +def check_budgets(inclusions: list[_Inclusion]) -> None: + """A service that consumes needs a budget. There is no sensible default: too + low silently serialises a workload, too high is the memory bound the budget + exists to enforce, quietly removed.""" + for inc in inclusions: + budget = inc.max_slots if inc.max_slots is not None else inc.service.max_slots + if budget is None and inc.service.consumers: + raise ValueError( + f"service {inc.service.name!r} registers " + f"{len(inc.service.consumers)} consumer(s) but has no max_slots — " + f"set it on the Service or at include()" + ) class App: - def __init__(self, *, redis: str, namespace: str = "") -> None: + def __init__( + self, + *, + redis: str, + namespace: str = "", + default_ttl: str | float = "10s", + max_depth: int = 8, + stream_maxlen: int | None = 10_000, + task_ttl: str | float = "1h", + claim_block: str | float = "5s", + sweeper: bool = True, + switch_interval: float = DEFAULT_INTERVAL_S, + ) -> None: self.redis: str = redis - self.namespace: str = ( - namespace # prefixes every stream key — isolate dev/staging on one Redis + self.namespace: str = namespace # prefixes every key — isolates envs + self.limits = Limits( + default_ttl_ms=to_ms(default_ttl, allow_zero=False) or 10_000, + max_depth=max_depth, + stream_maxlen=stream_maxlen, + task_ttl_ms=to_ms(task_ttl, allow_zero=False) or 3_600_000, ) - self._pools: list[_PoolInclusion] = [] - self._schedulers: list[_SchedulerInclusion] = [] + # How long a worker parks waiting for work. Also the granularity of a + # pause: a worker already inside a claim is woken by an arriving message + # and will handle it, so pausing takes full effect within one window. + self.claim_block_ms: int = to_ms(claim_block, allow_zero=False) or BLOCK_MS + # A large fleet does not need every process polling the due set; a few + # dedicated sweepers are enough, and this is how you opt the rest out. + self.sweeper: bool = sweeper + self.switch_interval: float = switch_interval + self._services: list[_Inclusion] = [] def include( self, - component: Pool | Scheduler, + service: Service, *, - enabled: bool = True, # kill-switch (static bool for now) - max_slots: int | None = None, # override the pool budget - config: Config | None = None, # injected into the lifespan + enabled: bool = True, + max_slots: int | None = None, + config: Config | None = None, ) -> None: - """EXPLICIT wiring, in main. An `enabled=False` component is NEITHER - started NOR armed.""" + """Explicit wiring, in main. + + ``enabled=False`` is a *wiring* decision, taken once: the service is not + constructed, its lifespan never runs, and nothing it declares exists in + this process. That is different from a runtime switch, which pauses + something mounted and leaves its resources up. Use this one to keep a + browser from launching at all; use a switch to stop traffic for ten + minutes. + """ if not enabled: return - if isinstance(component, Pool): - self._pools.append(_PoolInclusion(component, max_slots, config)) - elif isinstance(component, Scheduler): - self._schedulers.append(_SchedulerInclusion(component, config)) - else: - raise TypeError( - f"include() expects a Pool or a Scheduler, got {component!r}" - ) + if not isinstance(service, Service): + raise TypeError(f"include() expects a Service, got {service!r}") + self._services.append(_Inclusion(service, max_slots, config)) # ── boot ────────────────────────────────────────────────────────────────── def start(self) -> None: - """Blocking. Log the topology, then run pools' consumers and schedulers' - producers until interrupted (Ctrl-C).""" + """Blocking. Run until interrupted (Ctrl-C).""" self._log_topology() try: asyncio.run(self._serve()) except KeyboardInterrupt: log.info("interrupted — shutting down") + def _budget(self, inc: _Inclusion) -> int: + resolved = inc.max_slots if inc.max_slots is not None else inc.service.max_slots + return resolved if resolved is not None else 1 + + def _slot_count(self) -> int: + return sum( + sum( + reg.max_slots if reg.max_slots is not None else self._budget(inc) + for reg in inc.service.consumers + ) + for inc in self._services + ) + def _log_topology(self) -> None: - # cross-pool uniqueness is validated by _serve (the run path); this is a - # pure logger. _ensure_logging() log.info("App(redis=%s, namespace=%r) — topology:", self.redis, self.namespace) - for pool_inc in self._pools: - pool = pool_inc.pool - budget = ( - pool_inc.max_slots if pool_inc.max_slots is not None else pool.max_slots - ) + for inc in self._services: + service = inc.service + budget = self._budget(inc) override = ( - f" (overridden from {pool.max_slots})" - if pool_inc.max_slots is not None - and pool_inc.max_slots != pool.max_slots + f" (overridden from {service.max_slots})" + if inc.max_slots is not None and inc.max_slots != service.max_slots else "" ) - lifespan = getattr(pool._lifespan, "__name__", None) + lifespan = getattr(service._lifespan, "__name__", None) log.info( - " Pool %r max_slots=%d%s lifespan=%s", - pool.name, + " Service %r max_slots=%d%s lifespan=%s", + service.name, budget, override, lifespan, ) - for reg in pool._flows: + for reg in service.consumers: + slots = reg.max_slots if reg.max_slots is not None else budget + kind = "event" if reg.is_event else "action" + group = f" group={reg.group!r}" if reg.is_event else "" log.info( - " flow %s consumes %r → stream %r slots=%d", + " %s %r → stream %r%s slots=%d (%s)", + kind, + reg.name, + reg.stream, + group, + slots, reg.handler.__name__, - reg.consumes, - stream_name(reg.consumes), - reg.max_slots if reg.max_slots is not None else budget, ) - - for sched_inc in self._schedulers: - sched = sched_inc.scheduler - lifespan = getattr(sched._lifespan, "__name__", None) - log.info(" Scheduler %r lifespan=%s", sched.name, lifespan) - for ereg in sched._every: + for ereg in service._every: log.info( " every %gs id=%r (%s)", ereg.interval, ereg.id, ereg.handler.__name__, ) - for oreg in sched._once: + for oreg in service._once: when = f" after {oreg.delay:g}s" if oreg.delay else "" - log.info( - " once%s id=%r (%s)", - when, - oreg.id, - oreg.handler.__name__, - ) + log.info(" once%s id=%r (%s)", when, oreg.id, oreg.handler.__name__) async def _serve(self, broker: Broker | None = None) -> None: - """Enter container lifespans, spawn slot workers + producer loops, run - until cancelled. Accepts an injected broker for testing; else opens one - from self.redis.""" - _check_unique_consumes([inc.pool for inc in self._pools]) + """Validate, wire, and run until cancelled. Accepts an injected broker for + testing; otherwise opens one sized to the topology.""" + services = [inc.service for inc in self._services] + check_registrations(services) + check_budgets(self._services) + own_broker = broker is None + # Opened *after* the topology walk on purpose: redis-py caps a pool at + # 100 connections, and every blocked worker holds one for its whole claim + # window. Discovering that ceiling under load looks like a crash, not a + # misconfiguration. bk = ( broker if broker is not None - else connect(self.redis, namespace=self.namespace) + else connect( + self.redis, + namespace=self.namespace, + max_connections=self._slot_count() + _SPARE_CONNECTIONS, + max_block_s=self.claim_block_ms / 1000, + ) ) - pod = f"{socket.gethostname()}-{os.getpid()}" + + node = Node(bk) + switches = SwitchBoard(bk, interval_s=self.switch_interval) + await switches.refresh() # start with the truth, not with "all enabled" + try: async with AsyncExitStack() as stack: tasks: list[asyncio.Task[None]] = [] - for inc in self._pools: - pool = inc.pool - budget = ( - inc.max_slots if inc.max_slots is not None else pool.max_slots - ) + for inc in self._services: + service = inc.service + budget = self._budget(inc) config: Config = inc.config if inc.config is not None else {} resources: Resources = {} - if pool._lifespan is not None: + if service._lifespan is not None: resources = await _enter_lifespan( - stack, f"pool {pool.name!r}", pool._lifespan, config + stack, + f"service {service.name!r}", + service._lifespan, + config, ) - pool_sem = asyncio.Semaphore(budget) - for reg in pool._flows: - await bk.ensure_stream(stream_name(reg.consumes)) + ctx = RuntimeContext( + bk, + resources=resources, + config=config, + node=node, + switches=switches, + limits=self.limits, + ) + semaphore = asyncio.Semaphore(budget) + + for reg in service.consumers: + await bk.ensure_group(reg.stream, reg.group, start=reg.start) n = reg.max_slots if reg.max_slots is not None else budget for i in range(n): - consumer = f"{pod}:{reg.consumes}:{i}" + name = f"{node.id}:{reg.name}:{i}" tasks.append( asyncio.create_task( slot_worker( bk, - reg.handler, - consumes=reg.consumes, - consumer=consumer, - resources=resources, - config=config, - pool_sem=pool_sem, + reg, + service_name=service.name, + consumer=name, + ctx=ctx, + semaphore=semaphore, + limits=self.limits, + switches=switches, + block_ms=self.claim_block_ms, ), - name=consumer, + name=name, ) ) - n_slots = len(tasks) - for inc_s in self._schedulers: - sched = inc_s.scheduler - sconfig: Config = inc_s.config if inc_s.config is not None else {} - sresources: Resources = {} - if sched._lifespan is not None: - sresources = await _enter_lifespan( - stack, f"scheduler {sched.name!r}", sched._lifespan, sconfig - ) - for ereg in sched._every: + for ereg in service._every: tasks.append( asyncio.create_task( run_every( - bk, ereg, resources=sresources, config=sconfig + ereg, + ctx=ctx, + service_name=service.name, + switches=switches, ), - name=f"every:{sched.name}:{ereg.id}", + name=f"every:{service.name}:{ereg.id}", ) ) - for oreg in sched._once: + for oreg in service._once: tasks.append( asyncio.create_task( run_once( - bk, oreg, resources=sresources, config=sconfig + oreg, + ctx=ctx, + service_name=service.name, + switches=switches, ), - name=f"once:{sched.name}:{oreg.id}", + name=f"once:{service.name}:{oreg.id}", ) ) - n_producers = len(tasks) - n_slots - if not tasks: - log.info("nothing to run") - return + n_work = len(tasks) + tasks.append(asyncio.create_task(node.pump(), name="reply-pump")) + tasks.append(asyncio.create_task(switches.run(), name="switches")) + if self.sweeper: + tasks.append( + asyncio.create_task( + run_sweeper(bk, maxlen=self.limits.stream_maxlen), + name="sweeper", + ) + ) + log.info( - "serving %d task(s): %d consumer slot(s) + %d producer(s)", + "serving %d task(s): %d worker(s)/producer(s) + %d runtime task(s)", len(tasks), - n_slots, - n_producers, + n_work, + len(tasks) - n_work, ) try: await asyncio.gather(*tasks) - except asyncio.CancelledError: - # Ctrl-C cancels us; cancel the workers and wait for them to - # unwind (return_exceptions so a worker's teardown error can't - # mask the shutdown) before the lifespans and broker close. + except BaseException: + # Any failure, not just Ctrl-C. If a task raises and this + # only caught CancelledError, the drain would be skipped and + # AsyncExitStack would tear down lifespans while other + # handlers were still using ctx.resources. log.info("draining %d task(s)", len(tasks)) for t in tasks: t.cancel() @@ -264,3 +388,24 @@ async def _serve(self, broker: Broker | None = None) -> None: finally: if own_broker: await bk.aclose() + + +async def run_sweeper( + broker: Broker, + *, + maxlen: int | None, + interval_s: float = _SWEEP_INTERVAL_S, + batch: int = _SWEEP_BATCH, +) -> None: + """Move delayed messages onto their streams once they come due. + + Every process runs one; ``ZREM`` decides which of them owns each message, so + they cooperate without coordinating. A full batch loops straight round rather + than sleeping, so the batch size caps the *idle* poll rate and not throughput. + The sleep is jittered so N processes do not all wake on the same boundary. + """ + while True: + moved = await broker.sweep(now_ms(), limit=batch, maxlen=maxlen) + if moved >= batch: + continue + await asyncio.sleep(interval_s * (0.5 + random.random())) diff --git a/runtime/broker.py b/runtime/broker.py index 985f72f..ba4acf9 100644 --- a/runtime/broker.py +++ b/runtime/broker.py @@ -1,56 +1,172 @@ -"""Broker — the message-transport seam. +"""Broker — the message-transport seam, plus the key naming that routes onto it. -The runtime depends only on this Protocol; the concrete backend (Redis Streams -today, in ``redis_io.py``) is chosen at the App wiring root. The queue is the -only seam between producers and consumers, made literal here: ``append`` -produces, ``claim``/``ack`` consume with at-least-once delivery. +The runtime depends only on this Protocol; the concrete backend (Redis, in +``redis_io.py``) is chosen at the App wiring root. Everything that touches Redis +goes through here — including the reply channel, the task records, the delay set +and the switch registry. That is deliberate: several boot tests hand +``App._serve`` a hand-written fake broker and a deliberately unusable URL, so any +subsystem that reached past this seam for its own connection would drag a real +Redis back into tests that have no business needing one. -``stream_name`` lives here because it is backend-agnostic routing — which logical -stream an event type maps to (1:1 with a flow). It is deterministic, so a -producer in one process and a consumer in another agree on the stream by sharing -only the event-type string. +The naming functions live here rather than in the Redis module because they are +backend-agnostic routing: which logical stream carries a name, which group a +subscriber joins. They are deterministic, so a producer in one process and a +consumer in another agree by sharing only a string. + +Two shapes of destination, and the difference is the whole point: + + act:{name} one stream, ONE group → exactly one executant (call, dispatch) + evt:{name} one stream, N groups → one copy per subscriber (emit) """ -from typing import Final, Protocol +from dataclasses import dataclass +from typing import Any, Final, Protocol + +from runtime.envelope import Envelope, Kind +from runtime.types import Params, Payload + +ACTION_PREFIX: Final = "act" +EVENT_PREFIX: Final = "evt" + +WORKERS_GROUP: Final = "workers" +"""The single consumer group on every action stream. One group means Redis hands +each request to exactly one member, which is what makes an action have one +executant however many replicas are running.""" + +DUE_KEY: Final = "due" +SWITCHES_KEY: Final = "switches" + +START_TAIL: Final = "$" +START_HEAD: Final = "0" + + +def action_stream(name: str) -> str: + return f"{ACTION_PREFIX}:{name}" + + +def event_stream(name: str) -> str: + return f"{EVENT_PREFIX}:{name}" + + +def stream_for(kind: Kind, name: str) -> str: + """Where a message goes, from its envelope alone — so the delay sweeper can + route a message whose registration it has never seen.""" + return event_stream(name) if kind == "event" else action_stream(name) + + +def reply_key(node_id: str) -> str: + """One reply channel per process, not one per call. Correlation happens in + memory against the envelope id.""" + return f"reply:{node_id}" + + +def task_key(task_id: str) -> str: + return f"task:{task_id}" + -from runtime.types import Event, Payload +@dataclass(frozen=True) +class Delivery: + """One message, decoded.""" -STREAM_PREFIX: Final = "flow" + id: str + """The broker's message id — what ``ack`` needs, not the envelope id.""" + envelope: Envelope + params: Params -def stream_name(event_type: str) -> str: - """The logical stream that carries a given event type (1:1 with a flow).""" - return f"{STREAM_PREFIX}:{event_type}" +@dataclass(frozen=True) +class Undecodable: + """Bytes arrived that are not a message this version understands. + + A separate return type rather than an exception, because the worker has to + *act* on it — ack it and log it, so it does not sit in the pending list + forever. Raising out of ``claim`` would take the rest of the batch with it. + """ + + id: str + reason: str class Broker(Protocol): - """What the runtime needs from a queue backend — a consumer-group-style - at-least-once contract. A Redis-Streams impl lives in ``redis_io.py``; a SQL - impl (e.g. Postgres ``SELECT … FOR UPDATE SKIP LOCKED``) would implement the - same five methods. Reclaim of stale-unacked messages is not implemented yet.""" + """What the runtime needs from its backend. + + A SQL implementation is plausible for all of it: ``SELECT … FOR UPDATE SKIP + LOCKED`` for the streams, a table plus LISTEN/NOTIFY for replies, a + ``due_at`` column for the delay set, a row per task. - async def append(self, stream: str, payload: Payload) -> str: - """Produce: add an event to a stream. Returns the message id.""" + Reclaim of stale-unacked messages is not implemented — see the README on what + at-least-once does and does not currently buy you. + """ + + # ── streams ─────────────────────────────────────────────────────────────── + async def append( + self, + stream: str, + envelope: Envelope, + payload: Payload, + *, + maxlen: int | None = None, + ) -> str: + """Produce: add a message. Envelope and payload are stored separately so + a reader can check a deadline without decoding a body it may be about to + discard. Returns the broker message id.""" ... - async def ensure_stream(self, stream: str) -> None: - """Idempotently set up a stream for consumption (Redis: create the - consumer group with MKSTREAM; SQL: ensure the table/partition).""" + async def ensure_group(self, stream: str, group: str, *, start: str) -> None: + """Idempotently create the stream and a consumer group on it. ``start`` is + ``START_TAIL`` for a new subscriber (see only what happens from now on) or + ``START_HEAD`` to replay everything retained.""" ... async def claim( - self, stream: str, *, consumer: str, count: int, block_ms: int - ) -> list[tuple[str, Event]]: + self, stream: str, group: str, *, consumer: str, count: int, block_ms: int + ) -> list[Delivery | Undecodable]: """Hand up to ``count`` never-yet-delivered messages to ``consumer``, - parking up to ``block_ms`` if the stream is empty. Returns - ``[(message_id, event), ...]``; empty on timeout. An un-acked message - stays pending for redelivery (reclaim is not implemented yet).""" + parking up to ``block_ms`` if the stream is empty. Empty list on timeout. + An un-acked message stays pending.""" ... - async def ack(self, stream: str, message_id: str) -> None: - """Mark a message done (Redis: XACK — drops it from the pending set).""" + async def ack(self, stream: str, group: str, message_id: str) -> None: + """Mark a message done — drop it from this group's pending set.""" ... - async def aclose(self) -> None: - """Close the underlying connection.""" + # ── reply rendezvous ────────────────────────────────────────────────────── + async def push_reply(self, key: str, reply: dict[str, Any], *, ttl_ms: int) -> None: + """Deliver a call's outcome to the channel the caller named. The TTL is + what stops a dead caller's channel accumulating forever.""" ... + + async def pop_reply(self, key: str, *, timeout_s: float) -> dict[str, Any] | None: + """Block for one reply, or None if the window elapsed.""" + ... + + # ── task records ────────────────────────────────────────────────────────── + async def put_task(self, key: str, record: dict[str, Any], *, ttl_ms: int) -> None: + """Write a dispatch's status. Whole-record write, not a partial update — + the states are few and a transition always knows everything it needs.""" + ... + + async def get_task(self, key: str) -> dict[str, Any] | None: ... + + # ── delayed delivery ────────────────────────────────────────────────────── + async def schedule( + self, due_ms: int, stream: str, envelope: Envelope, payload: Payload + ) -> None: + """Hold a message until ``due_ms``, after which it becomes ordinary.""" + ... + + async def sweep(self, now_ms: int, *, limit: int, maxlen: int | None) -> int: + """Move everything now due onto its stream. Safe to run in every process + concurrently — whoever removes a message from the due set owns it. + Returns how many *this* process moved.""" + ... + + # ── switches ────────────────────────────────────────────────────────────── + async def switches_get_all(self) -> dict[str, bool]: + """The whole registry in one round trip. Absent means enabled, so this + only ever reports what has been explicitly paused.""" + ... + + async def switch_set(self, name: str, enabled: bool) -> None: ... + + async def aclose(self) -> None: ... diff --git a/runtime/context.py b/runtime/context.py index c66e4ff..6f21106 100644 --- a/runtime/context.py +++ b/runtime/context.py @@ -1,56 +1,182 @@ -"""Context — what handlers and producers receive: resource access + emit. - -`Context` is the type contract (a Protocol). `Handler` is the consumer type (an -async `(ctx, event)`); `ProducerFn` is the producer type (an async `(ctx)`). -`RuntimeContext` is the concrete implementation the runtime hands to both: it -carries the container's shared resources + the injected config, and routes -`emit` to the right stream. A consumer slot and a producer each get one with -their container's lifespan resources (or an empty mapping if no lifespan). +"""Context — what handlers and producers hold: resources, config, and the verbs. + +Three verbs, differing on two axes worth keeping straight, because the transport +is the same for all of them and only the intent differs: + + call() one executant a reply, awaited caller coupled in time + dispatch() one executant no reply, a task id caller not coupled + emit() 0..N reactors no reply fan-out + +`call` and `dispatch` are the same message on the same stream through the same +consumer group; the only difference is whether a reply channel is named. `emit` +differs in kind — N independent subscribers, each getting a copy. + +Everything issued from inside a handler inherits that handler's envelope: same +request id, one level deeper, provenance recorded, metadata carried, budget +narrowed. That is what stops the audit chain breaking the moment a flow turns +asynchronous — a `dispatch` fired from inside a `call` is still visibly part of +the same request. """ +import asyncio +import logging from collections.abc import Awaitable, Callable -from typing import Any, Protocol +from dataclasses import dataclass +from typing import Any, Final, Protocol + +from runtime.broker import Broker, stream_for, task_key +from runtime.duration import to_ms +from runtime.envelope import Envelope, make_envelope, new_id, now_ms +from runtime.errors import CallTimeout, Disabled, MaxDepth +from runtime.node import Node +from runtime.switches import SwitchBoard +from runtime.types import Config, Payload, Resources + +log = logging.getLogger("runtime") + +TASK_PENDING: Final = "pending" +TASK_DONE: Final = "done" +TASK_FAILED: Final = "failed" +TASK_EXPIRED: Final = "expired" + -from runtime.broker import Broker, stream_name -from runtime.types import Config, Event, Resources +@dataclass(frozen=True) +class Limits: + """Process-wide knobs, set on the App and threaded down to every Context.""" + + default_ttl_ms: int = 10_000 + """Budget for a ``call`` that does not name one. Deliberately short — an + unbounded call holds a worker slot indefinitely, and a long default makes + that failure look like a hang.""" + + max_depth: int = 8 + """How deep a chain of calls may go before it is refused. Bounds the runaway + case — an action that calls itself — without pretending to solve the general + cycle problem.""" + + stream_maxlen: int | None = 10_000 + """Approximate cap on every stream. Acking does not delete, and durable + fan-out means entries outlive all their readers, so without this every + stream grows forever.""" + + task_ttl_ms: int = 3_600_000 class Context(Protocol): - """Passed to every handler and producer. Gives access to shared resources - and to emit.""" + """Passed to every handler and producer.""" resources: Resources - """Resources exposed by the container's lifespan (POOL scope). - E.g. ``ctx.resources["browser"]``. Shared by everything in the container.""" + """What the service's lifespan yielded — shared across everything in it.""" config: Config - """Config injected at mount time via ``App.include(config=...)``.""" + """Injected at mount time via ``App.include(config=...)``.""" + + params: Any + """The body being handled — the same object the handler received as its + second argument. A plain dict, or an instance of the registration's ``params`` + model when it declares one. Empty for a producer, which handles nothing.""" + + # Read-only: these describe the message being handled, so nothing downstream + # gets to rewrite its own provenance. A plain attribute on a test double + # still satisfies them. + @property + def meta(self) -> dict[str, Any]: + """User metadata that arrived with the message, and that propagates to + whatever this handler sends.""" + ... + + @property + def request_id(self) -> str: + """Shared by every message descended from one origin — the audit spine.""" + ... + + @property + def level(self) -> int: + """How deep in a call chain this message is. 0 straight from a producer.""" + ... + + @property + def caller(self) -> str | None: + """The action or event whose handler produced this message. None from a + producer, which has nothing above it.""" + ... + + async def call( + self, + action: str, + /, + *, + ttl: str | float | None = None, + delay: str | float | None = None, + meta: dict[str, Any] | None = None, + **params: Any, + ) -> Any: + """Ask one executant for a result and wait for it. Raises whatever the + handler raised, ``CallTimeout`` if the budget ran out, or ``Disabled`` if + the target is paused.""" + ... - async def emit(self, event_type: str, /, **payload: Any) -> None: - """Push an event into the queue. Mostly producer-side, but a handler may - re-emit (chaining). (Deduplication via a deterministic job-id per bucket, - for idempotent producers, is planned.)""" + async def dispatch( + self, + action: str, + /, + *, + ttl: str | float | None = None, + delay: str | float | None = None, + meta: dict[str, Any] | None = None, + **params: Any, + ) -> str: + """Hand work to one executant without waiting. Returns a task id that + ``task_status`` can be asked about.""" ... + async def emit( + self, + event: str, + /, + *, + ttl: str | float | None = None, + delay: str | float | None = None, + meta: dict[str, Any] | None = None, + **params: Any, + ) -> None: + """Announce something to every subscriber. No reply, no executant — if + nobody subscribes, nothing happens and nothing is wrong.""" + ... -Handler = Callable[[Context, Event], Awaitable[None]] -"""A flow: an async ``(ctx, event)`` handler for ONE event type. One worker -(slot) calls it sequentially per event; the pool semaphore bounds how many run -at once. Must be idempotent (AT-LEAST-ONCE → replay possible at reclaim). Two -scopes only: POOL (shared resources via the lifespan, reached as ctx.resources) -and HANDLE (per-event setup/teardown in the body, a plain try/finally).""" + async def task_status(self, task_id: str) -> dict[str, Any] | None: ... + + def remaining(self) -> float | None: + """Seconds of budget left on the message being handled, or None if it has + no deadline. Goes negative once past it.""" + ... + + def is_enabled(self, name: str) -> bool: ... + + async def set_enabled(self, name: str, enabled: bool) -> None: ... + + +ActionHandler = Callable[[Context, Any], Awaitable[Any]] +"""An action: an async ``(ctx, params)`` whose return value becomes the reply. +One worker runs it per message; the service semaphore bounds how many run at +once. Must be idempotent — delivery is at-least-once.""" + +EventHandler = Callable[[Context, Any], Awaitable[None]] +"""An event subscription. Same shape as an action, but nothing is listening for a +return value, so it has none.""" ProducerFn = Callable[[Context], Awaitable[None]] -"""A producer: an async ``(ctx)`` body that emits events. A scheduler calls it on -a schedule (``@every`` interval, or ``@once``); shared resources (an API/DB -client) come from the scheduler's lifespan, reached as ctx.resources.""" +"""A producer: an async ``(ctx)`` the runtime calls on a schedule.""" class RuntimeContext: - """Concrete Context. ``emit`` resolves event type → stream and appends via the - broker — no dedup yet. Emitting an event type that has no consumer in this - process is fine and expected (a producer-only process emits to streams its - own process never reads).""" + """The concrete Context. + + One is built per worker slot and per producer, carrying the service's shared + resources and config for its whole life. The per-message parts are attached + with ``for_message``, so a worker does not rebuild the shared half for every + message it handles. + """ def __init__( self, @@ -58,10 +184,195 @@ def __init__( *, resources: Resources | None = None, config: Config | None = None, + node: Node | None = None, + switches: SwitchBoard | None = None, + limits: Limits | None = None, + envelope: Envelope | None = None, + params: Any = None, ) -> None: self.resources: Resources = resources if resources is not None else {} self.config: Config = config if config is not None else {} - self._broker: Broker = broker + self.params: Any = params if params is not None else {} + self._broker = broker + self._node = node + self._switches = switches + self._limits = limits if limits is not None else Limits() + self._envelope = envelope + + # ── what the message being handled says about itself ────────────────────── + @property + def meta(self) -> dict[str, Any]: + return self._envelope.meta if self._envelope is not None else {} + + @property + def request_id(self) -> str: + return self._envelope.rid if self._envelope is not None else "" + + @property + def level(self) -> int: + return self._envelope.lvl if self._envelope is not None else 0 + + @property + def caller(self) -> str | None: + return self._envelope.caller if self._envelope is not None else None + + def remaining(self) -> float | None: + if self._envelope is None: + return None + left = self._envelope.remaining_ms() + return None if left is None else left / 1000 + + def for_message(self, envelope: Envelope, params: Any) -> "RuntimeContext": + """A view bound to one message. Cheap — the shared half is by reference.""" + return RuntimeContext( + self._broker, + resources=self.resources, + config=self.config, + node=self._node, + switches=self._switches, + limits=self._limits, + envelope=envelope, + params=params, + ) + + # ── switches ────────────────────────────────────────────────────────────── + def is_enabled(self, name: str) -> bool: + return self._switches is None or self._switches.is_enabled(name) + + async def set_enabled(self, name: str, enabled: bool) -> None: + if self._switches is None: + raise RuntimeError("no switchboard on this context — build it via App") + await self._switches.set(name, enabled) + + # ── the verbs ───────────────────────────────────────────────────────────── + async def call( + self, + action: str, + /, + *, + ttl: str | float | None = None, + delay: str | float | None = None, + meta: dict[str, Any] | None = None, + **params: Any, + ) -> Any: + if self._node is None: + raise RuntimeError( + "call() needs a rendezvous — build the Context through App, or " + "use dispatch()/emit(), which do not wait for a reply" + ) + # Fail before queueing rather than waiting out the budget on a consumer + # that is deliberately not listening. Only `call` checks: deferring work + # for a paused consumer is exactly what dispatch and emit are for. + if not self.is_enabled(action): + raise Disabled(f"{action} is paused") + + ttl_ms = self._limits.default_ttl_ms if ttl is None else to_ms(ttl) + if not ttl_ms: + raise ValueError( + "call() needs a deadline: ttl=0 would hold a worker slot forever" + ) + delay_ms = to_ms(delay) + envelope = make_envelope( + kind="call", + name=action, + parent=self._envelope, + ttl_ms=ttl_ms, + delay_ms=delay_ms, + meta=meta, + reply=self._node.reply_key, + ) + self._guard_depth(envelope) + + future = self._node.register(envelope.id) + timeout_s = self._wait_budget_s(envelope) + try: + await self._put(envelope, params, delay_ms) + return await asyncio.wait_for(future, timeout_s) + except TimeoutError: + raise CallTimeout( + f"{action} did not reply within {timeout_s:.3g}s" + ) from None + finally: + self._node.discard(envelope.id) + + async def dispatch( + self, + action: str, + /, + *, + ttl: str | float | None = None, + delay: str | float | None = None, + meta: dict[str, Any] | None = None, + **params: Any, + ) -> str: + task_id = new_id() + delay_ms = to_ms(delay) + envelope = make_envelope( + kind="dispatch", + name=action, + parent=self._envelope, + ttl_ms=to_ms(ttl), + delay_ms=delay_ms, + meta=meta, + task=task_id, + ) + self._guard_depth(envelope) + await self._broker.put_task( + task_key(task_id), + {"state": TASK_PENDING, "action": action}, + ttl_ms=self._limits.task_ttl_ms, + ) + await self._put(envelope, params, delay_ms) + return task_id + + async def emit( + self, + event: str, + /, + *, + ttl: str | float | None = None, + delay: str | float | None = None, + meta: dict[str, Any] | None = None, + **params: Any, + ) -> None: + delay_ms = to_ms(delay) + envelope = make_envelope( + kind="event", + name=event, + parent=self._envelope, + ttl_ms=to_ms(ttl), + delay_ms=delay_ms, + meta=meta, + ) + self._guard_depth(envelope) + await self._put(envelope, params, delay_ms) + + async def task_status(self, task_id: str) -> dict[str, Any] | None: + return await self._broker.get_task(task_key(task_id)) + + # ── internals ───────────────────────────────────────────────────────────── + def _guard_depth(self, envelope: Envelope) -> None: + if envelope.lvl > self._limits.max_depth: + raise MaxDepth( + f"{envelope.name} is {envelope.lvl} deep " + f"(max_depth={self._limits.max_depth}) — probably a cycle", + data={"rid": envelope.rid, "level": envelope.lvl}, + ) + + async def _put( + self, envelope: Envelope, payload: Payload, delay_ms: int | None + ) -> None: + stream = stream_for(envelope.kind, envelope.name) + if delay_ms: + await self._broker.schedule(now_ms() + delay_ms, stream, envelope, payload) + else: + await self._broker.append( + stream, envelope, payload, maxlen=self._limits.stream_maxlen + ) - async def emit(self, event_type: str, /, **payload: Any) -> None: - await self._broker.append(stream_name(event_type), payload) + def _wait_budget_s(self, envelope: Envelope) -> float: + """How long to wait for this call's reply. Taken from the envelope so an + inherited (narrower) parent budget governs the wait as well as the + message, and so a delayed call waits out its delay too.""" + left = envelope.remaining_ms() + return self._limits.default_ttl_ms / 1000 if left is None else left / 1000 diff --git a/runtime/duration.py b/runtime/duration.py new file mode 100644 index 0000000..e8a8fa7 --- /dev/null +++ b/runtime/duration.py @@ -0,0 +1,54 @@ +"""Duration parsing — the one place a human-written interval becomes a number. + +Shared by the scheduler (``@every("5m")``) and the envelope (``ttl="30s"``, +``delay="10s"``). It lives in its own module because the envelope cannot import +the service container — the container imports the context, which imports the +envelope — and duplicating a unit table is how ``"1m"`` ends up meaning a minute +in one file and a millisecond in another. + +Deadlines are epoch milliseconds everywhere in this runtime, so ``to_ms`` is the +form most callers want; ``parse_duration`` returns seconds because that is what +``asyncio.sleep`` takes. +""" + +from typing import Final + +# Longer suffixes first so "ms"/"min" win over "s"/"m". +_UNITS: Final[list[tuple[str, float]]] = [ + ("ms", 0.001), + ("min", 60.0), + ("h", 3600.0), + ("s", 1.0), + ("m", 60.0), +] + + +def parse_duration(value: str | float, *, allow_zero: bool = False) -> float: + """Duration → seconds. Accepts a number (seconds) or a string with a unit + suffix: ``"500ms"``, ``"1.5s"``, ``"10min"``, ``"1h"`` (bare number = seconds). + Intervals must be > 0; a `once` delay or a ``ttl`` may be 0 — pass + ``allow_zero=True`` for those.""" + if isinstance(value, (int, float)): + seconds = float(value) + else: + s = value.strip() + for suffix, mult in _UNITS: + if s.endswith(suffix): + seconds = float(s[: -len(suffix)]) * mult + break + else: + seconds = float(s) # bare number string → seconds (raises on garbage) + if seconds < 0 or (seconds == 0 and not allow_zero): + raise ValueError( + f"duration must be {'>= 0' if allow_zero else '> 0'}: {value!r}" + ) + return seconds + + +def to_ms(value: str | float | None, *, allow_zero: bool = True) -> int | None: + """Duration → whole milliseconds, or None if the caller passed None. The + None-in/None-out shape lets an optional ``ttl=``/``delay=`` flow straight + through without every call site writing the same conditional.""" + if value is None: + return None + return round(parse_duration(value, allow_zero=allow_zero) * 1000) diff --git a/runtime/envelope.py b/runtime/envelope.py new file mode 100644 index 0000000..541c7fe --- /dev/null +++ b/runtime/envelope.py @@ -0,0 +1,143 @@ +"""The envelope — everything about a message that is not its body. + +Serialized into its own stream field (``meta``), beside the untouched payload in +``data``. Keeping them apart means a sweeper checking a due time, or a worker +checking a deadline, never has to decode the body it is not going to read. + +The envelope is also what makes a chain of calls legible: ``rid`` is the same for +every message descended from one origin, ``lvl`` says how deep, ``caller`` says +what produced this one. That is the spine of the audit trail — a `dispatch` or an +`emit` issued from inside a handler carries it forward exactly as a `call` does, +so the chain does not break the moment a flow turns asynchronous. + + COMPATIBILITY RULE — every field added after v0.3 MUST have a default. + +There is no dead-letter queue. One required field added later turns every message +from a producer that has not been redeployed into a poison pill that no consumer +can decode and nothing can quarantine. + +All times here are epoch **milliseconds**, with no exceptions anywhere in the +runtime. Mixing units across a wire format is a silent factor-of-1000 bug. +""" + +import time +import uuid +from typing import Any, Final, Literal + +from pydantic import BaseModel, ConfigDict, Field + +ENVELOPE_VERSION: Final = 3 + +type Kind = Literal["call", "dispatch", "event"] +"""Which verb produced this message. `call` and `dispatch` share one stream and +one consumer group (exactly one executant); `event` fans out to one group per +subscriber. The kind also decides what the worker does with the outcome: reply, +task hash, or nothing.""" + + +def now_ms() -> int: + """Wall-clock epoch milliseconds. Wall clock and not `monotonic` because a + deadline set on one host is compared on another — which is exactly why + deadlines assume roughly-synced clocks (see README).""" + return int(time.time() * 1000) + + +def new_id() -> str: + return uuid.uuid4().hex + + +class Envelope(BaseModel): + """What travels in the ``meta`` stream field. + + ``extra="ignore"`` is set explicitly rather than left to pydantic's default, + because it is the half of the compatibility rule that lets an old worker read + a new producer's message. The other half is the defaults. + """ + + model_config = ConfigDict(extra="ignore") + + v: int = ENVELOPE_VERSION + id: str + rid: str + """Request id — shared by every message descended from one origin.""" + lvl: int = 0 + """Call depth. 0 for anything a producer emits; +1 per hop. Gates max_depth.""" + kind: Kind + name: str + caller: str | None = None + """The action or event whose handler produced this message; None from a producer.""" + reply: str | None = None + """Reply key. Set only for kind="call" — its presence is what tells a worker + that someone is waiting.""" + dl: int | None = None + """Absolute deadline, epoch ms. None means no deadline.""" + meta: dict[str, Any] = Field(default_factory=dict) + """User metadata. Propagated to children (shallow merge, child wins).""" + task: str | None = None + """Task id. Set only for kind="dispatch".""" + + def remaining_ms(self, *, at: int | None = None) -> int | None: + """Budget left, or None if there is no deadline. Can go negative — the + caller decides whether that means 'drop it' or 'already too late'.""" + if self.dl is None: + return None + return self.dl - (now_ms() if at is None else at) + + +def derive_deadline( + parent_dl: int | None, ttl_ms: int | None, *, base_ms: int +) -> int | None: + """The child's deadline, given the parent's and an optional own ttl. + + | parent | ttl | result | + |--------|------------|-------------------------------------------| + | None | t | base + t | + | d | t | min(d, base + t) — can only narrow | + | d | 0 / None | d — inherited unchanged | + | None | 0 / None | None | + + A falsy ttl inherits rather than clearing: ``ttl=0`` must never *widen* a + budget it was handed, or one careless leaf would let a chain outlive the + request that started it. ``base_ms`` is the due time, not the emit time, so + a delayed message's budget starts when it becomes deliverable. + """ + if not ttl_ms: + return parent_dl + own = base_ms + ttl_ms + return own if parent_dl is None else min(parent_dl, own) + + +def make_envelope( + *, + kind: Kind, + name: str, + parent: Envelope | None = None, + ttl_ms: int | None = None, + delay_ms: int | None = None, + meta: dict[str, Any] | None = None, + reply: str | None = None, + task: str | None = None, +) -> Envelope: + """Build the envelope for an outgoing message, inheriting from the one being + handled (``parent``) when there is one. + + Events deliberately do **not** inherit the parent's deadline. An event has no + outcome channel, so an inherited deadline means a saturated subscriber drops + notifications silently with nothing anywhere to observe it — under exactly the + load this runtime exists to generate. An event that genuinely should expire + says so with its own ``ttl``. + """ + base_ms = now_ms() + (delay_ms or 0) + inherited_dl = parent.dl if (parent is not None and kind != "event") else None + return Envelope( + id=new_id(), + rid=parent.rid if parent is not None else new_id(), + lvl=parent.lvl + 1 if parent is not None else 0, + kind=kind, + name=name, + caller=parent.name if parent is not None else None, + reply=reply, + dl=derive_deadline(inherited_dl, ttl_ms, base_ms=base_ms), + meta={**(parent.meta if parent is not None else {}), **(meta or {})}, + task=task, + ) diff --git a/runtime/errors.py b/runtime/errors.py new file mode 100644 index 0000000..b1eac16 --- /dev/null +++ b/runtime/errors.py @@ -0,0 +1,128 @@ +"""Typed errors — the failure half of the request/reply contract. + +An action either returns a value or fails. When it fails, the failure has to +survive a trip through Redis and be raised again in the caller's process, so it +is serialized to three fields and rebuilt on the other side. ``await ctx.call()`` +therefore raises something meaningful instead of timing out or handing back a +stringified traceback. + +``code`` is the discriminator. Each subclass claims one and registers itself, so +``from_wire`` can rebuild the right class; a code this process has never heard of +(an executant running a newer version) degrades to ``RemoteError`` carrying the +original code rather than failing to import. + +Moleculer also puts ``type`` and ``retryable`` on the wire. Both are omitted +here on purpose: nothing in this version retries, and a wire field with no +consumer is a field you cannot change once producers exist. +""" + +from typing import Any, ClassVar + +from runtime.types import Payload + + +class ActionError(Exception): + """Base for every failure that can cross the queue. + + Subclass it in application code when a domain failure should reach the + caller intact — set ``CODE`` to something unique and the registry does the + rest. Anything that is *not* an ActionError still crosses (see ``wrap``), it + just arrives as a ``RemoteError``. + """ + + CODE: ClassVar[str] = "ERROR" + + _registry: ClassVar[dict[str, type["ActionError"]]] = {} + + def __init__( + self, message: str, *, data: Payload | None = None, code: str | None = None + ) -> None: + super().__init__(message) + self.message: str = message + self.data: Payload = data if data is not None else {} + # Instance-level so RemoteError can carry a code it does not own. + self.code: str = code if code is not None else self.CODE + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + ActionError._registry[cls.CODE] = cls + + def to_wire(self) -> dict[str, Any]: + return {"code": self.code, "message": self.message, "data": self.data} + + @classmethod + def from_wire(cls, wire: dict[str, Any]) -> "ActionError": + """Rebuild the error a worker reported. An unrecognised code becomes a + RemoteError that still carries it, so a caller can branch on + ``err.code`` even for classes it does not have.""" + code = str(wire.get("code", RemoteError.CODE)) + message = str(wire.get("message", "")) + data = wire.get("data") or {} + klass = cls._registry.get(code) + if klass is None: + return RemoteError(message, data=data, code=code) + return klass(message, data=data) + + @classmethod + def wrap(cls, exc: BaseException) -> "ActionError": + """Whatever a handler raised, as something reportable. An ActionError + passes through untouched; anything else keeps its class name in the + message and in ``data`` so the caller can still tell a ValueError from a + KeyError without sharing code with the executant.""" + if isinstance(exc, ActionError): + return exc + name = type(exc).__name__ + return RemoteError(f"{name}: {exc}", data={"exception": name}) + + +class RemoteError(ActionError): + """The handler failed with something that is not an ActionError, or with a + code this process does not recognise.""" + + CODE: ClassVar[str] = "REMOTE" + + +class CallTimeout(ActionError): + """The caller's budget ran out before a reply arrived. Says nothing about + whether the work ran — it may still be executing, or may never have been + claimed.""" + + CODE: ClassVar[str] = "TIMEOUT" + + +class Expired(ActionError): + """The message was claimed after its deadline had already passed, so the + handler was never called. Distinct from CallTimeout: this is the executant's + verdict, and it means the work definitely did not run.""" + + CODE: ClassVar[str] = "EXPIRED" + + +class ParamsInvalid(ActionError): + """The payload did not satisfy the action's ``params`` model. Never worth + redelivering — the same bytes will fail the same way.""" + + CODE: ClassVar[str] = "PARAMS_INVALID" + + +class Disabled(ActionError): + """The target is paused by a runtime switch. Raised at the caller before + anything is queued, because waiting for a consumer that is deliberately not + listening only burns the budget.""" + + CODE: ClassVar[str] = "DISABLED" + + +class MaxDepth(ActionError): + """The call chain exceeded ``App(max_depth=...)``. Almost always an action + that calls itself, directly or through a cycle.""" + + CODE: ClassVar[str] = "MAX_DEPTH" + + +class Undeliverable(ActionError): + """The rendezvous went away while the caller was waiting — the reply pump + stopped, so no reply can ever arrive. Fails the call immediately instead of + letting it sit until its deadline for a reason nothing will ever log.""" + + CODE: ClassVar[str] = "UNDELIVERABLE" diff --git a/runtime/node.py b/runtime/node.py new file mode 100644 index 0000000..f666d0e --- /dev/null +++ b/runtime/node.py @@ -0,0 +1,119 @@ +"""Node — the process-level rendezvous that makes ``ctx.call`` possible. + +A call cannot wait on the queue: the reply arrives on a channel shared by every +call this process has outstanding, and something has to match it back to the one +coroutine that is waiting. That something is here — a map from envelope id to a +Future, plus one pump coroutine draining the channel. + +One channel per process, not one per call. Correlation is in memory, so a reply +costs a list push and a pop rather than a key per request. + +The pump is started by ``App._serve`` and lives in its task list like any worker. +That placement is not incidental: an unsupervised pump that dies takes every +subsequent call in the process with it, silently, each one waiting out its own +deadline for a reason nothing would ever log. Because it *is* supervised, its +teardown can do the one thing no timeout can — tell the callers that no reply is +ever coming (see ``_fail_all``). +""" + +import asyncio +import logging +import os +import socket +import uuid +from typing import Any + +from runtime.broker import Broker, reply_key +from runtime.errors import ActionError, Undeliverable + +log = logging.getLogger("runtime") + +_POP_TIMEOUT_S = 1.0 +"""How long the pump parks per iteration. Kept well under the client socket +timeout — see ``redis_io.connect`` for why that coupling matters.""" + + +def new_node_id() -> str: + """Unique per process *instance*. + + Host and pid alone are not enough. Two Apps in one process — which the + examples and any integration test do — would share a reply channel, each + pump would win about half the pops, and each would discard the other's + replies as unknown ids. Half of all calls would time out for no visible + reason. A pid is also reused after a container restart. + """ + return f"{socket.gethostname()}-{os.getpid()}-{uuid.uuid4().hex[:8]}" + + +class Node: + """The calling half of a process: its reply channel and its pending calls.""" + + def __init__(self, broker: Broker, *, node_id: str | None = None) -> None: + self.id: str = node_id if node_id is not None else new_node_id() + self.reply_key: str = reply_key(self.id) + self._broker = broker + self._pending: dict[str, asyncio.Future[Any]] = {} + + def register(self, message_id: str) -> asyncio.Future[Any]: + """Claim a slot for a call that is about to be sent.""" + future: asyncio.Future[Any] = asyncio.get_running_loop().create_future() + self._pending[message_id] = future + return future + + def discard(self, message_id: str) -> None: + """Give up on a call — timed out, cancelled, or already resolved. A reply + that arrives afterwards finds nothing waiting and is dropped, which is + harmless and expected.""" + self._pending.pop(message_id, None) + + def resolve(self, reply: dict[str, Any]) -> None: + """Hand one reply to whoever is waiting for it.""" + message_id = str(reply.get("id", "")) + future = self._pending.pop(message_id, None) + if future is None or future.done(): + # An orphan: the caller already gave up, or this process never sent + # it. Nothing to do — see the class docstring. + return + if reply.get("ok"): + future.set_result(reply.get("result")) + else: + error = reply.get("error") + future.set_exception( + ActionError.from_wire(error if isinstance(error, dict) else {}) + ) + + async def pump(self) -> None: + """Drain the reply channel until cancelled. Runs as an App task.""" + log.debug("reply pump up [%s]", self.id) + try: + while True: + reply = await self._broker.pop_reply( + self.reply_key, timeout_s=_POP_TIMEOUT_S + ) + if reply is not None: + self.resolve(reply) + except asyncio.CancelledError: + raise + except Exception: + # Loudly: this is the failure that would otherwise present as every + # subsequent call in the process hanging for no stated reason. + log.exception("reply pump crashed [%s]", self.id) + raise + finally: + self._fail_all() + log.debug("reply pump down [%s]", self.id) + + def _fail_all(self) -> None: + """Break every outstanding call, because nothing can deliver to them now. + + This is what lets ``ctx.call`` use a plain per-call timer instead of a + sweeper coroutine: a timeout handles a slow executant, but only the pump + knows when the channel itself has gone away, and a caller should learn + that immediately rather than at the end of its budget. + """ + pending, self._pending = self._pending, {} + for future in pending.values(): + if not future.done(): + future.set_exception(Undeliverable("reply pump stopped")) + if pending: + log.warning("reply pump stopped with %d call(s) in flight", len(pending)) diff --git a/runtime/pool.py b/runtime/pool.py deleted file mode 100644 index d23154b..0000000 --- a/runtime/pool.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Pool — the consumer subsystem: the declarative container AND its worker. - -A `Pool` is a unit of consumption AND of isolation. Split into distinct pools by -COST PROFILE (Playwright ≠ SSE ≠ API) or to isolate blast radius. Everything runs -on ONE asyncio event loop. The imperative `register` is the ground truth; the -`flow` decorator is a shell that delegates to it (single code path). - -`slot_worker` (bottom of the file) is the runner App._serve spawns per slot — -kept here, next to `Pool`/`FlowRegistration` it consumes, mirroring how -`scheduler.py` holds both `Scheduler` and its runners (`run_every`/`run_once`). -""" - -import asyncio -import logging -import warnings -from collections.abc import Callable -from dataclasses import dataclass -from typing import TypeVar - -from runtime.broker import Broker, stream_name -from runtime.context import Handler, RuntimeContext -from runtime.types import Config, Event, Lifespan, Resources - -log = logging.getLogger("runtime") - -_HandlerFn = TypeVar("_HandlerFn", bound=Handler) - - -@dataclass(frozen=True) -class FlowRegistration: - """Inspectable state of a registered flow (testable without the runtime).""" - - handler: Handler - consumes: str # the event type (unique within the pool) - max_slots: int | None = None # optional cap; None = can take the whole budget - - -class Pool: - def __init__( - self, - name: str, - *, - max_slots: int, # shared BUDGET of the pool (memory bound) - lifespan: Lifespan | None = None, # shared resource (POOL scope) - ) -> None: - if max_slots < 1: - raise ValueError(f"{name}: max_slots must be >= 1, got {max_slots}") - self.name: str = name - self.max_slots: int = max_slots - self._lifespan: Lifespan | None = lifespan - self._flows: list[FlowRegistration] = [] - - # ── IMPERATIVE API: the runtime's ground truth. Everything goes through here. - def register( - self, handler: Handler, *, consumes: str, max_slots: int | None = None - ) -> FlowRegistration: - """First-class entry point. Usable dynamically (conditional registration - by flag, in a loop, etc.). ``handler`` is an async ``(ctx, event)`` fn.""" - reg = FlowRegistration(handler, consumes=consumes, max_slots=max_slots) - self._validate(reg) - self._flows.append(reg) - return reg - - # ── SUGAR: the decorator is just a shell that DELEGATES to register. - # It does NOTHING the method doesn't do — a single code path. - def flow( - self, *, consumes: str, max_slots: int | None = None - ) -> Callable[[_HandlerFn], _HandlerFn]: - def deco(fn: _HandlerFn) -> _HandlerFn: - self.register(fn, consumes=consumes, max_slots=max_slots) - return fn # function returned unchanged → testable bare - - return deco - - def _validate(self, reg: FlowRegistration) -> None: - # - consumes unique WITHIN this pool (one event type → one logical flow) - if any(f.consumes == reg.consumes for f in self._flows): - raise ValueError( - f"{self.name}: event type already consumed: {reg.consumes!r}" - ) - # - per-flow cap > pool budget = useless (warning, not error) - if reg.max_slots is not None and reg.max_slots > self.max_slots: - warnings.warn( - f"{self.name}/{reg.consumes}: max_slots>{self.max_slots} has no effect", - stacklevel=2, - ) - - -# ───────────────────────────────────────────────────────────────────────────── -# Slot worker — one concurrency slot running a flow handler. -# -# Per (pool, flow) the runtime spawns N slot workers, each a distinct broker -# consumer of the flow's stream (competing consumers). A worker runs a sequential -# claim → handle → ack loop; the pool semaphore wraps the handler call so total -# in-flight across the pool never exceeds the budget. No per-slot state object: -# shared resources come from the pool lifespan (ctx.resources); per-event -# setup/teardown lives in the body. -# -# Deliberately naive (no reclaim, no dead-letter, no heartbeat yet): a handler -# that raises is logged and left UN-ACKED, so the message stays pending for a -# future reclaim loop to pick up. -# ───────────────────────────────────────────────────────────────────────────── - -_BLOCK_MS = 5000 # how long claim() parks when idle; cancellation is immediate - - -async def slot_worker( - broker: Broker, - handler: Handler, - *, - consumes: str, - consumer: str, - resources: Resources, - config: Config, - pool_sem: asyncio.Semaphore, -) -> None: - ctx = RuntimeContext(broker, resources=resources, config=config) - stream = stream_name(consumes) - log.debug("slot up [%s] consuming %r", consumer, consumes) - try: - while True: - msgs = await broker.claim( - stream, consumer=consumer, count=1, block_ms=_BLOCK_MS - ) - if not msgs: - # A real broker parks on claim (socket I/O = a real suspension - # point). Some clients (e.g. fakeredis) return instantly, so yield - # here explicitly to avoid pinning the event loop in a tight loop. - await asyncio.sleep(0) - continue - for msg_id, event in msgs: - log.debug("handle %r [%s] id=%s", consumes, consumer, msg_id) - if await _process(handler, ctx, event, pool_sem, consumer, msg_id): - await broker.ack(stream, msg_id) - finally: - log.debug("slot down [%s]", consumer) - - -async def _process( - handler: Handler, - ctx: RuntimeContext, - event: Event, - pool_sem: asyncio.Semaphore, - consumer: str, - msg_id: str, -) -> bool: - """Run the handler for one event under the pool budget. Returns True if it - succeeded (→ ack), False if it raised (→ leave it pending for redelivery).""" - async with pool_sem: - try: - await handler(ctx, event) - return True - except Exception: - log.exception("handler failed; leaving unacked [%s %s]", consumer, msg_id) - return False diff --git a/runtime/redis_io.py b/runtime/redis_io.py index 6a8bcfb..d0a6889 100644 --- a/runtime/redis_io.py +++ b/runtime/redis_io.py @@ -1,24 +1,36 @@ -"""RedisBroker — the Redis Streams implementation of the Broker seam. +"""RedisBroker — the Redis implementation of the Broker seam. -Deliberately minimal: no dedup, no reclaim, no retry yet (planned). The event -payload is json-encoded into a single stream field, so producers and consumers -never argue about field layout. The consumer-group name is an internal Redis-ism -(a SQL backend has no equivalent), so it lives here, not in the Broker contract. +Streams carry messages, a list carries replies, a sorted set holds delayed +messages until they are due, hashes hold task records and the switch registry. +No Lua: the one place that needs arbitration between processes (the delay sweep) +gets it from ``ZREM``'s return value, which is enough and keeps the test suite on +``fakeredis`` without pulling in a Lua runtime. + +Deliberately still missing: dedup, reclaim, retry. See the README. + +Two connection settings are load-bearing and neither can be left to the library +default — see ``connect``. """ import asyncio import json from typing import Any, Final, cast +from pydantic import ValidationError from redis.asyncio import Redis from redis.exceptions import ResponseError from redis.exceptions import TimeoutError as RedisTimeoutError -from runtime.broker import Broker -from runtime.types import Event, Payload +from runtime.broker import DUE_KEY, SWITCHES_KEY, Broker, Delivery, Undecodable +from runtime.envelope import Envelope +from runtime.types import Payload +META_FIELD: Final = "meta" PAYLOAD_FIELD: Final = "data" -GROUP: Final = "workers" + +_MEMBER_SEP: Final = "\n" +"""Separates stream / envelope / payload inside one due-set member. Safe because +``json.dumps`` escapes newlines, so neither JSON part can contain a raw one.""" # We open every client with decode_responses=True, so every reply is `str`. The # redis-py stubs are response-type-agnostic (they return bytes|str|int|… unions @@ -27,56 +39,102 @@ _RawRead = list[tuple[str, list[tuple[str, dict[str, str]]]]] -def to_fields(payload: Payload) -> dict[str, str]: - """Encode an event payload into stream fields (one json field).""" - return {PAYLOAD_FIELD: json.dumps(payload, separators=(",", ":"))} +def _dumps(value: Any) -> str: + return json.dumps(value, separators=(",", ":")) + + +def _cancelled() -> bool: + """True when a redis-py TimeoutError is really our own cancellation. + + ``async_timeout`` converts a cancellation that lands inside a blocking read + into a TimeoutError, so a shutdown looks exactly like an idle poll. Without + this check a worker loops forever instead of unwinding. + """ + task = asyncio.current_task() + return task is not None and task.cancelling() > 0 + +def to_fields(envelope: Envelope, payload: Payload) -> dict[str, str]: + """Encode a message into stream fields — envelope and body kept apart so a + deadline check never has to decode a body it may be about to discard.""" + return {META_FIELD: envelope.model_dump_json(), PAYLOAD_FIELD: _dumps(payload)} -def from_fields(fields: dict[str, str]) -> Event: - """Decode stream fields back into the event payload.""" - raw = fields.get(PAYLOAD_FIELD) - return json.loads(raw) if raw is not None else {} + +def from_fields(msg_id: str, fields: dict[str, str]) -> Delivery | Undecodable: + """Decode stream fields back into a message. Never raises: an undecodable + message must reach the worker so it can be acked and logged rather than + becoming a poison pill nothing can quarantine.""" + raw_meta = fields.get(META_FIELD) + if raw_meta is None: + return Undecodable(msg_id, f"missing {META_FIELD!r} field") + try: + envelope = Envelope.model_validate_json(raw_meta) + except ValidationError as e: + return Undecodable(msg_id, f"bad envelope: {e}") + + raw_payload = fields.get(PAYLOAD_FIELD) + if raw_payload is None: + return Delivery(msg_id, envelope, {}) + try: + params = json.loads(raw_payload) + except json.JSONDecodeError as e: + return Undecodable(msg_id, f"bad payload: {e}") + if not isinstance(params, dict): + return Undecodable(msg_id, f"payload is {type(params).__name__}, not an object") + return Delivery(msg_id, envelope, params) class RedisBroker: - """Broker backed by Redis Streams + a consumer group. Implements the Broker - Protocol structurally (no inheritance needed).""" + """Implements the Broker Protocol structurally (no inheritance needed).""" def __init__(self, redis: Redis, *, namespace: str = "") -> None: self._redis = redis self._namespace = namespace - def _key(self, stream: str) -> str: - """Physical Redis key for a logical stream, scoped by the namespace so - several environments (e.g. dev + staging) can share one Redis without - their streams and consumer groups mixing. Empty namespace = no prefix.""" - return f"{self._namespace}:{stream}" if self._namespace else stream + def _key(self, logical: str) -> str: + """Physical key for a logical name, scoped by the namespace so several + environments can share one Redis without their streams, groups, reply + channels or switches mixing. Empty namespace = no prefix.""" + return f"{self._namespace}:{logical}" if self._namespace else logical - async def append(self, stream: str, payload: Payload) -> str: + # ── streams ─────────────────────────────────────────────────────────────── + async def append( + self, + stream: str, + envelope: Envelope, + payload: Payload, + *, + maxlen: int | None = None, + ) -> str: return cast( str, - await self._redis.xadd(self._key(stream), cast(Any, to_fields(payload))), + await self._redis.xadd( + self._key(stream), + cast(Any, to_fields(envelope, payload)), + maxlen=maxlen, + approximate=True, + ), ) - async def ensure_stream(self, stream: str) -> None: - # Create the consumer group + stream (MKSTREAM). Idempotent: an existing - # group (BUSYGROUP) is a no-op, so it's safe on every boot / every replica. + async def ensure_group(self, stream: str, group: str, *, start: str) -> None: + # MKSTREAM creates the stream if absent. An existing group (BUSYGROUP) is + # a no-op, so this is safe on every boot and every replica. try: await self._redis.xgroup_create( - name=self._key(stream), groupname=GROUP, id="0", mkstream=True + name=self._key(stream), groupname=group, id=start, mkstream=True ) except ResponseError as e: if "BUSYGROUP" not in str(e): raise async def claim( - self, stream: str, *, consumer: str, count: int, block_ms: int - ) -> list[tuple[str, Event]]: + self, stream: str, group: str, *, consumer: str, count: int, block_ms: int + ) -> list[Delivery | Undecodable]: try: resp = cast( "_RawRead | None", await self._redis.xreadgroup( - groupname=GROUP, + groupname=group, consumername=consumer, streams={self._key(stream): ">"}, count=count, @@ -84,31 +142,144 @@ async def claim( ), ) except RedisTimeoutError: - # The BLOCK window elapsed with no new messages: redis-py surfaces the - # client read-timeout as TimeoutError. Treat it as an empty poll. A - # cancelled read (shutdown) reaches here the same way, so honor a - # pending cancellation instead of looping. - task = asyncio.current_task() - if task is not None and task.cancelling(): + # The BLOCK window elapsed with no new messages. Honour a pending + # cancellation instead of looping (see _cancelled). + if _cancelled(): raise asyncio.CancelledError from None return [] if not resp: return [] - out: list[tuple[str, Event]] = [] - for _stream, messages in resp: - for msg_id, fields in messages: - out.append((msg_id, from_fields(fields))) - return out + return [ + from_fields(msg_id, fields) + for _stream, messages in resp + for msg_id, fields in messages + ] + + async def ack(self, stream: str, group: str, message_id: str) -> None: + await self._redis.xack(self._key(stream), group, message_id) + + # ── reply rendezvous ────────────────────────────────────────────────────── + async def push_reply(self, key: str, reply: dict[str, Any], *, ttl_ms: int) -> None: + k = self._key(key) + async with self._redis.pipeline(transaction=False) as pipe: + pipe.lpush(k, _dumps(reply)) + pipe.pexpire(k, ttl_ms) + await pipe.execute() + + async def pop_reply(self, key: str, *, timeout_s: float) -> dict[str, Any] | None: + try: + res = cast( + "tuple[str, str] | None", + await self._redis.blpop([self._key(key)], timeout=timeout_s), + ) + except RedisTimeoutError: + if _cancelled(): + raise asyncio.CancelledError from None + return None + if res is None: + return None + _popped_from, raw = res + try: + decoded = json.loads(raw) + except json.JSONDecodeError: + return None + return decoded if isinstance(decoded, dict) else None + + # ── task records ────────────────────────────────────────────────────────── + async def put_task(self, key: str, record: dict[str, Any], *, ttl_ms: int) -> None: + k = self._key(key) + mapping = {field: _dumps(value) for field, value in record.items()} + async with self._redis.pipeline(transaction=False) as pipe: + pipe.hset(k, mapping=cast(Any, mapping)) + pipe.pexpire(k, ttl_ms) + await pipe.execute() + + async def get_task(self, key: str) -> dict[str, Any] | None: + raw = cast(dict[str, str], await self._redis.hgetall(self._key(key))) + if not raw: + return None + return {field: json.loads(value) for field, value in raw.items()} + + # ── delayed delivery ────────────────────────────────────────────────────── + async def schedule( + self, due_ms: int, stream: str, envelope: Envelope, payload: Payload + ) -> None: + fields = to_fields(envelope, payload) + # The envelope id travels inside the member, which is what keeps two + # identical delayed payloads from collapsing into one: sorted-set members + # are a SET, so a member derived from the body alone would silently + # overwrite its twin's due time instead of queueing a second message. + member = _MEMBER_SEP.join((stream, fields[META_FIELD], fields[PAYLOAD_FIELD])) + await self._redis.zadd(self._key(DUE_KEY), {member: due_ms}) + + async def sweep(self, now_ms: int, *, limit: int, maxlen: int | None) -> int: + key = self._key(DUE_KEY) + members = cast( + list[str], + await self._redis.zrangebyscore(key, "-inf", now_ms, start=0, num=limit), + ) + if not members: + return 0 + + # ZREM arbitrates: with N processes sweeping the same set, exactly one + # gets a 1 back for a given member and therefore owns moving it. One + # pipelined batch, not one round trip per message. + async with self._redis.pipeline(transaction=False) as pipe: + for member in members: + pipe.zrem(key, member) + removed = cast(list[int], await pipe.execute()) + + winners = [m for m, won in zip(members, removed, strict=True) if won] + if not winners: + return 0 + + async with self._redis.pipeline(transaction=False) as pipe: + for member in winners: + stream, meta, payload = member.split(_MEMBER_SEP, 2) + pipe.xadd( + self._key(stream), + cast(Any, {META_FIELD: meta, PAYLOAD_FIELD: payload}), + maxlen=maxlen, + approximate=True, + ) + await pipe.execute() + return len(winners) - async def ack(self, stream: str, message_id: str) -> None: - await self._redis.xack(self._key(stream), GROUP, message_id) + # ── switches ────────────────────────────────────────────────────────────── + async def switches_get_all(self) -> dict[str, bool]: + raw = cast(dict[str, str], await self._redis.hgetall(self._key(SWITCHES_KEY))) + return {name: value != "0" for name, value in raw.items()} + + async def switch_set(self, name: str, enabled: bool) -> None: + await self._redis.hset(self._key(SWITCHES_KEY), name, "1" if enabled else "0") async def aclose(self) -> None: await self._redis.aclose() -def connect(url: str, *, namespace: str = "") -> Broker: +def connect( + url: str, + *, + namespace: str = "", + max_connections: int | None = None, + max_block_s: float = 5.0, +) -> Broker: """Open a Redis-backed broker (``decode_responses=True`` → str in/out). - ``namespace`` prefixes every stream key to isolate environments sharing one - Redis. When a second backend lands, this becomes a scheme-dispatching factory.""" - return RedisBroker(Redis.from_url(url, decode_responses=True), namespace=namespace) + + ``max_block_s`` is the longest blocking command this client will issue, and + it sets ``socket_timeout`` above it. This is not a tuning knob — redis-py + defaults ``socket_timeout`` to 5 s, which is *exactly* the runtime's default + XREADGROUP BLOCK window, so every idle worker races its own read timeout and + quietly reconnects every few seconds, forever. + + ``max_connections`` likewise defaults to 100 in redis-py while every blocked + worker holds one for its whole window. The App computes it from the topology + rather than letting a busy deployment discover the ceiling as a crash. + """ + redis: Redis = Redis.from_url( + url, + decode_responses=True, + socket_timeout=max_block_s + 5.0, + max_connections=max_connections, + ) + return RedisBroker(redis, namespace=namespace) diff --git a/runtime/scheduler.py b/runtime/scheduler.py deleted file mode 100644 index 7102554..0000000 --- a/runtime/scheduler.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Scheduler — the producer-side container, mirror of Pool. - -A Scheduler groups producers that share an (optional) lifespan resource and is -included into the App like a Pool. Each producer is an async ``(ctx)`` body -registered with ``@scheduler.every(interval)``; the runtime calls it on that -fixed interval, passing a Context whose ``resources`` come from the scheduler's -lifespan (this is how a cron-style producer gets its API/DB client). - - Scheduler ↔ Pool @scheduler.every ↔ @pool.flow producer ↔ handler - -Two triggers today: `every` (periodic — loops on a fixed interval) and `once` (a -one-shot producer that fires a single time at boot, after an optional `delay`). -Further specialized triggers — real calendar `cron` (5-field, level-triggered) -and stochastic load (Poisson, ramps) — are planned for when they actually diverge. -""" - -import asyncio -import logging -from collections.abc import Callable -from dataclasses import dataclass -from typing import Final, TypeVar - -from runtime.broker import Broker -from runtime.context import ProducerFn, RuntimeContext -from runtime.types import Config, Lifespan, Resources - -log = logging.getLogger("runtime") - -# Duration suffixes → seconds. Longer suffixes first so "ms"/"min" win over "s"/"m". -_UNITS: Final[list[tuple[str, float]]] = [ - ("ms", 0.001), - ("min", 60.0), - ("h", 3600.0), - ("s", 1.0), - ("m", 60.0), -] - - -def parse_duration(value: str | float, *, allow_zero: bool = False) -> float: - """Duration → seconds. Accepts a number (seconds) or a string with a unit - suffix: ``"500ms"``, ``"1.5s"``, ``"10min"``, ``"1h"`` (bare number = seconds). - Intervals must be > 0; a `once` delay may be 0 (fire at boot) — pass - ``allow_zero=True`` for that.""" - if isinstance(value, (int, float)): - seconds = float(value) - else: - s = value.strip() - for suffix, mult in _UNITS: - if s.endswith(suffix): - seconds = float(s[: -len(suffix)]) * mult - break - else: - seconds = float(s) # bare number string → seconds (raises on garbage) - if seconds < 0 or (seconds == 0 and not allow_zero): - raise ValueError( - f"duration must be {'>= 0' if allow_zero else '> 0'}: {value!r}" - ) - return seconds - - -@dataclass(frozen=True) -class EveryRegistration: - """Inspectable state of a periodic (`every`) producer (testable without the runtime).""" - - handler: ProducerFn - interval: float # seconds between ticks - id: str # identity / base of the deterministic dedup key (planned) - - -@dataclass(frozen=True) -class OnceRegistration: - """Inspectable state of a one-shot producer (testable without the runtime).""" - - handler: ProducerFn - delay: float # seconds to wait before the single fire (0 = at boot) - id: str - - -_ProducerFnT = TypeVar("_ProducerFnT", bound=ProducerFn) - - -class Scheduler: - def __init__(self, name: str, *, lifespan: Lifespan | None = None) -> None: - self.name: str = name - self._lifespan: Lifespan | None = lifespan - self._every: list[EveryRegistration] = [] - self._once: list[OnceRegistration] = [] - - # ── IMPERATIVE API: the runtime's ground truth. Everything goes through here. - def register_every( - self, fn: ProducerFn, *, interval: str | float, id: str | None = None - ) -> EveryRegistration: - reg = EveryRegistration(fn, parse_duration(interval), id or fn.__name__) - self._every.append(reg) - return reg - - def register_once( - self, fn: ProducerFn, *, delay: str | float = 0, id: str | None = None - ) -> OnceRegistration: - reg = OnceRegistration( - fn, parse_duration(delay, allow_zero=True), id or fn.__name__ - ) - self._once.append(reg) - return reg - - # ── SUGAR: the decorators are shells that DELEGATE to register (single path). - def every( - self, interval: str | float, *, id: str | None = None - ) -> Callable[[_ProducerFnT], _ProducerFnT]: - def deco(fn: _ProducerFnT) -> _ProducerFnT: - self.register_every(fn, interval=interval, id=id) - return fn # function returned unchanged → testable bare - - return deco - - def once( - self, *, delay: str | float = 0, id: str | None = None - ) -> Callable[[_ProducerFnT], _ProducerFnT]: - def deco(fn: _ProducerFnT) -> _ProducerFnT: - self.register_once(fn, delay=delay, id=id) - return fn # function returned unchanged → testable bare - - return deco - - -async def run_every( - broker: Broker, - reg: EveryRegistration, - *, - resources: Resources, - config: Config, -) -> None: - """Drive one periodic (`every`) producer: call its body on a fixed interval - (emit-then-sleep, so it fires once at boot). A body that raises is logged and - the loop continues — one bad tick must not kill the schedule. Cancellation - propagates out cleanly.""" - ctx = RuntimeContext(broker, resources=resources, config=config) - while True: - try: - await reg.handler(ctx) - except Exception: - log.exception("every %r tick failed", reg.id) - await asyncio.sleep(reg.interval) - - -async def run_once( - broker: Broker, - reg: OnceRegistration, - *, - resources: Resources, - config: Config, -) -> None: - """Drive a one-shot producer: wait its (optional) delay, then call the body - exactly once and return. A body that raises is logged, not re-raised — one bad - one-shot must not crash the app. Cancellation propagates out cleanly.""" - ctx = RuntimeContext(broker, resources=resources, config=config) - if reg.delay: - await asyncio.sleep(reg.delay) - try: - await reg.handler(ctx) - except Exception: - log.exception("once %r failed", reg.id) diff --git a/runtime/service.py b/runtime/service.py new file mode 100644 index 0000000..e5ac733 --- /dev/null +++ b/runtime/service.py @@ -0,0 +1,530 @@ +"""Service — the container, and the workers that run what it declares. + +A service is one domain: its actions, its event subscriptions, its producers, the +resources they share, and the concurrency budget they compete for. It is also the +unit of isolation — split by COST PROFILE (Playwright ≠ SSE ≠ API) so a flood of +expensive work cannot starve cheap work — and the unit of the master switch. + +Consumers and producers live together because the three-verb model makes +consume-and-produce the normal case: the moment a handler calls another action, +its process is simultaneously executant and caller. Splitting those into separate +container types only forced one domain to live in two files. + +The imperative ``register_*`` methods are the ground truth; the decorators are +shells that delegate to them, so there is a single code path and dynamic +registration (in a loop, behind a flag) is a first-class option. + +``slot_worker`` and the producer runners live here too, next to the registrations +they consume. +""" + +import asyncio +import json +import logging +import warnings +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, Final, TypeVar + +from pydantic import BaseModel, ValidationError + +from runtime.broker import ( + START_HEAD, + START_TAIL, + WORKERS_GROUP, + Broker, + Delivery, + Undecodable, + action_stream, + event_stream, + task_key, +) +from runtime.context import ( + TASK_DONE, + TASK_EXPIRED, + TASK_FAILED, + ActionHandler, + EventHandler, + Limits, + ProducerFn, + RuntimeContext, +) +from runtime.duration import parse_duration +from runtime.envelope import Envelope +from runtime.errors import ActionError, Expired, ParamsInvalid +from runtime.switches import SwitchBoard +from runtime.types import Lifespan + +log = logging.getLogger("runtime") + +BLOCK_MS: Final = 5000 +"""How long a claim parks when idle. Cancellation is still immediate. Note the +coupling to the client socket timeout — see ``redis_io.connect``.""" + +_IDLE_SLEEP_S: Final = 0.005 +"""Yield after an empty claim. A real broker parks inside ``claim`` (socket I/O +is a genuine suspension point), but fakeredis returns from XREADGROUP instantly +and is not woken by a concurrent write, so without this the test suite pins the +event loop and starves the reply pump and the sweeper.""" + +_PAUSE_POLL_S: Final = 0.25 +"""How often a paused worker looks again. Small enough that resuming feels +immediate, large enough to cost nothing while paused.""" + + +# ───────────────────────────────────────────────────────────────────────────── +# Registrations +# ───────────────────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class ActionRegistration: + """An action: exactly one executant, whatever the replica count. That is what + the single consumer group buys — Redis hands each request to one member.""" + + handler: ActionHandler + name: str + params: type[BaseModel] | None = None + max_slots: int | None = None + + is_event: bool = False + + @property + def stream(self) -> str: + return action_stream(self.name) + + @property + def group(self) -> str: + return WORKERS_GROUP + + @property + def start(self) -> str: + # An action stream's group is created once and outlives every worker, so + # where it starts only matters the first time. HEAD means work queued + # before the first consumer ever booted is not silently discarded. + return START_HEAD + + +@dataclass(frozen=True) +class EventRegistration: + """A subscription: one consumer group per subscriber, so each gets a copy.""" + + handler: EventHandler + name: str + group: str + params: type[BaseModel] | None = None + max_slots: int | None = None + + is_event: bool = True + + @property + def stream(self) -> str: + return event_stream(self.name) + + @property + def start(self) -> str: + # TAIL: a subscriber deployed into a live system sees what happens from + # now on. Starting at HEAD would dump the entire retained stream into a + # fresh pool the moment it booted. + return START_TAIL + + +type Consumer = ActionRegistration | EventRegistration + + +@dataclass(frozen=True) +class EveryRegistration: + handler: ProducerFn + interval: float + id: str + + +@dataclass(frozen=True) +class OnceRegistration: + handler: ProducerFn + delay: float + id: str + + +_ActionFn = TypeVar("_ActionFn", bound=ActionHandler) +_EventFn = TypeVar("_EventFn", bound=EventHandler) +_ProducerFnT = TypeVar("_ProducerFnT", bound=ProducerFn) + + +class Service: + def __init__( + self, + name: str, + *, + max_slots: int | None = None, + lifespan: Lifespan | None = None, + ) -> None: + if max_slots is not None and max_slots < 1: + raise ValueError(f"{name}: max_slots must be >= 1, got {max_slots}") + self.name: str = name + self.max_slots: int | None = max_slots + self._lifespan: Lifespan | None = lifespan + self._actions: list[ActionRegistration] = [] + self._events: list[EventRegistration] = [] + self._every: list[EveryRegistration] = [] + self._once: list[OnceRegistration] = [] + + @property + def consumers(self) -> list[Consumer]: + return [*self._actions, *self._events] + + # ── imperative API — the ground truth ───────────────────────────────────── + def register_action( + self, + handler: ActionHandler, + *, + name: str, + params: type[BaseModel] | None = None, + max_slots: int | None = None, + ) -> ActionRegistration: + reg = ActionRegistration(handler, name, params=params, max_slots=max_slots) + if any(a.name == name for a in self._actions): + raise ValueError(f"{self.name}: action already registered: {name!r}") + self._warn_if_cap_useless(reg.max_slots, name) + self._actions.append(reg) + return reg + + def register_event( + self, + handler: EventHandler, + *, + name: str, + group: str | None = None, + params: type[BaseModel] | None = None, + max_slots: int | None = None, + ) -> EventRegistration: + resolved = group if group is not None else self.name + if any(e.name == name and e.group == resolved for e in self._events): + raise ValueError( + f"{self.name}: already subscribed to {name!r} as group " + f"{resolved!r} — two handlers in one service would compete for " + f"each message. Pass group=... if you want both to run." + ) + reg = EventRegistration( + handler, name, resolved, params=params, max_slots=max_slots + ) + self._warn_if_cap_useless(reg.max_slots, name) + self._events.append(reg) + return reg + + def register_every( + self, fn: ProducerFn, *, interval: str | float, id: str | None = None + ) -> EveryRegistration: + reg = EveryRegistration(fn, parse_duration(interval), id or fn.__name__) + self._every.append(reg) + return reg + + def register_once( + self, fn: ProducerFn, *, delay: str | float = 0, id: str | None = None + ) -> OnceRegistration: + reg = OnceRegistration( + fn, parse_duration(delay, allow_zero=True), id or fn.__name__ + ) + self._once.append(reg) + return reg + + # ── decorators — shells that delegate, so there is one code path ────────── + def action( + self, + name: str, + *, + params: type[BaseModel] | None = None, + max_slots: int | None = None, + ) -> Callable[[_ActionFn], _ActionFn]: + def deco(fn: _ActionFn) -> _ActionFn: + self.register_action(fn, name=name, params=params, max_slots=max_slots) + return fn # returned unchanged → still testable bare + + return deco + + def event( + self, + name: str, + *, + group: str | None = None, + params: type[BaseModel] | None = None, + max_slots: int | None = None, + ) -> Callable[[_EventFn], _EventFn]: + def deco(fn: _EventFn) -> _EventFn: + self.register_event( + fn, name=name, group=group, params=params, max_slots=max_slots + ) + return fn + + return deco + + def every( + self, interval: str | float, *, id: str | None = None + ) -> Callable[[_ProducerFnT], _ProducerFnT]: + def deco(fn: _ProducerFnT) -> _ProducerFnT: + self.register_every(fn, interval=interval, id=id) + return fn + + return deco + + def once( + self, *, delay: str | float = 0, id: str | None = None + ) -> Callable[[_ProducerFnT], _ProducerFnT]: + def deco(fn: _ProducerFnT) -> _ProducerFnT: + self.register_once(fn, delay=delay, id=id) + return fn + + return deco + + def _warn_if_cap_useless(self, cap: int | None, name: str) -> None: + if cap is not None and self.max_slots is not None and cap > self.max_slots: + warnings.warn( + f"{self.name}/{name}: max_slots>{self.max_slots} has no effect", + stacklevel=3, + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Slot worker — one concurrency slot running one registration. +# +# Per (service, registration) the runtime spawns N workers, each a distinct +# broker consumer of the same group (competing consumers). A worker loops +# claim → handle → report → ack; the service semaphore wraps ONLY the handler +# call, so total in-flight work across the service never exceeds the budget while +# an expired burst or a validation failure costs no permit at all. +# +# Every message is acked, whatever happened to it. With no reclaim, leaving a +# message un-acked does not mean "retry later" — it means lost anyway, plus a +# pending entry attached to a consumer name that will never exist again. +# ───────────────────────────────────────────────────────────────────────────── + + +def _jsonable(value: Any) -> Any: + """A handler's return value, as something that can cross the wire. + + Handlers routinely return pydantic models, so those are dumped. Everything + else is checked here rather than deep inside the broker: failing at this + point means the caller gets a typed error it can act on, instead of a reply + that never arrives and a timeout that says nothing about why. + """ + if isinstance(value, BaseModel): + value = value.model_dump(mode="json") + json.dumps(value) # raises here, where the failure is still reportable + return value + + +async def slot_worker( + broker: Broker, + reg: Consumer, + *, + service_name: str, + consumer: str, + ctx: RuntimeContext, + semaphore: asyncio.Semaphore, + limits: Limits, + switches: SwitchBoard | None = None, + block_ms: int = BLOCK_MS, +) -> None: + log.debug("slot up [%s] on %r", consumer, reg.name) + try: + while True: + if switches is not None and not switches.is_enabled(service_name, reg.name): + # Paused: never enter the claim, so the stream simply grows and + # drains on resume. Work already in a handler is not interrupted. + await asyncio.sleep(_PAUSE_POLL_S) + continue + + messages = await broker.claim( + reg.stream, reg.group, consumer=consumer, count=1, block_ms=block_ms + ) + if not messages: + await asyncio.sleep(_IDLE_SLEEP_S) + continue + + for message in messages: + try: + await _handle( + broker, reg, message, ctx=ctx, sem=semaphore, lim=limits + ) + await broker.ack(reg.stream, reg.group, message.id) + except asyncio.CancelledError: + raise + except Exception: + # One message must not take down the worker. Without this, + # anything raising outside the handler's own guard — a reply + # that will not serialize, a blip on the ack — kills this + # task, which propagates out of App's gather and stops the + # whole process. + log.exception( + "worker error on %s [%s] — continuing", + reg.stream, + message.id, + ) + finally: + log.debug("slot down [%s]", consumer) + + +async def _handle( + broker: Broker, + reg: Consumer, + message: Delivery | Undecodable, + *, + ctx: RuntimeContext, + sem: asyncio.Semaphore, + lim: Limits, +) -> None: + if isinstance(message, Undecodable): + # Nothing can be reported to — the envelope is what carried the reply + # address. Log loudly; the ack in the caller stops it becoming a poison + # pill nobody can quarantine. + log.error("undecodable message on %s: %s", reg.stream, message.reason) + return + + envelope = message.envelope + # Not typed as Params: once a registration declares a model, what the handler + # receives is an instance of it, not the dict that came off the wire. + body: Any = message.params + + left = envelope.remaining_ms() + if left is not None and left <= 0: + await _report( + broker, + envelope, + lim, + error=Expired( + f"{reg.name} deadline passed {-left}ms before it was claimed" + ), + ) + return + + if reg.params is not None: + try: + body = reg.params.model_validate(body) + except ValidationError as e: + # Never worth redelivering — the same bytes fail the same way. The + # caller in slot_worker acks it regardless. + await _report( + broker, + envelope, + lim, + error=ParamsInvalid(f"{reg.name}: {e}", data={"errors": e.errors()}), + ) + return + + async with sem: + try: + result = await reg.handler(ctx.for_message(envelope, body), body) + except Exception as e: + await _report(broker, envelope, lim, error=ActionError.wrap(e)) + return + await _report(broker, envelope, lim, result=None if reg.is_event else result) + + +async def _report( + broker: Broker, + envelope: Envelope, + lim: Limits, + *, + result: Any = None, + error: ActionError | None = None, +) -> None: + """Tell whoever is listening how it went — a waiting caller, a task record, + or (for an event) the log.""" + if error is None and (envelope.reply is not None or envelope.task is not None): + # Only serialize when somebody is actually listening, and turn a value + # that will not cross the wire into a reportable failure rather than an + # exception thrown while reporting. + try: + result = _jsonable(result) + except Exception as e: + error, result = ActionError.wrap(e), None + + if envelope.reply is not None: + # Outlive the caller's own wait comfortably, or a reply can expire out + # from under a caller that is still waiting and surface as a timeout. + ttl_ms = max(lim.default_ttl_ms, envelope.remaining_ms() or 0) * 2 + await broker.push_reply( + envelope.reply, + { + "id": envelope.id, + "ok": error is None, + "result": result, + "error": error.to_wire() if error is not None else None, + }, + ttl_ms=ttl_ms, + ) + + if envelope.task is not None: + state = TASK_DONE if error is None else TASK_FAILED + if isinstance(error, Expired): + state = TASK_EXPIRED + await broker.put_task( + task_key(envelope.task), + { + "state": state, + "action": envelope.name, + "result": result, + "error": error.to_wire() if error is not None else None, + }, + ttl_ms=lim.task_ttl_ms, + ) + + if error is not None and envelope.reply is None and envelope.task is None: + log.error( + "%s failed with nowhere to report it: %s [rid=%s id=%s]", + envelope.name, + error.message, + envelope.rid, + envelope.id, + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Producer runners +# ───────────────────────────────────────────────────────────────────────────── + + +async def run_every( + reg: EveryRegistration, + *, + ctx: RuntimeContext, + service_name: str, + switches: SwitchBoard | None = None, +) -> None: + """Drive a periodic producer: call the body, then sleep. A body that raises + is logged and the loop continues — one bad tick must not kill the schedule. + + A paused producer skips the body but keeps the loop, so the cadence is intact + the instant it is resumed. + """ + while True: + if switches is None or switches.is_enabled(service_name, reg.id): + try: + await reg.handler(ctx) + except Exception: + log.exception("every %r tick failed", reg.id) + await asyncio.sleep(reg.interval) + + +async def run_once( + reg: OnceRegistration, + *, + ctx: RuntimeContext, + service_name: str, + switches: SwitchBoard | None = None, +) -> None: + """Drive a one-shot producer: wait its delay, then call the body once. A body + that raises is logged, not re-raised — one bad one-shot must not crash boot. + + If it is paused when its moment arrives it waits, and runs on resume.""" + if reg.delay: + await asyncio.sleep(reg.delay) + # Wait out a pause rather than skipping. A one-shot has exactly one chance to + # run, so returning here would make a switch DISCARD work — the one thing + # pausing is documented never to do. + while switches is not None and not switches.is_enabled(service_name, reg.id): + await asyncio.sleep(_PAUSE_POLL_S) + try: + await reg.handler(ctx) + except Exception: + log.exception("once %r failed", reg.id) diff --git a/runtime/switches.py b/runtime/switches.py new file mode 100644 index 0000000..1201427 --- /dev/null +++ b/runtime/switches.py @@ -0,0 +1,128 @@ +"""Switches — pause and resume parts of a running system without a deploy. + +Two levels. A service name is a master switch over everything it registers; each +action, event subscription and producer also has its own. Pausing stops +*consumption* — the stream keeps growing and drains when you resume — so it is a +throttle and a maintenance lever, never a way to discard work. + +Not to be confused with ``App.include(enabled=...)``, which is a wiring decision +taken once at boot: a service left out there is never constructed and its +lifespan never runs. A switch pauses something that *is* mounted, and does not +release its resources. Both are useful; conflating them is how you end up +surprised that pausing a browser pool did not close the browser. + +The whole registry is read in one round trip on a timer and answered from memory +after that, so a check costs nothing on the hot path. The cost is latency, from +two sources: a flip is seen within one refresh interval, and a worker already +parked in a claim will still handle whatever that claim returns — pausing stops +a worker taking NEW work, it cannot retract a read already in flight. So a pause +is fully in effect after roughly one refresh interval plus one claim window. +That is fine for an operational lever, and it is exactly why this is not a safety +mechanism. + +Absent means enabled. Nothing has to be seeded, and a Redis with no switches +behaves exactly as if the feature did not exist. +""" + +import asyncio +import logging +from typing import Final + +from runtime.broker import Broker + +log = logging.getLogger("runtime") + +DEFAULT_INTERVAL_S: Final = 2.0 + + +class SwitchBoard: + """An in-process snapshot of the switch registry, refreshed on a timer.""" + + def __init__( + self, broker: Broker, *, interval_s: float = DEFAULT_INTERVAL_S + ) -> None: + self._broker = broker + self._interval_s = interval_s + self._paused: frozenset[str] = frozenset() + self._generation = 0 + """Bumped by every local ``set``. A refresh that started before one + landed is holding a stale read and must not publish it.""" + + def is_enabled(self, *names: str) -> bool: + """True when none of ``names`` is paused. + + Callers pass the whole chain — service name and registration name — so + the master switch is just "one of the names is off", with no precedence + rules to remember. + """ + return not any(name in self._paused for name in names) + + async def refresh(self) -> None: + """Re-read the registry. A failure keeps the previous snapshot rather + than pausing the world: losing Redis should not silently stop a system + that is otherwise able to run.""" + generation = self._generation + try: + switches = await self._broker.switches_get_all() + except Exception: + log.exception("switch refresh failed; keeping the previous snapshot") + return + if generation != self._generation: + # A local set() landed while this read was in flight. It wrote to + # Redis and is strictly newer than what we are holding, so publishing + # this would silently undo it. + return + self._paused = frozenset( + name for name, enabled in switches.items() if not enabled + ) + + async def run(self) -> None: + """Refresh forever. Runs as an App task.""" + while True: + await self.refresh() + await asyncio.sleep(self._interval_s) + + async def set(self, name: str, enabled: bool) -> None: + """Flip a switch and apply it locally at once, so the caller's own next + check reflects what it just did instead of waiting for the timer.""" + await self._broker.switch_set(name, enabled) + self._paused = self._paused - {name} if enabled else self._paused | {name} + self._generation += 1 + + +# ── from outside a running App ─────────────────────────────────────────────── +# +# A short-lived connection, so an operator, a test, or a deploy script can flip a +# switch without standing up an App. + + +async def _with_broker(url: str, namespace: str): # type: ignore[no-untyped-def] + from runtime.redis_io import connect + + return connect(url, namespace=namespace) + + +async def enable(url: str, name: str, *, namespace: str = "") -> None: + broker = await _with_broker(url, namespace) + try: + await broker.switch_set(name, True) + finally: + await broker.aclose() + + +async def disable(url: str, name: str, *, namespace: str = "") -> None: + broker = await _with_broker(url, namespace) + try: + await broker.switch_set(name, False) + finally: + await broker.aclose() + + +async def status(url: str, *, namespace: str = "") -> dict[str, bool]: + """Everything explicitly set, in one call. Names that were never touched do + not appear — they are enabled.""" + broker = await _with_broker(url, namespace) + try: + return await broker.switches_get_all() + finally: + await broker.aclose() diff --git a/runtime/types.py b/runtime/types.py index 2ccd0df..59d382a 100644 --- a/runtime/types.py +++ b/runtime/types.py @@ -10,20 +10,23 @@ from typing import Any type Payload = dict[str, Any] -"""The kwargs handed to ``ctx.emit(type, **payload)`` — the body of an event.""" +"""The kwargs handed to a verb — ``ctx.call(name, **payload)`` — the body of a +message on its way out.""" -type Event = dict[str, Any] -"""A delivered event: the decoded payload a flow handler receives.""" +type Params = dict[str, Any] +"""A delivered body: what a handler receives as its second argument, and as +``ctx.params``. Plain dict unless the registration declares a ``params`` model, +in which case the handler gets the validated instance instead.""" type Resources = dict[str, Any] """What a ``lifespan`` yields; exposed to handlers/producers as ``ctx.resources``.""" type Config = dict[str, Any] -"""Deployment config injected via ``App.include(config=...)``; reaches the -lifespan as its second argument and handlers/producers as ``ctx.config``.""" +"""Deployment config injected via ``App.include(config=...)``; passed to the +lifespan as its only argument and reaching handlers/producers as ``ctx.config``.""" type Lifespan = Callable[[Config], AbstractAsyncContextManager[Resources]] -"""A container's shared-resource factory (POOL scope). Called once with the +"""A service's shared-resource factory (SERVICE scope). Called once with the deployment ``config`` and returns an async context manager whose yielded mapping becomes ``ctx.resources`` and whose teardown runs on shutdown. Typically an ``@asynccontextmanager`` async generator.""" diff --git a/tests/conftest.py b/tests/conftest.py index 9269afd..3eaed1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,26 @@ import os import socket +import uuid +import pytest import pytest_asyncio REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") +BACKEND = os.environ.get("RUNTIME_TEST_BACKEND", "auto") +"""``auto`` (default) uses a real Redis when one is listening, else fakeredis. +Set ``fake`` or ``real`` to pin it; CI pins both and runs the suite twice on every +Python version. Worth pinning because the two are not interchangeable: a real +XREADGROUP BLOCK is woken by an arriving message and fakeredis's is not, and a +real Redis is shared across the whole run while fakeredis is fresh per test.""" + + def _real_redis_reachable(host: str = "localhost", port: int = 6379) -> bool: + if BACKEND == "fake": + return False + if BACKEND == "real": + return True try: socket.create_connection((host, port), timeout=0.5).close() return True @@ -32,10 +46,24 @@ async def redis(): await r.aclose() +@pytest.fixture +def namespace(): + """A fresh key prefix per test. + + When a real Redis is reachable the `redis` fixture uses it, and it is shared + across every test and every run with no teardown. Streams could dodge that by + generating unique names, but switches, task records, the reply channels and + the due set all live under fixed keys — so without this, one test pausing an + action leaves it paused for everything that follows. + """ + return f"t{uuid.uuid4().hex[:10]}" + + @pytest_asyncio.fixture -async def broker(redis): +async def broker(redis, namespace): """A RedisBroker over the SAME client as the `redis` fixture, so a test can - emit through the broker and assert on the raw client (shared data).""" + emit through the broker and assert on the raw client. Raw assertions must + prefix the key with the `namespace` fixture.""" from runtime.redis_io import RedisBroker - return RedisBroker(redis) + return RedisBroker(redis, namespace=namespace) diff --git a/tests/helpers.py b/tests/helpers.py index d758a3f..6569979 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,6 +1,11 @@ import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from typing import Any +from runtime import App +from runtime.broker import Broker + async def wait_for_output(capsys: Any, *needles: str, timeout: float = 2.0) -> str: out = "" @@ -14,3 +19,24 @@ async def wait_for_output(capsys: Any, *needles: str, timeout: float = 2.0) -> s out += capsys.readouterr().out missing = [needle for needle in needles if needle not in out] raise AssertionError(f"missing output {missing!r}; got {out!r}") + + +@asynccontextmanager +async def serving(app: App, broker: Broker) -> AsyncIterator[asyncio.Task[None]]: + """Run an App against an injected broker for the body of the block, then + cancel it and let it unwind. The broker injection is what keeps these tests + off a real Redis — every subsystem, including the reply pump and the sweeper, + goes through the same seam.""" + task = asyncio.create_task(app._serve(broker)) + try: + yield task + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + # If _serve died on its own, that is the root cause and it must win over + # whatever the body asserted. Swallowing it here turns every boot failure + # into a mystifying "nothing happened". + if not task.cancelled(): + failure = task.exception() + if failure is not None: + raise failure diff --git a/tests/test_api.py b/tests/test_api.py deleted file mode 100644 index 0f49534..0000000 --- a/tests/test_api.py +++ /dev/null @@ -1,148 +0,0 @@ -import logging - -import pytest - -from runtime import App, Pool, Scheduler -from runtime.app import _check_unique_consumes -from runtime.broker import stream_name - - -def make_pool(name="p", budget=10): - return Pool(name, max_slots=budget) - - -async def _noop(ctx, event): ... - - -def test_validate_duplicate_consumes_within_pool(): - pool = make_pool() - pool.register(_noop, consumes="x") - with pytest.raises(ValueError): - pool.register(_noop, consumes="x") - - -def test_decorator_returns_func_unchanged(): - pool = make_pool() - - @pool.flow(consumes="y") - async def handler(ctx, event): ... - - assert pool._flows[0].handler is handler # decorator returns the fn unchanged - assert len(pool._flows) == 1 - - -def test_cap_over_budget_warns_not_raises(): - pool = make_pool(budget=5) - with pytest.warns(UserWarning): - pool.register(_noop, consumes="z", max_slots=10) - - -def test_pool_rejects_nonpositive_max_slots(): - for bad in (0, -1): - with pytest.raises(ValueError, match="max_slots"): - Pool("p", max_slots=bad) - - -def test_stream_name_mapping(): - assert stream_name("restock") == "flow:restock" - - -def test_check_unique_consumes_ok(): - p1, p2 = make_pool("p1"), make_pool("p2") - p1.register(_noop, consumes="restock") - p2.register(_noop, consumes="audit") - _check_unique_consumes([p1, p2]) # distinct → no raise - - -def test_check_unique_consumes_across_pools(): - p1, p2 = make_pool("p1"), make_pool("p2") - p1.register(_noop, consumes="dup") - p2.register(_noop, consumes="dup") - with pytest.raises(ValueError, match="fan-out"): - _check_unique_consumes([p1, p2]) - - -def test_include_skips_disabled_and_collects_pool(): - app = App(redis="redis://x") - pool = make_pool() - pool.register(_noop, consumes="restock") - app.include(pool, enabled=False, max_slots=3, config={"x": 1}) - assert app._pools == [] - app.include(pool, enabled=True, max_slots=4, config={"x": 2}) - assert len(app._pools) == 1 - inc = app._pools[0] - assert inc.pool is pool - assert inc.max_slots == 4 - assert inc.config == {"x": 2} - - -_sched = Scheduler("s") - - -@_sched.every("5min", id="c1") -async def _producer(ctx): ... - - -def test_include_scheduler_and_rejects_garbage(): - app = App(redis="redis://x") - app.include(_sched, enabled=True, config={"rate": "fast"}) - assert len(app._schedulers) == 1 - inc = app._schedulers[0] - assert inc.scheduler is _sched - assert inc.config == {"rate": "fast"} - with pytest.raises(TypeError): - app.include(object()) - - -def test_log_topology(caplog): - # start() blocks on the event loop, so assert on the topology log helper. - app = App(redis="redis://x") - pool = make_pool() - pool.register(_noop, consumes="restock") - app.include(pool) - app.include(_sched) - with caplog.at_level(logging.INFO, logger="runtime"): - app._log_topology() - assert "topology" in caplog.text.lower() - assert "restock" in caplog.text - assert "Scheduler" in caplog.text - assert "c1" in caplog.text - - -def test_start_logs_topology_then_runs_serve(monkeypatch): - app = App(redis="redis://x") - calls: list[str] = [] - - def fake_log_topology(self): - calls.append("log") - - async def fake_serve(self): - calls.append("serve") - - monkeypatch.setattr(App, "_log_topology", fake_log_topology) - monkeypatch.setattr(App, "_serve", fake_serve) - - app.start() - - assert calls == ["log", "serve"] - - -def test_start_handles_keyboard_interrupt(monkeypatch, caplog): - app = App(redis="redis://x") - calls: list[str] = [] - - def fake_log_topology(self): - calls.append("log") - - async def fake_serve(self): - calls.append("serve") - raise KeyboardInterrupt - - monkeypatch.setattr(App, "_log_topology", fake_log_topology) - monkeypatch.setattr(App, "_serve", fake_serve) - - with caplog.at_level(logging.INFO, logger="runtime"): - app.start() - - assert calls == ["log", "serve"] - assert "interrupted" in caplog.text diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..eefd8bf --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,366 @@ +"""Boot, wiring, lifespans, connection sizing, and the switch behaviours the +operator-facing API rests on.""" + +import asyncio +from contextlib import asynccontextmanager + +import pytest + +from runtime import App, Service, Undeliverable, switches +from runtime.envelope import make_envelope +from runtime.node import Node, new_node_id +from runtime.service import run_every, run_once +from runtime.switches import SwitchBoard +from tests.helpers import serving + + +async def noop(ctx, params): ... + + +# ── wiring ─────────────────────────────────────────────────────────────────── + + +async def test_a_disabled_service_never_runs_its_lifespan(broker): + """The documented contract is stronger than 'not appended': this is how you + stop a browser launching at all, so the lifespan must not run.""" + opened = [] + + @asynccontextmanager + async def lifespan(config): + opened.append("yes") + yield {} + + svc = Service("svc", max_slots=1, lifespan=lifespan) + svc.register_action(noop, name="a") + + app = App(redis="unused://") + app.include(svc, enabled=False) + async with serving(app, broker): + await asyncio.sleep(0.1) + + assert opened == [] + assert app._services == [] + + +def test_include_rejects_something_that_is_not_a_service(): + with pytest.raises(TypeError, match="expects a Service"): + App(redis="unused://").include(object()) # type: ignore[arg-type] + + +async def test_a_lifespan_that_is_not_a_context_manager_fails_at_boot(broker): + """A common mistake — forgetting @asynccontextmanager — caught with a named + error instead of a protocol failure deep in the stack.""" + + # The actual mistake: an async generator with the @asynccontextmanager + # decorator left off, which yields an async_generator rather than a context + # manager. + async def not_a_cm(config): + yield {} + + svc = Service("svc", max_slots=1, lifespan=not_a_cm) + svc.register_action(noop, name="a") + app = App(redis="unused://") + app.include(svc) + + with pytest.raises(TypeError, match="async context manager"): + await app._serve(broker) + + +async def test_lifespans_are_torn_down_on_shutdown(broker): + events = [] + + @asynccontextmanager + async def lifespan(config): + events.append("open") + try: + yield {} + finally: + events.append("close") + + svc = Service("svc", max_slots=1, lifespan=lifespan) + svc.register_action(noop, name="a") + app = App(redis="unused://") + app.include(svc) + + async with serving(app, broker): + await asyncio.sleep(0.05) + + assert events == ["open", "close"] + + +async def test_an_already_entered_lifespan_is_closed_when_a_later_one_fails(broker): + """Otherwise a boot failure leaks whatever the earlier services opened.""" + events = [] + + @asynccontextmanager + async def good(config): + events.append("open-good") + try: + yield {} + finally: + events.append("close-good") + + @asynccontextmanager + async def bad(config): + raise RuntimeError("cannot connect") + yield {} # pragma: no cover + + first = Service("first", max_slots=1, lifespan=good) + first.register_action(noop, name="a") + second = Service("second", max_slots=1, lifespan=bad) + second.register_action(noop, name="b") + + app = App(redis="unused://") + app.include(first) + app.include(second) + + with pytest.raises(RuntimeError, match="cannot connect"): + await app._serve(broker) + + assert events == ["open-good", "close-good"] + + +# ── connection sizing ──────────────────────────────────────────────────────── + + +def test_slot_count_sums_budgets_overrides_and_per_registration_caps(): + """It sizes the connection pool. Getting it wrong shows up under load as a + crash, not as a misconfiguration.""" + a = Service("a", max_slots=4) + a.register_action(noop, name="a1") + a.register_action(noop, name="a2", max_slots=2) + b = Service("b", max_slots=1) + b.register_action(noop, name="b1") + + app = App(redis="unused://") + app.include(a) + app.include(b, max_slots=10) # override wins over the Service's own + + assert app._slot_count() == 4 + 2 + 10 + + +def test_connect_sets_the_two_defaults_that_are_wrong_for_this_runtime(monkeypatch): + """redis-py's socket_timeout default is exactly the runtime's BLOCK window, + and its connection cap is unrelated to how many workers will block.""" + captured = {} + + class FakeRedis: + @classmethod + def from_url(cls, url, **kwargs): + captured.update(kwargs) + return cls() + + import runtime.redis_io as redis_io + + monkeypatch.setattr(redis_io, "Redis", FakeRedis) + redis_io.connect("redis://x", max_connections=24, max_block_s=5.0) + + assert captured["socket_timeout"] == 10.0, "must exceed the longest block" + assert captured["max_connections"] == 24 + assert captured["decode_responses"] is True + + +# ── the reply rendezvous ───────────────────────────────────────────────────── + + +def test_node_ids_are_unique_per_instance(): + """Two Apps in one process would otherwise share a reply channel and each + silently discard half the other's replies.""" + assert new_node_id() != new_node_id() + + +async def test_a_dead_pump_fails_waiting_calls_at_once(broker): + """The whole reason the pump is supervised: a timeout handles a slow + executant, but only the pump knows the channel itself has gone away.""" + node = Node(broker) + pump = asyncio.create_task(node.pump()) + await asyncio.sleep(0.05) + future = node.register("msg-1") + + pump.cancel() + await asyncio.gather(pump, return_exceptions=True) + + with pytest.raises(Undeliverable): + await asyncio.wait_for(future, 1.0) + + +async def test_a_reply_for_an_unknown_call_is_dropped(broker): + """A late reply arrives after the caller gave up. Resolving a missing or + already-done future would raise inside the pump and take every OTHER call + down with it.""" + node = Node(broker) + + node.resolve({"id": "never-sent", "ok": True, "result": 1}) + + future = node.register("msg-1") + node.resolve({"id": "msg-1", "ok": True, "result": 1}) + node.resolve({"id": "msg-1", "ok": True, "result": 2}) # must not raise + assert await future == 1 + + +# ── switches ───────────────────────────────────────────────────────────────── + + +async def test_pausing_a_service_pauses_everything_it_registers(broker): + """The master switch — the reason a service name may not collide with a + registration name.""" + handled, ticked = [], [] + svc = Service("catalog", max_slots=1) + + @svc.action("catalog.sync") + async def sync(ctx, params): + handled.append(1) + + @svc.every("0.02s", id="doctor") + async def doctor(ctx): + ticked.append(1) + + await broker.switch_set("catalog", False) # the SERVICE name, not the action + app = App(redis="unused://", switch_interval=0.02, claim_block="0.05s") + app.include(svc) + + async with serving(app, broker): + await asyncio.sleep(0.3) + assert ticked == [], "the producer should be paused by the master switch" + await broker.append( + svc._actions[0].stream, + make_envelope(kind="dispatch", name="catalog.sync"), + {}, + ) + await asyncio.sleep(0.3) + + assert handled == [], "the action should be paused by the master switch" + + +async def test_a_failed_refresh_keeps_the_previous_snapshot(broker): + """Losing Redis must not silently resume everything that was paused.""" + + class Broken: + def __init__(self, inner): + self._inner = inner + + def __getattr__(self, name): + return getattr(self._inner, name) + + async def switches_get_all(self): + raise RuntimeError("redis is gone") + + board = SwitchBoard(broker) + await board.set("paused.thing", False) + assert not board.is_enabled("paused.thing") + + board._broker = Broken(broker) + await board.refresh() + + assert not board.is_enabled("paused.thing"), "a failed refresh resumed it" + + +async def test_a_stale_refresh_cannot_undo_a_local_flip(broker): + """refresh() awaits its read then assigns, so without the generation counter + a set() landing mid-read would be silently clobbered.""" + released = asyncio.Event() + + class SlowRead: + def __init__(self, inner): + self._inner = inner + + def __getattr__(self, name): + return getattr(self._inner, name) + + async def switches_get_all(self): + await released.wait() + return {} # a stale view: nothing paused + + async def switch_set(self, name, enabled): + return await self._inner.switch_set(name, enabled) + + board = SwitchBoard(SlowRead(broker)) + refreshing = asyncio.create_task(board.refresh()) + await asyncio.sleep(0.02) + + await board.set("thing", False) # lands while the read is in flight + released.set() + await refreshing + + assert not board.is_enabled("thing"), "the stale read undid the local flip" + + +async def test_the_operator_helpers_honour_the_namespace(broker, redis, namespace): + """`switches.disable` opens its own connection; against the wrong key it + would look like the switch simply did nothing.""" + url = "redis://localhost:6379/0" + + import runtime.switches as switches_module + + async def fake_connect(u, *, namespace=""): + from runtime.redis_io import RedisBroker + + return RedisBroker(redis, namespace=namespace) + + original = switches_module._with_broker + switches_module._with_broker = lambda u, ns: fake_connect(u, namespace=ns) + try: + await switches.disable(url, "thing", namespace=namespace) + assert await switches.status(url, namespace=namespace) == {"thing": False} + await switches.enable(url, "thing", namespace=namespace) + assert await switches.status(url, namespace=namespace) == {"thing": True} + finally: + switches_module._with_broker = original + + assert await redis.hget(f"{namespace}:switches", "thing") == "1" + + +# ── producer resilience ────────────────────────────────────────────────────── + + +async def test_one_bad_tick_does_not_kill_the_schedule(broker, caplog): + ticks = [] + svc = Service("svc") + + @svc.every("0.01s", id="flaky") + async def flaky(ctx): + ticks.append(1) + if len(ticks) == 1: + raise RuntimeError("first tick failed") + + with caplog.at_level("ERROR", logger="runtime"): + task = asyncio.create_task( + run_every(svc._every[0], ctx=None, service_name="svc") + ) + await asyncio.sleep(0.15) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + assert len(ticks) > 1, "the loop stopped after the failing tick" + assert "every 'flaky' tick failed" in caplog.text + + +async def test_a_one_shot_waits_out_its_delay(broker): + fired = [] + svc = Service("svc") + + @svc.once(delay="0.2s", id="warmup") + async def warmup(ctx): + fired.append(1) + + task = asyncio.create_task(run_once(svc._once[0], ctx=None, service_name="svc")) + await asyncio.sleep(0.05) + assert fired == [], "fired before its delay elapsed" + await asyncio.wait_for(task, 2.0) + + assert fired == [1] + + +async def test_a_failing_one_shot_is_logged_and_returns(broker, caplog): + svc = Service("svc") + + @svc.once(id="boom") + async def boom(ctx): + raise RuntimeError("nope") + + with caplog.at_level("ERROR", logger="runtime"): + await asyncio.wait_for( + run_once(svc._once[0], ctx=None, service_name="svc"), 2.0 + ) + + assert "once 'boom' failed" in caplog.text diff --git a/tests/test_context.py b/tests/test_context.py deleted file mode 100644 index ade9262..0000000 --- a/tests/test_context.py +++ /dev/null @@ -1,39 +0,0 @@ -import json -import uuid - -from runtime.broker import stream_name -from runtime.context import RuntimeContext - - -async def test_emit_lands_on_stream(broker, redis): - """ctx.emit(type, **payload) appends the json payload to the routed stream — - even with no consumer set up (producer-only-process case).""" - event_type = f"restock-{uuid.uuid4().hex}" - ctx = RuntimeContext(broker) - assert ctx.resources == {} and ctx.config == {} # producer-style defaults - - await ctx.emit(event_type, sku="ABC", qty=3) - - entries = await redis.xrange(stream_name(event_type)) - assert len(entries) == 1 - _msg_id, fields = entries[0] - assert json.loads(fields["data"]) == {"sku": "ABC", "qty": 3} - - await redis.delete(stream_name(event_type)) - - -async def test_emit_then_consume_roundtrip(broker, redis): - event_type = f"shopping-{uuid.uuid4().hex}" - stream = stream_name(event_type) - await broker.ensure_stream(stream) - - ctx = RuntimeContext(broker, config={"base_url": "http://x"}) - assert ctx.config == {"base_url": "http://x"} - await ctx.emit(event_type, name="world") - - msgs = await broker.claim(stream, consumer="c1", count=1, block_ms=500) - assert len(msgs) == 1 - _id, payload = msgs[0] - assert payload == {"name": "world"} - - await redis.delete(stream) diff --git a/tests/test_duration.py b/tests/test_duration.py new file mode 100644 index 0000000..b10dc04 --- /dev/null +++ b/tests/test_duration.py @@ -0,0 +1,47 @@ +import pytest + +from runtime.duration import parse_duration, to_ms + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("1s", 1.0), + ("1.5s", 1.5), + ("10min", 600.0), + ("1h", 3600.0), + ("500ms", 0.5), + ("2m", 120.0), + ("10", 10.0), + (3, 3.0), + ], +) +def test_parse_duration(value, expected): + assert parse_duration(value) == expected + + +@pytest.mark.parametrize("bad", ["0s", "-1s", "abc", "10/min"]) +def test_parse_duration_rejects_bad(bad): + with pytest.raises(ValueError): + parse_duration(bad) + + +def test_parse_duration_allows_zero_when_asked(): + assert parse_duration("0s", allow_zero=True) == 0.0 + assert parse_duration(0, allow_zero=True) == 0.0 + with pytest.raises(ValueError): + parse_duration("-1s", allow_zero=True) + + +@pytest.mark.parametrize( + ("value", "expected"), + [("500ms", 500), ("1.5s", 1500), ("10min", 600_000), (3, 3000), (0, 0)], +) +def test_to_ms(value, expected): + assert to_ms(value) == expected + + +def test_to_ms_passes_none_through(): + """An optional ttl=/delay= flows straight to the envelope without every call + site repeating the same conditional.""" + assert to_ms(None) is None diff --git a/tests/test_envelope.py b/tests/test_envelope.py new file mode 100644 index 0000000..7d41178 --- /dev/null +++ b/tests/test_envelope.py @@ -0,0 +1,132 @@ +import pytest + +from runtime.envelope import ( + ENVELOPE_VERSION, + Envelope, + derive_deadline, + make_envelope, +) + +BASE = 1_000_000 + + +@pytest.mark.parametrize( + ("parent_dl", "ttl_ms", "expected"), + [ + (None, 5_000, BASE + 5_000), # no parent, own ttl + (BASE + 2_000, 5_000, BASE + 2_000), # parent is narrower — parent wins + (BASE + 9_000, 5_000, BASE + 5_000), # own is narrower — own wins + (BASE + 2_000, 0, BASE + 2_000), # ttl=0 inherits, never widens + (BASE + 2_000, None, BASE + 2_000), # no ttl inherits + (None, 0, None), # nothing anywhere + (None, None, None), + ], +) +def test_derive_deadline_truth_table(parent_dl, ttl_ms, expected): + assert derive_deadline(parent_dl, ttl_ms, base_ms=BASE) == expected + + +def test_a_child_can_only_narrow_the_budget_it_was_handed(): + """The property the whole cascade rests on: no leaf of a call chain can + outlive the request that started it, however generous its own ttl.""" + root = make_envelope(kind="call", name="a", ttl_ms=5_000) + + child = make_envelope(kind="call", name="b", parent=root, ttl_ms=99_000) + + assert child.dl == root.dl + + +def test_each_root_starts_its_own_correlation(): + first = make_envelope(kind="call", name="a", meta={"tenant": "x"}) + second = make_envelope(kind="call", name="a") + + assert first.rid != second.rid, "unrelated requests must not share a rid" + assert first.lvl == 0 + assert first.caller is None + assert first.meta == {"tenant": "x"} + assert first.v == ENVELOPE_VERSION + + +def test_child_inherits_correlation_and_records_provenance(): + root = make_envelope(kind="call", name="a", ttl_ms=5_000, meta={"tenant": "x"}) + + child = make_envelope(kind="dispatch", name="b", parent=root, meta={"step": 1}) + + assert child.rid == root.rid + assert child.id != root.id + assert child.lvl == 1 + assert child.caller == "a" + assert child.meta == {"tenant": "x", "step": 1}, "shallow merge, child wins" + + +def test_child_meta_overrides_the_parents_value_for_the_same_key(): + root = make_envelope(kind="call", name="a", meta={"tenant": "x"}) + + child = make_envelope(kind="call", name="b", parent=root, meta={"tenant": "y"}) + + assert child.meta == {"tenant": "y"} + + +def test_mutating_a_childs_meta_does_not_reach_back_into_the_parent(): + root = make_envelope(kind="call", name="a", meta={"tenant": "x"}) + + child = make_envelope(kind="call", name="b", parent=root) + child.meta["step"] = 1 + + assert root.meta == {"tenant": "x"} + + +def test_events_do_not_inherit_the_parents_deadline(): + """An event has no outcome channel, so an inherited deadline would make a + saturated subscriber drop notifications silently.""" + root = make_envelope(kind="call", name="a", ttl_ms=5_000) + + event = make_envelope(kind="event", name="e", parent=root) + + assert event.dl is None + assert event.rid == root.rid, "correlation still propagates" + assert event.caller == "a" + + +def test_an_event_can_still_set_its_own_deadline(): + root = make_envelope(kind="call", name="a", ttl_ms=5_000) + + event = make_envelope(kind="event", name="e", parent=root, ttl_ms=1_000) + + assert event.dl is not None + + +def test_a_delayed_messages_budget_starts_at_the_due_time(): + """Otherwise call(ttl="5s", delay="10s") would always arrive expired.""" + env = make_envelope(kind="dispatch", name="d", ttl_ms=5_000, delay_ms=10_000) + + remaining = env.remaining_ms() + assert remaining is not None and remaining > 14_000 + + +def test_remaining_is_none_without_a_deadline(): + assert make_envelope(kind="event", name="e").remaining_ms() is None + + +def test_remaining_goes_negative_once_past_the_deadline(): + env = make_envelope(kind="call", name="a", ttl_ms=1_000) + assert env.dl is not None + + assert env.remaining_ms(at=env.dl + 500) == -500 + + +def test_an_old_reader_ignores_a_field_it_has_never_seen(): + """Half of the forward-compatibility rule. The other half is that every + field added after v0.3 carries a default — otherwise a message from an + un-upgraded producer becomes a poison pill, and there is no dead-letter.""" + env = make_envelope(kind="call", name="a") + raw = env.model_dump_json()[:-1] + ',"field_from_the_future":1}' + + assert Envelope.model_validate_json(raw).id == env.id + + +def test_only_the_identifying_fields_are_required(): + """Guards the compatibility rule against the next person to add a field.""" + required = {name for name, f in Envelope.model_fields.items() if f.is_required()} + + assert required == {"id", "rid", "kind", "name"} diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..1045d84 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,66 @@ +import pytest + +from runtime.errors import ( + ActionError, + CallTimeout, + Expired, + ParamsInvalid, + RemoteError, +) + + +def test_wire_round_trip_rebuilds_the_class(): + wire = ParamsInvalid("bad sku", data={"field": "sku"}).to_wire() + assert wire == { + "code": "PARAMS_INVALID", + "message": "bad sku", + "data": {"field": "sku"}, + } + + back = ActionError.from_wire(wire) + assert isinstance(back, ParamsInvalid) + assert back.message == "bad sku" + assert back.data == {"field": "sku"} + + +def test_unknown_code_degrades_to_remote_error_keeping_the_code(): + """An executant on a newer version can report a code this process has never + heard of. It must still raise something, and `err.code` must survive so the + caller can branch on it.""" + back = ActionError.from_wire({"code": "FROM_THE_FUTURE", "message": "?"}) + + assert isinstance(back, RemoteError) + assert back.code == "FROM_THE_FUTURE" + + +def test_missing_wire_fields_do_not_explode(): + back = ActionError.from_wire({}) + + assert isinstance(back, RemoteError) + assert back.data == {} + + +def test_wrap_passes_action_errors_through_untouched(): + original = CallTimeout("gave up") + assert ActionError.wrap(original) is original + + +def test_wrap_keeps_the_class_name_of_a_plain_exception(): + """The caller shares no code with the executant, so the only way it can tell + a ValueError from a KeyError is if the name crosses the wire.""" + wrapped = ActionError.wrap(ValueError("boom")) + + assert wrapped.code == "REMOTE" + assert wrapped.message == "ValueError: boom" + assert wrapped.data == {"exception": "ValueError"} + + +def test_every_error_is_catchable_as_the_base(): + for err in (CallTimeout("a"), Expired("b"), ParamsInvalid("c"), RemoteError("d")): + with pytest.raises(ActionError): + raise err + + +def test_subclasses_register_distinct_codes(): + codes = [c.CODE for c in ActionError._registry.values()] + assert len(codes) == len(set(codes)), f"duplicate error codes: {codes}" diff --git a/tests/test_example_minimal.py b/tests/test_example_minimal.py deleted file mode 100644 index c6808ef..0000000 --- a/tests/test_example_minimal.py +++ /dev/null @@ -1,24 +0,0 @@ -import asyncio - -from fakeredis.aioredis import FakeRedis - -from examples.minimal.main import build_app -from runtime.redis_io import RedisBroker -from tests.helpers import wait_for_output - - -async def test_minimal_example_end_to_end(capsys): - """Smoke: the real examples.minimal app, end-to-end on a fakeredis-backed - broker. The scheduler's producer emits → the greet handler prints. Asserts - the full producer → broker → handler path through App._serve.""" - app = build_app() - broker = RedisBroker(FakeRedis(decode_responses=True)) - serve = asyncio.create_task(app._serve(broker)) - try: - out = await wait_for_output(capsys, "[minimal] hello world") - finally: - serve.cancel() - await asyncio.gather(serve, return_exceptions=True) - await broker.aclose() - - assert "[minimal] hello world" in out diff --git a/tests/test_example_orders.py b/tests/test_example_orders.py deleted file mode 100644 index cf20abc..0000000 --- a/tests/test_example_orders.py +++ /dev/null @@ -1,34 +0,0 @@ -import asyncio - -from fakeredis.aioredis import FakeRedis - -from examples.orders.pipeline import load, warehouse -from runtime import App -from runtime.redis_io import RedisBroker -from tests.helpers import wait_for_output - - -async def test_orders_pipeline_end_to_end(capsys): - """The orders pipeline, both sides co-located in one process on fakeredis - (main.py runs them as two processes against a real Redis). The load Scheduler - (catalog lifespan) emits order.placed → the warehouse Pool (store lifespan) - fulfills — exercising producer- AND consumer-side lifespan resources.""" - app = App(redis="unused://") - app.include(warehouse, config={"dsn": "memory://"}) - app.include(load) - - broker = RedisBroker(FakeRedis(decode_responses=True)) - serve = asyncio.create_task(app._serve(broker)) - try: - out = await wait_for_output( - capsys, - "[producer] placed order", - "[consumer] fulfilled order", - ) - finally: - serve.cancel() - await asyncio.gather(serve, return_exceptions=True) - await broker.aclose() - - assert "[producer] placed order" in out - assert "[consumer] fulfilled order" in out diff --git a/tests/test_example_poisson.py b/tests/test_example_poisson.py deleted file mode 100644 index c934f4f..0000000 --- a/tests/test_example_poisson.py +++ /dev/null @@ -1,32 +0,0 @@ -import asyncio -import random - -from fakeredis.aioredis import FakeRedis - -from examples.poisson.main import build_app -from runtime.redis_io import RedisBroker -from tests.helpers import wait_for_output - - -async def test_poisson_example_end_to_end(capsys): - """The poisson app on fakeredis: a scheduler opens a window, the `generator` - pool consumes it and emits a Poisson burst of `request` events, the `workers` - pool consumes those — exercising a pool that is consumer AND producer. The - seed fixes the draw so at least one arrival is emitted.""" - random.seed(0) - app = build_app() - broker = RedisBroker(FakeRedis(decode_responses=True)) - serve = asyncio.create_task(app._serve(broker)) - try: - out = await wait_for_output( - capsys, - "[generator] window", - "[workers] handling request", - ) - finally: - serve.cancel() - await asyncio.gather(serve, return_exceptions=True) - await broker.aclose() - - assert "[generator] window" in out - assert "[workers] handling request" in out diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..bfd42b9 --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,99 @@ +"""The examples are the acceptance fixture: if they run, the public API works. + +Each builds its real App and runs it against a fake Redis, so these break the +moment an example drifts from the API it is supposed to demonstrate. +""" + +from runtime import App +from tests.helpers import serving, wait_for_output + + +async def test_minimal_dispatches_and_handles(broker, capsys): + from examples.minimal.main import build_app + + async with serving(build_app(), broker): + await wait_for_output(capsys, "[minimal] hello world") + + +async def test_rpc_shows_a_value_an_error_and_a_timeout(broker, capsys): + from examples.rpc.main import build_app + + async with serving(build_app(), broker): + await wait_for_output( + capsys, + "[rpc] quote came back:", + "OUT_OF_STOCK", + "bad params rejected before the handler ran", + "[rpc] gave up:", + timeout=8.0, + ) + + +async def test_jobs_shows_task_status_delay_and_a_switch(broker, capsys): + from examples.jobs.main import build_app + + async with serving(build_app(), broker): + await wait_for_output( + capsys, + "[jobs] dispatched", + "task finished: done", + "scheduled a build for", + "queued, not lost", + "resumed; the backlog drains", + timeout=10.0, + ) + + +async def test_events_reach_every_subscriber(broker, capsys): + from examples.events.main import build_app + + async with serving(build_app(), broker): + await wait_for_output( + capsys, + "[storefront] order", + "[analytics] indexed", + "[billing] invoiced", + "[audit] recorded", + timeout=8.0, + ) + + +async def test_poisson_service_consumes_and_produces(broker, capsys): + import random + + from examples.poisson.main import build_app + + random.seed(0) # so the window is not empty + async with serving(build_app(), broker): + await wait_for_output( + capsys, "[generator] window", "[workers] handling request", timeout=8.0 + ) + + +async def test_orders_pipeline_end_to_end(broker, capsys): + """The orders example ships as two processes; here both sides run in one App, + which is the point — they are coupled only by the action name.""" + from examples.orders.pipeline import load, warehouse + + app = App(redis="unused://") + app.include(warehouse, config={"dsn": "memory://"}) + app.include(load) + + async with serving(app, broker): + await wait_for_output( + capsys, + "[producer] placed order", + "[consumer] fulfilled order", + timeout=8.0, + ) + + +def test_playwright_topology_builds_without_playwright_installed(): + """The heavy import is lazy, inside the lifespan, so the example can be read + and its wiring inspected without a browser toolchain.""" + from examples.playwright.main import build_app + + app = build_app() + app._log_topology() + + assert [inc.service.name for inc in app._services] == ["browser"] diff --git a/tests/test_redis_io.py b/tests/test_redis_io.py index 9b99c03..650befb 100644 --- a/tests/test_redis_io.py +++ b/tests/test_redis_io.py @@ -1,69 +1,324 @@ import uuid -from runtime.broker import stream_name -from runtime.redis_io import PAYLOAD_FIELD, RedisBroker, from_fields, to_fields +import pytest +from runtime.broker import ( + START_HEAD, + START_TAIL, + WORKERS_GROUP, + Delivery, + Undecodable, + action_stream, + event_stream, + reply_key, + task_key, +) +from runtime.envelope import make_envelope, now_ms +from runtime.redis_io import ( + META_FIELD, + PAYLOAD_FIELD, + RedisBroker, + from_fields, + to_fields, +) -def test_field_codec_roundtrips_nested_payload(): - payload = { - "sku": "ABC", - "qty": 3, - "tags": ["sale", "priority"], - "meta": {"fragile": True, "note": None}, - } - fields = to_fields(payload) +def a_stream() -> str: + return action_stream(f"t-{uuid.uuid4().hex}") + + +# ── codec ──────────────────────────────────────────────────────────────────── + + +def test_envelope_and_payload_land_in_separate_fields(): + """So a deadline or due-time check never decodes a body it may discard.""" + env = make_envelope(kind="call", name="a", ttl_ms=5_000) + + fields = to_fields(env, {"sku": "barrel", "qty": 2}) + + assert set(fields) == {META_FIELD, PAYLOAD_FIELD} + assert '"sku":"barrel"' in fields[PAYLOAD_FIELD] + + +def test_codec_round_trip(): + env = make_envelope(kind="dispatch", name="a", meta={"tenant": "x"}) + payload = {"nested": {"list": [1, 2, 3]}, "n": None} + + got = from_fields("1-0", to_fields(env, payload)) + + assert isinstance(got, Delivery) + assert got.id == "1-0" + assert got.envelope.id == env.id + assert got.envelope.meta == {"tenant": "x"} + assert got.params == payload + + +@pytest.mark.parametrize( + ("fields", "reason_fragment"), + [ + ({}, "missing"), + ({META_FIELD: "not json"}, "bad envelope"), + ({META_FIELD: '{"id":"x"}'}, "bad envelope"), # missing required fields + ], +) +def test_undecodable_is_reported_not_raised(fields, reason_fragment): + """It has to reach the worker so it can be acked and logged — raising would + take the rest of the batch down and leave a poison pill pending.""" + got = from_fields("1-0", fields) + + assert isinstance(got, Undecodable) + assert reason_fragment in got.reason + + +def test_undecodable_payload_is_reported(): + env = make_envelope(kind="call", name="a") + + got = from_fields("1-0", {META_FIELD: env.model_dump_json(), PAYLOAD_FIELD: "["}) + + assert isinstance(got, Undecodable) + + +def test_non_object_payload_is_rejected(): + env = make_envelope(kind="call", name="a") + + got = from_fields("1-0", {META_FIELD: env.model_dump_json(), PAYLOAD_FIELD: "[1]"}) - assert set(fields) == {PAYLOAD_FIELD} - assert from_fields(fields) == payload + assert isinstance(got, Undecodable) + assert "not an object" in got.reason -def test_from_fields_missing_payload_returns_empty_event(): - assert from_fields({"other": "{}"}) == {} +def test_missing_payload_field_decodes_to_empty_params(): + env = make_envelope(kind="event", name="e") + got = from_fields("1-0", {META_FIELD: env.model_dump_json()}) -async def test_redisbroker_roundtrip(broker, redis): - stream = stream_name(f"test-{uuid.uuid4().hex}") + assert isinstance(got, Delivery) + assert got.params == {} - await broker.ensure_stream(stream) - await broker.ensure_stream(stream) # idempotent: BUSYGROUP is a no-op - msg_id = await broker.append(stream, {"sku": "ABC", "qty": 3}) - assert msg_id +# ── streams ────────────────────────────────────────────────────────────────── - msgs = await broker.claim(stream, consumer="c1", count=1, block_ms=1000) - assert len(msgs) == 1 - read_id, payload = msgs[0] - assert read_id == msg_id - assert payload == {"sku": "ABC", "qty": 3} - await broker.ack(stream, read_id) +async def test_append_claim_ack_round_trip(broker): + stream = a_stream() + await broker.ensure_group(stream, WORKERS_GROUP, start=START_HEAD) + env = make_envelope(kind="dispatch", name="a") - # nothing left undelivered - assert await broker.claim(stream, consumer="c1", count=1, block_ms=100) == [] + msg_id = await broker.append(stream, env, {"x": 1}) + got = await broker.claim( + stream, WORKERS_GROUP, consumer="c1", count=10, block_ms=50 + ) - await redis.delete(stream) + assert len(got) == 1 + assert isinstance(got[0], Delivery) + assert got[0].id == msg_id + assert got[0].envelope.id == env.id + assert got[0].params == {"x": 1} + await broker.ack(stream, WORKERS_GROUP, msg_id) + left = await broker.claim( + stream, WORKERS_GROUP, consumer="c1", count=10, block_ms=50 + ) + assert left == [] -async def test_namespace_isolates_streams(redis): - """Two brokers with different namespaces over one Redis don't see each other's - messages — the case for running dev + staging against a single Redis.""" - event = stream_name(f"test-{uuid.uuid4().hex}") + +async def test_ensure_group_is_idempotent(broker): + """Safe on every boot and every replica — BUSYGROUP is a no-op.""" + stream = a_stream() + + await broker.ensure_group(stream, WORKERS_GROUP, start=START_HEAD) + await broker.ensure_group(stream, WORKERS_GROUP, start=START_HEAD) + + +async def test_a_tail_subscriber_does_not_replay_history(broker): + """Deploying a new event subscriber must not dump the retained stream into + it. This is what v0.3 changes from v0.2's start-at-0.""" + stream = event_stream(f"t-{uuid.uuid4().hex}") + await broker.ensure_group(stream, "early", start=START_HEAD) + await broker.append(stream, make_envelope(kind="event", name="e"), {"n": 1}) + + await broker.ensure_group(stream, "late", start=START_TAIL) + + assert await broker.claim(stream, "late", consumer="c", count=10, block_ms=50) == [] + caught_up = await broker.claim(stream, "early", consumer="c", count=10, block_ms=50) + assert len(caught_up) == 1 + + +async def test_every_group_gets_its_own_copy(broker): + """Fan-out: N subscribers, N groups, one copy each — as opposed to an action, + where one group means exactly one executant.""" + stream = event_stream(f"t-{uuid.uuid4().hex}") + for group in ("analytics", "billing"): + await broker.ensure_group(stream, group, start=START_HEAD) + + await broker.append(stream, make_envelope(kind="event", name="e"), {"n": 1}) + + for group in ("analytics", "billing"): + got = await broker.claim(stream, group, consumer="c", count=10, block_ms=50) + assert len(got) == 1, f"{group} did not get its copy" + + +async def test_namespace_isolates_two_environments(redis): dev = RedisBroker(redis, namespace="dev") - stg = RedisBroker(redis, namespace="staging") + prod = RedisBroker(redis, namespace="prod") + stream = a_stream() + for broker in (dev, prod): + await broker.ensure_group(stream, WORKERS_GROUP, start=START_HEAD) + + await dev.append(stream, make_envelope(kind="dispatch", name="a"), {"who": "dev"}) + + seen_by_prod = await prod.claim( + stream, WORKERS_GROUP, consumer="c", count=10, block_ms=50 + ) + seen_by_dev = await dev.claim( + stream, WORKERS_GROUP, consumer="c", count=10, block_ms=50 + ) + assert seen_by_prod == [] + assert len(seen_by_dev) == 1 + + +# ── reply rendezvous ───────────────────────────────────────────────────────── + + +async def test_reply_round_trip(broker): + key = reply_key(f"node-{uuid.uuid4().hex}") + + await broker.push_reply(key, {"id": "abc", "ok": True, "result": 42}, ttl_ms=60_000) + + assert await broker.pop_reply(key, timeout_s=1) == { + "id": "abc", + "ok": True, + "result": 42, + } + + +async def test_pop_reply_returns_none_when_nothing_arrives(broker): + key = reply_key(f"node-{uuid.uuid4().hex}") + + assert await broker.pop_reply(key, timeout_s=0.05) is None + + +async def test_reply_channel_expires(broker, redis, namespace): + """A caller that dies must not leave its channel accumulating forever.""" + key = reply_key(f"node-{uuid.uuid4().hex}") + + await broker.push_reply(key, {"id": "abc"}, ttl_ms=30_000) + + assert 0 < await redis.pttl(f"{namespace}:{key}") <= 30_000 + + +# ── task records ───────────────────────────────────────────────────────────── + + +async def test_task_record_round_trip(broker): + key = task_key(uuid.uuid4().hex) + + await broker.put_task(key, {"state": "done", "result": {"n": 1}}, ttl_ms=60_000) + + assert await broker.get_task(key) == {"state": "done", "result": {"n": 1}} + + +async def test_missing_task_is_none(broker): + assert await broker.get_task(task_key("never-written")) is None + + +# ── delayed delivery ───────────────────────────────────────────────────────── + + +async def test_scheduled_message_is_not_delivered_before_it_is_due(broker): + stream = a_stream() + await broker.ensure_group(stream, WORKERS_GROUP, start=START_HEAD) + env = make_envelope(kind="dispatch", name="a", delay_ms=60_000) + + await broker.schedule(now_ms() + 60_000, stream, env, {"x": 1}) + + assert await broker.sweep(now_ms(), limit=100, maxlen=None) == 0 + left = await broker.claim( + stream, WORKERS_GROUP, consumer="c", count=10, block_ms=50 + ) + assert left == [] + + +async def test_sweep_moves_a_due_message_onto_its_stream(broker): + stream = a_stream() + await broker.ensure_group(stream, WORKERS_GROUP, start=START_HEAD) + env = make_envelope(kind="dispatch", name="a") + + await broker.schedule(now_ms() - 1, stream, env, {"x": 1}) + moved = await broker.sweep(now_ms(), limit=100, maxlen=None) + + assert moved == 1 + got = await broker.claim(stream, WORKERS_GROUP, consumer="c", count=10, block_ms=50) + assert len(got) == 1 + assert isinstance(got[0], Delivery) + assert got[0].envelope.id == env.id + assert got[0].params == {"x": 1} + + +async def test_ten_identical_delayed_messages_all_survive(broker): + """Sorted-set members are a SET. A member derived from the payload alone + would make these ten collapse into one — silently, with the last emit + overwriting the earlier ones' due times. The envelope id inside the member is + what prevents it, and this is the most likely thing a user of a load + simulator does.""" + stream = a_stream() + await broker.ensure_group(stream, WORKERS_GROUP, start=START_HEAD) + due = now_ms() - 1 + + for _ in range(10): + env = make_envelope(kind="dispatch", name="a") + await broker.schedule(due, stream, env, {"identical": "payload"}) + + assert await broker.sweep(now_ms(), limit=100, maxlen=None) == 10 + got = await broker.claim( + stream, WORKERS_GROUP, consumer="c", count=100, block_ms=50 + ) + assert len(got) == 10 + + +async def test_two_sweepers_never_move_the_same_message_twice(broker, redis): + """ZREM arbitrates between processes: exactly one gets the 1 back and owns + moving it. Without that, every replica would deliver every delayed message.""" + other = RedisBroker(redis) + stream = a_stream() + await broker.ensure_group(stream, WORKERS_GROUP, start=START_HEAD) + due = now_ms() - 1 + for _ in range(5): + await broker.schedule(due, stream, make_envelope(kind="dispatch", name="a"), {}) + + moved = await broker.sweep(now_ms(), limit=100, maxlen=None) + moved_other = await other.sweep(now_ms(), limit=100, maxlen=None) + + assert moved + moved_other == 5 + got = await broker.claim( + stream, WORKERS_GROUP, consumer="c", count=100, block_ms=50 + ) + assert len(got) == 5 + + +async def test_sweep_honours_the_limit(broker): + stream = a_stream() + due = now_ms() - 1 + for _ in range(5): + await broker.schedule(due, stream, make_envelope(kind="dispatch", name="a"), {}) + + assert await broker.sweep(now_ms(), limit=2, maxlen=None) == 2 + assert await broker.sweep(now_ms(), limit=100, maxlen=None) == 3 + + +# ── switches ───────────────────────────────────────────────────────────────── + + +async def test_absent_switch_means_enabled(broker): + """Nothing has to be seeded, and a fresh Redis behaves exactly like today.""" + assert await broker.switches_get_all() == {} - for b in (dev, stg): - await b.ensure_stream(event) - await dev.append(event, {"env": "dev"}) - await stg.append(event, {"env": "staging"}) - dev_msgs = await dev.claim(event, consumer="c", count=10, block_ms=100) - stg_msgs = await stg.claim(event, consumer="c", count=10, block_ms=100) +async def test_switch_set_and_read_back(broker): + await broker.switch_set("catalog", False) + await broker.switch_set("catalog.sync", True) - assert [p["env"] for _, p in dev_msgs] == ["dev"] - assert [p["env"] for _, p in stg_msgs] == ["staging"] - assert ( - await redis.exists(f"dev:{event}", f"staging:{event}") == 2 - ) # keys are prefixed + switches = await broker.switches_get_all() - await redis.delete(f"dev:{event}", f"staging:{event}") + assert switches["catalog"] is False + assert switches["catalog.sync"] is True diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py deleted file mode 100644 index 1101191..0000000 --- a/tests/test_scheduler.py +++ /dev/null @@ -1,206 +0,0 @@ -import asyncio -import uuid - -import pytest - -from runtime import Scheduler -from runtime.broker import stream_name -from runtime.scheduler import parse_duration, run_every, run_once - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - ("1s", 1.0), - ("1.5s", 1.5), - ("10min", 600.0), - ("1h", 3600.0), - ("500ms", 0.5), - ("2m", 120.0), - ("10", 10.0), - (3, 3.0), - ], -) -def test_parse_duration(value, expected): - assert parse_duration(value) == expected - - -@pytest.mark.parametrize("bad", ["0s", "-1s", "abc", "10/min"]) -def test_parse_duration_rejects_bad(bad): - with pytest.raises(ValueError): - parse_duration(bad) - - -def test_parse_duration_delay_allows_zero(): - assert parse_duration("0s", allow_zero=True) == 0.0 - assert parse_duration(0, allow_zero=True) == 0.0 - with pytest.raises(ValueError): - parse_duration("-1s", allow_zero=True) - - -def test_every_registers(): - sched = Scheduler("s") - - @sched.every("10min", id="restock") - async def restock(ctx): ... - - reg = sched._every[0] - assert reg.handler is restock - assert reg.interval == 600.0 - assert reg.id == "restock" - - -def test_every_id_defaults_to_fn_name(): - sched = Scheduler("s") - - @sched.every("1s") - async def tick(ctx): ... - - assert sched._every[0].id == "tick" - - -def test_once_registers(): - sched = Scheduler("s") - - @sched.once(id="warmup") - async def warmup(ctx): ... - - reg = sched._once[0] - assert reg.handler is warmup - assert reg.delay == 0.0 - assert reg.id == "warmup" - - -def test_once_with_delay(): - sched = Scheduler("s") - - @sched.once(delay="10min") - async def warmup(ctx): ... - - assert sched._once[0].delay == 600.0 - - -def test_once_id_defaults_to_fn_name(): - sched = Scheduler("s") - - @sched.once() - async def warmup(ctx): ... - - assert sched._once[0].id == "warmup" - - -async def test_run_every_emits_with_resources(broker, redis): - """A producer reads its scheduler's lifespan resources via ctx.resources and - emits — producer-side shared resources come from the scheduler's lifespan.""" - event_type = f"load-{uuid.uuid4().hex}" - state = {"n": 0} - done = asyncio.Event() - - sched = Scheduler("s") - - @sched.every("0.01s", id="t") - async def prod(ctx): - assert ctx.resources["api"] == "API" # from the scheduler lifespan - state["n"] += 1 - await ctx.emit(event_type, i=state["n"]) - if state["n"] >= 3: - done.set() - - reg = sched._every[0] - task = asyncio.create_task( - run_every(broker, reg, resources={"api": "API"}, config={}) - ) - await asyncio.wait_for(done.wait(), 2.0) - task.cancel() - await asyncio.gather(task, return_exceptions=True) - - entries = await redis.xrange(stream_name(event_type)) - assert len(entries) >= 3 - await redis.delete(stream_name(event_type)) - - -async def test_run_every_logs_and_continues_after_tick_failure(caplog, broker, redis): - event_type = f"load-{uuid.uuid4().hex}" - state = {"n": 0} - done = asyncio.Event() - sched = Scheduler("s") - - @sched.every("0.01s", id="flaky") - async def prod(ctx): - state["n"] += 1 - if state["n"] == 1: - raise RuntimeError("first tick failed") - await ctx.emit(event_type, recovered=True) - done.set() - - with caplog.at_level("ERROR", logger="runtime"): - task = asyncio.create_task( - run_every(broker, sched._every[0], resources={}, config={}) - ) - await asyncio.wait_for(done.wait(), 2.0) - task.cancel() - await asyncio.gather(task, return_exceptions=True) - - assert "every 'flaky' tick failed" in caplog.text - entries = await redis.xrange(stream_name(event_type)) - assert len(entries) == 1 - await redis.delete(stream_name(event_type)) - - -async def test_run_once_emits_once_with_resources(broker, redis): - """A one-shot producer reads its scheduler's lifespan resources and emits - exactly once; the runner returns on its own (no cancel needed).""" - event_type = f"load-{uuid.uuid4().hex}" - state = {"n": 0} - - sched = Scheduler("s") - - @sched.once(id="t") - async def prod(ctx): - assert ctx.resources["api"] == "API" # from the scheduler lifespan - state["n"] += 1 - await ctx.emit(event_type, i=state["n"]) - - await asyncio.wait_for( - run_once(broker, sched._once[0], resources={"api": "API"}, config={}), 2.0 - ) - - assert state["n"] == 1 - entries = await redis.xrange(stream_name(event_type)) - assert len(entries) == 1 - await redis.delete(stream_name(event_type)) - - -async def test_run_once_waits_for_delay(broker, redis): - """The delay defers the single fire — nothing is emitted before it elapses.""" - event_type = f"load-{uuid.uuid4().hex}" - sched = Scheduler("s") - - @sched.once(delay="0.2s", id="t") - async def prod(ctx): - await ctx.emit(event_type, ok=True) - - task = asyncio.create_task( - run_once(broker, sched._once[0], resources={}, config={}) - ) - await asyncio.sleep(0.02) # well before the delay elapses - assert len(await redis.xrange(stream_name(event_type))) == 0 - await asyncio.wait_for(task, 2.0) - assert len(await redis.xrange(stream_name(event_type))) == 1 - await redis.delete(stream_name(event_type)) - - -async def test_run_once_logs_and_returns_after_failure(caplog, broker, redis): - """A one-shot that raises is logged and the runner still returns (no loop).""" - sched = Scheduler("s") - - @sched.once(id="boom") - async def prod(ctx): - raise RuntimeError("nope") - - with caplog.at_level("ERROR", logger="runtime"): - await asyncio.wait_for( - run_once(broker, sched._once[0], resources={}, config={}), 2.0 - ) - - assert "once 'boom' failed" in caplog.text diff --git a/tests/test_serve.py b/tests/test_serve.py deleted file mode 100644 index 70b8e33..0000000 --- a/tests/test_serve.py +++ /dev/null @@ -1,262 +0,0 @@ -import asyncio -import uuid -from contextlib import asynccontextmanager - -import pytest - -import runtime.app as app_module -from runtime import App, Pool, Scheduler -from runtime.broker import stream_name - - -class FakeBroker: - def __init__(self) -> None: - self.ensured: list[str] = [] - self.closed = False - - async def append(self, stream, payload): - return "1-0" - - async def ensure_stream(self, stream): - self.ensured.append(stream) - - async def claim(self, stream, *, consumer, count, block_ms): - return [] - - async def ack(self, stream, message_id): - pass - - async def aclose(self): - self.closed = True - - -async def test_serve_enters_lifespan_runs_handle_and_closes(broker, redis): - consumes = f"ev-{uuid.uuid4().hex}" - stream = stream_name(consumes) - - life: list[str] = [] - handled = asyncio.Event() - - @asynccontextmanager - async def resource(config): - life.append("open") - try: - yield {"tag": config["tag"]} # POOL-scope resource from config - finally: - life.append("close") - - pool = Pool("p", max_slots=1, lifespan=resource) - - @pool.flow(consumes=consumes) - async def handler(ctx, event): - assert ctx.resources["tag"] == "T" # lifespan resource reached the handler - assert event == {"hello": 1} - handled.set() - - app = App(redis="unused://") - app.include(pool, config={"tag": "T"}) - - # pre-seed one event, then run the consumer side with the injected broker - await broker.ensure_stream(stream) - await broker.append(stream, {"hello": 1}) - - serve = asyncio.create_task(app._serve(broker)) - await asyncio.wait_for(handled.wait(), 3.0) - serve.cancel() - await asyncio.gather(serve, return_exceptions=True) - - assert life == ["open", "close"] # lifespan finally ran on shutdown - await redis.delete(stream) - - -async def test_serve_producer_drives_consumer(broker, redis): - """End-to-end: a producer in the same App emits events that the pool's flow - consumes — producer → broker → handler, all under one _serve.""" - event_type = f"sess-{uuid.uuid4().hex}" - handled = asyncio.Event() - seen: list[dict] = [] - - pool = Pool("p", max_slots=2) - - @pool.flow(consumes=event_type) - async def handler(ctx, event): - seen.append(event) - handled.set() - - sched = Scheduler("load") - - @sched.every("0.01s", id="load") - async def emit(ctx): - await ctx.emit(event_type, hit=1) - - app = App(redis="unused://") - app.include(pool) - app.include(sched) - - serve = asyncio.create_task(app._serve(broker)) - await asyncio.wait_for(handled.wait(), 3.0) - serve.cancel() - await asyncio.gather(serve, return_exceptions=True) - - assert seen and seen[0] == {"hit": 1} - await redis.delete(stream_name(event_type)) - - -async def test_serve_rejects_non_acm_lifespan(broker): - """A lifespan that doesn't return an async context manager (the classic - forgot-@asynccontextmanager mistake) fails fast at boot with a clear error.""" - - def not_a_cm(config): # missing @asynccontextmanager - return {"oops": True} - - pool = Pool("p", max_slots=1, lifespan=not_a_cm) # type: ignore[arg-type] - - @pool.flow(consumes="x") - async def handler(ctx, event): ... - - app = App(redis="unused://") - app.include(pool, config={}) - - with pytest.raises(TypeError, match="async context manager"): - await app._serve(broker) - - -async def test_serve_spawns_workers_from_overrides_and_flow_caps(monkeypatch): - started = asyncio.Event() - captured: list[dict] = [] - - async def fake_slot_worker( - broker, handler, *, consumes, consumer, resources, config, pool_sem - ): - captured.append( - { - "consumes": consumes, - "consumer": consumer, - "resources": resources, - "config": config, - "pool_sem": pool_sem, - } - ) - if len(captured) == 5: - started.set() - await asyncio.Event().wait() - - monkeypatch.setattr(app_module, "slot_worker", fake_slot_worker) - - @asynccontextmanager - async def resource(config): - yield {"tag": config["tag"]} - - pool = Pool("p", max_slots=10, lifespan=resource) - - @pool.flow(consumes="default") - async def default_handler(ctx, event): ... - - @pool.flow(consumes="limited", max_slots=2) - async def limited_handler(ctx, event): ... - - app = App(redis="unused://") - app.include(pool, max_slots=3, config={"tag": "T"}) - broker = FakeBroker() - - serve = asyncio.create_task(app._serve(broker)) - await asyncio.wait_for(started.wait(), 1.0) - serve.cancel() - await asyncio.gather(serve, return_exceptions=True) - - assert broker.ensured == [stream_name("default"), stream_name("limited")] - assert [c["consumes"] for c in captured].count("default") == 3 - assert [c["consumes"] for c in captured].count("limited") == 2 - assert {id(c["pool_sem"]) for c in captured} == {id(captured[0]["pool_sem"])} - assert all(c["resources"] == {"tag": "T"} for c in captured) - assert all(c["config"] == {"tag": "T"} for c in captured) - assert not broker.closed - - -async def test_serve_enters_scheduler_lifespan_and_passes_config(monkeypatch): - started = asyncio.Event() - life: list[str] = [] - seen: dict = {} - - async def fake_run_every(broker, reg, *, resources, config): - seen["id"] = reg.id - seen["resources"] = resources - seen["config"] = config - started.set() - await asyncio.Event().wait() - - monkeypatch.setattr(app_module, "run_every", fake_run_every) - - @asynccontextmanager - async def resource(config): - life.append(f"open:{config['api_url']}") - try: - yield {"api": config["api_url"]} - finally: - life.append("close") - - sched = Scheduler("load", lifespan=resource) - - @sched.every("1s", id="tick") - async def tick(ctx): ... - - app = App(redis="unused://") - app.include(sched, config={"api_url": "https://api.example"}) - broker = FakeBroker() - - serve = asyncio.create_task(app._serve(broker)) - await asyncio.wait_for(started.wait(), 1.0) - serve.cancel() - await asyncio.gather(serve, return_exceptions=True) - - assert life == ["open:https://api.example", "close"] - assert seen == { - "id": "tick", - "resources": {"api": "https://api.example"}, - "config": {"api_url": "https://api.example"}, - } - assert not broker.closed - - -async def test_serve_closes_entered_lifespans_when_startup_fails(): - life: list[str] = [] - - @asynccontextmanager - async def pool_resource(config): - life.append("pool-open") - try: - yield {} - finally: - life.append("pool-close") - - @asynccontextmanager - async def broken_scheduler_resource(config): - raise RuntimeError("scheduler startup failed") - yield {} - - pool = Pool("p", max_slots=1, lifespan=pool_resource) - sched = Scheduler("s", lifespan=broken_scheduler_resource) - - @sched.every("1s") - async def tick(ctx): ... - - app = App(redis="unused://") - app.include(pool) - app.include(sched) - - with pytest.raises(RuntimeError, match="scheduler startup failed"): - await app._serve(FakeBroker()) - - assert life == ["pool-open", "pool-close"] - - -async def test_serve_closes_owned_broker_but_not_injected(monkeypatch): - owned = FakeBroker() - monkeypatch.setattr(app_module, "connect", lambda url, *, namespace="": owned) - - await App(redis="redis://owned")._serve() - assert owned.closed - - injected = FakeBroker() - await App(redis="unused://")._serve(injected) - assert not injected.closed diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..f012396 --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,200 @@ +import pytest + +from runtime import Service +from runtime.app import _Inclusion, check_budgets, check_registrations +from runtime.broker import START_HEAD, START_TAIL, WORKERS_GROUP + + +async def noop(ctx, params): ... + + +async def tick(ctx): ... + + +# ── registration ───────────────────────────────────────────────────────────── + + +def test_action_routes_to_the_action_stream_and_the_single_group(): + """One group is what makes an action have exactly one executant, however many + replicas are running.""" + svc = Service("catalog", max_slots=1) + + reg = svc.register_action(noop, name="catalog.sync") + + assert reg.stream == "act:catalog.sync" + assert reg.group == WORKERS_GROUP + assert reg.start == START_HEAD + assert reg.is_event is False + + +def test_event_subscribes_under_the_service_name_by_default(): + svc = Service("analytics", max_slots=4) + + reg = svc.register_event(noop, name="order.placed") + + assert reg.stream == "evt:order.placed" + assert reg.group == "analytics" + assert reg.start == START_TAIL, "a new subscriber must not replay history" + assert reg.is_event is True + + +def test_event_group_can_be_overridden(): + svc = Service("billing", max_slots=4) + + assert ( + svc.register_event(noop, name="order.placed", group="shared").group == "shared" + ) + + +def test_decorators_return_the_function_unchanged(): + """So a handler stays testable bare, without the runtime.""" + svc = Service("s", max_slots=1) + + @svc.action("a") + async def handler(ctx, params): + return "value" + + @svc.every("1s") + async def producer(ctx): ... + + assert svc._actions[0].handler is handler + assert svc._every[0].handler is producer + + +def test_producer_id_defaults_to_the_function_name(): + svc = Service("s") + + svc.register_every(tick, interval="10min") + svc.register_once(tick, delay="1s") + + assert svc._every[0].id == "tick" + assert svc._every[0].interval == 600.0 + assert svc._once[0].delay == 1.0 + + +def test_a_service_holds_consumers_and_producers_together(): + """The merge of Pool and Scheduler: one domain, one object, one switch.""" + svc = Service("catalog", max_slots=1) + svc.register_action(noop, name="catalog.sync") + svc.register_every(tick, interval="5m", id="doctor") + + assert len(svc.consumers) == 1 + assert len(svc._every) == 1 + + +def test_duplicate_action_in_one_service_is_rejected(): + svc = Service("s", max_slots=1) + svc.register_action(noop, name="a") + + with pytest.raises(ValueError, match="already registered"): + svc.register_action(noop, name="a") + + +def test_subscribing_twice_to_one_event_in_one_service_explains_the_fix(): + """Two handlers in one group compete instead of both running — which is + almost never what someone writing two handlers meant.""" + svc = Service("s", max_slots=1) + svc.register_event(noop, name="e") + + with pytest.raises(ValueError, match="group="): + svc.register_event(noop, name="e") + + +def test_two_handlers_on_one_event_are_fine_in_different_groups(): + svc = Service("s", max_slots=1) + svc.register_event(noop, name="e") + + svc.register_event(noop, name="e", group="second") + + assert len(svc._events) == 2 + + +def test_max_slots_must_be_positive(): + with pytest.raises(ValueError, match="max_slots"): + Service("s", max_slots=0) + + +def test_per_registration_cap_above_the_budget_warns(): + svc = Service("s", max_slots=4) + + with pytest.warns(UserWarning, match="no effect"): + svc.register_action(noop, name="a", max_slots=99) + + +# ── cross-service validation ───────────────────────────────────────────────── + + +def test_two_services_cannot_claim_the_same_action(): + a, b = Service("a", max_slots=1), Service("b", max_slots=1) + a.register_action(noop, name="shared") + b.register_action(noop, name="shared") + + with pytest.raises(ValueError, match="exactly one executant"): + check_registrations([a, b]) + + +def test_two_services_may_subscribe_to_the_same_event(): + """This is the whole point of fan-out: each gets its own copy.""" + a, b = Service("a", max_slots=1), Service("b", max_slots=1) + a.register_event(noop, name="order.placed") + b.register_event(noop, name="order.placed") + + check_registrations([a, b]) + + +def test_two_services_in_the_same_group_would_compete_and_are_rejected(): + a, b = Service("a", max_slots=1), Service("b", max_slots=1) + a.register_event(noop, name="order.placed") + b.register_event(noop, name="order.placed", group="a") + + with pytest.raises(ValueError, match="compete"): + check_registrations([a, b]) + + +def test_a_name_cannot_be_both_an_action_and_an_event(): + """They live on different streams, so this 'works' — call() and emit() just + quietly reach different handlers, which is worse than failing.""" + a, b = Service("a", max_slots=1), Service("b", max_slots=1) + a.register_action(noop, name="thing") + b.register_event(noop, name="thing") + + with pytest.raises(ValueError, match="both an action"): + check_registrations([a, b]) + + +def test_a_service_cannot_share_a_name_with_a_registration(): + """The master switch addresses a service by name, so this would make one + switch silently mean two things.""" + svc = Service("catalog", max_slots=1) + svc.register_action(noop, name="catalog") + + with pytest.raises(ValueError, match="master switch"): + check_registrations([svc]) + + +# ── budgets ────────────────────────────────────────────────────────────────── + + +def test_a_consuming_service_without_a_budget_is_refused_at_boot(): + """No default: too low silently serialises a workload, too high removes the + memory bound the budget exists to enforce.""" + svc = Service("s") + svc.register_action(noop, name="a") + + with pytest.raises(ValueError, match="no max_slots"): + check_budgets([_Inclusion(svc, None, None)]) + + +def test_a_producer_only_service_needs_no_budget(): + """It consumes nothing, so there is nothing to bound.""" + svc = Service("clock") + svc.register_every(tick, interval="1s") + + check_budgets([_Inclusion(svc, None, None)]) + + +def test_include_can_supply_the_budget(): + svc = Service("s") + svc.register_action(noop, name="a") + + check_budgets([_Inclusion(svc, 4, None)]) diff --git a/tests/test_verbs.py b/tests/test_verbs.py new file mode 100644 index 0000000..70b34f0 --- /dev/null +++ b/tests/test_verbs.py @@ -0,0 +1,709 @@ +"""The three verbs, end to end through a real App on a fake Redis.""" + +import asyncio + +import pytest +from pydantic import BaseModel + +from runtime import App, CallTimeout, Disabled, ParamsInvalid, Service +from runtime.broker import WORKERS_GROUP, action_stream, event_stream, task_key +from runtime.envelope import make_envelope, now_ms +from tests.helpers import serving + + +def an_app(**overrides) -> App: + # Short everything: these tests are about behaviour, not patience, and + # pytest-timeout is 30s. The claim window matters beyond speed — it is the + # granularity at which a pause takes effect, so a 5s default would make the + # switch tests either slow or racy. + settings = { + "redis": "unused://", + "default_ttl": "2s", + "switch_interval": 0.02, + "claim_block": "0.05s", + } + return App(**{**settings, **overrides}) + + +async def run_until(app, broker, done: asyncio.Event, timeout: float = 5.0) -> None: + async with serving(app, broker): + await asyncio.wait_for(done.wait(), timeout) + + +# ── call ───────────────────────────────────────────────────────────────────── + + +async def test_call_returns_the_handlers_value(broker): + math = Service("math", max_slots=2) + + @math.action("math.add") + async def add(ctx, params): + return params["a"] + params["b"] + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + seen["result"] = await ctx.call("math.add", ttl="2s", a=2, b=3) + done.set() + + app = an_app() + app.include(math) + app.include(caller) + await run_until(app, broker, done) + + assert seen["result"] == 5 + + +async def test_a_producer_can_call_an_action_in_its_own_service(broker): + """What the README quickstart shows. Safe because a producer holds no slot + permit — only handlers do — so it cannot starve the budget it is calling into. + The same shape between two HANDLERS in one service is the deadlock documented + in doc/limits.md.""" + pricing, seen, done = Service("pricing", max_slots=2), {}, asyncio.Event() + + @pricing.action("pricing.quote") + async def quote(ctx, params): + return {"total": round(19.99 * params["qty"], 2)} + + @pricing.every("0.02s") + async def tick(ctx): + seen["total"] = await ctx.call("pricing.quote", ttl="2s", qty=3) + done.set() + + app = an_app() + app.include(pricing) + await run_until(app, broker, done) + + assert seen["total"] == {"total": 59.97} + + +async def test_call_reraises_the_handlers_error_in_the_caller(broker): + """The point of typed errors: the caller gets something it can branch on, + not a timeout and not a stringified traceback.""" + svc = Service("svc", max_slots=1) + + @svc.action("boom") + async def boom(ctx, params): + raise ValueError("no good") + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + try: + await ctx.call("boom", ttl="2s") + except Exception as e: # noqa: BLE001 — capturing whatever surfaced + seen["error"] = e + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + assert seen["error"].code == "REMOTE" + assert "ValueError: no good" in seen["error"].message + assert seen["error"].data == {"exception": "ValueError"} + + +async def test_call_to_a_missing_action_times_out(broker): + """No registry, so an unknown name cannot fail fast — it waits out its + budget. Documented cost of implicit discovery.""" + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + try: + await ctx.call("nobody.listening", ttl="0.3s") + except CallTimeout as e: + seen["error"] = e + done.set() + + app = an_app() + app.include(caller) + await run_until(app, broker, done) + + assert isinstance(seen["error"], CallTimeout) + + +async def test_params_model_rejects_a_bad_body_before_the_handler_runs(broker): + class AddParams(BaseModel): + a: int + b: int + + svc, ran = Service("svc", max_slots=1), [] + + @svc.action("math.add", params=AddParams) + async def add(ctx, params): + ran.append(params) + return params.a + params.b + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + try: + await ctx.call("math.add", ttl="2s", a="not a number", b=1) + except ParamsInvalid as e: + seen["error"] = e + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + assert isinstance(seen["error"], ParamsInvalid) + assert ran == [], "the handler must not see a body that failed validation" + + +async def test_a_validated_body_reaches_the_handler_as_the_model(broker): + class AddParams(BaseModel): + a: int + b: int = 10 + + svc = Service("svc", max_slots=1) + + @svc.action("math.add", params=AddParams) + async def add(ctx, params): + assert isinstance(params, AddParams) + assert ctx.params is params + return params.a + params.b + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + seen["result"] = await ctx.call("math.add", ttl="2s", a=5) + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + assert seen["result"] == 15, "the model's default applied" + + +async def test_an_expired_message_is_dropped_without_running_the_handler(broker): + """The executant's verdict, and the reason a backlog of stale work costs + nothing to shed.""" + svc, ran = Service("svc", max_slots=1), [] + + @svc.action("slow") + async def slow(ctx, params): + ran.append(params) + + # Put a message on the stream that was already past its deadline when written. + stream = action_stream("slow") + await broker.ensure_group(stream, WORKERS_GROUP, start="0") + envelope = make_envelope(kind="dispatch", name="slow", ttl_ms=1) + envelope = envelope.model_copy(update={"dl": now_ms() - 5_000}) + await broker.append(stream, envelope, {"x": 1}) + + app = an_app() + app.include(svc) + async with serving(app, broker): + await asyncio.sleep(0.3) + + assert ran == [] + left = await broker.claim( + stream, WORKERS_GROUP, consumer="probe", count=10, block_ms=50 + ) + assert left == [], "it must still be acked, not left pending forever" + + +async def test_a_child_call_inherits_the_parents_correlation(broker): + """The audit spine: one request id for everything descended from one origin, + with depth and provenance recorded.""" + svc, seen = Service("svc", max_slots=4), {} + + @svc.action("outer") + async def outer(ctx, params): + seen["outer"] = (ctx.request_id, ctx.level, ctx.caller) + return await ctx.call("inner", ttl="2s") + + @svc.action("inner") + async def inner(ctx, params): + seen["inner"] = (ctx.request_id, ctx.level, ctx.caller) + return "done" + + caller, done = Service("caller"), asyncio.Event() + + @caller.once() + async def go(ctx): + await ctx.call("outer", ttl="4s") + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + outer_rid, outer_lvl, outer_caller = seen["outer"] + inner_rid, inner_lvl, inner_caller = seen["inner"] + assert inner_rid == outer_rid + assert inner_lvl == outer_lvl + 1 + assert outer_caller is None, "straight from a producer" + assert inner_caller == "outer" + + +# ── dispatch ───────────────────────────────────────────────────────────────── + + +async def test_dispatch_records_the_outcome_without_the_caller_waiting(broker): + svc = Service("svc", max_slots=1) + + @svc.action("work") + async def work(ctx, params): + return {"processed": params["n"]} + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + task_id = seen["task_id"] = await ctx.dispatch("work", n=7) + for _ in range(200): + record = await ctx.task_status(task_id) + if record["state"] != "pending": + seen["record"] = record + break + await asyncio.sleep(0.02) + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + assert seen["record"]["state"] == "done" + assert seen["record"]["result"] == {"processed": 7} + + +async def test_a_failed_dispatch_is_recorded_as_failed(broker): + svc = Service("svc", max_slots=1) + + @svc.action("work") + async def work(ctx, params): + raise KeyError("missing") + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + task_id = await ctx.dispatch("work") + for _ in range(200): + record = await ctx.task_status(task_id) + if record["state"] != "pending": + seen["record"] = record + break + await asyncio.sleep(0.02) + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + assert seen["record"]["state"] == "failed" + assert seen["record"]["error"]["data"] == {"exception": "KeyError"} + + +# ── emit ───────────────────────────────────────────────────────────────────── + + +async def test_emit_reaches_every_subscriber_exactly_once(broker): + analytics = Service("analytics", max_slots=2) + billing = Service("billing", max_slots=2) + seen: dict[str, list] = {"analytics": [], "billing": []} + + @analytics.event("order.placed") + async def index(ctx, params): + seen["analytics"].append(params["id"]) + + @billing.event("order.placed") + async def invoice(ctx, params): + seen["billing"].append(params["id"]) + + producer = Service("producer") + + @producer.once() + async def go(ctx): + await ctx.emit("order.placed", id="o1") + + app = an_app() + for service in (analytics, billing, producer): + app.include(service) + + async with serving(app, broker): + for _ in range(200): + if seen["analytics"] and seen["billing"]: + break + await asyncio.sleep(0.02) + + assert seen == {"analytics": ["o1"], "billing": ["o1"]} + + +async def test_emitting_with_nobody_subscribed_is_not_an_error(broker): + producer, done = Service("producer"), asyncio.Event() + + @producer.once() + async def go(ctx): + await ctx.emit("nobody.cares", n=1) + done.set() + + app = an_app() + app.include(producer) + await run_until(app, broker, done) + + +# ── delay ──────────────────────────────────────────────────────────────────── + + +async def test_a_delayed_message_arrives_only_after_its_delay(broker): + svc, arrived = Service("svc", max_slots=1), asyncio.Event() + + @svc.action("later") + async def later(ctx, params): + arrived.set() + + producer = Service("producer") + + @producer.once() + async def go(ctx): + await ctx.dispatch("later", delay="0.4s") + + app = an_app() + app.include(svc) + app.include(producer) + + async with serving(app, broker): + await asyncio.sleep(0.15) + assert not arrived.is_set(), "delivered before it was due" + await asyncio.wait_for(arrived.wait(), 5.0) + + +async def test_a_delayed_event_also_waits(broker): + """delay is a property of the message, not of dispatch.""" + analytics, arrived = Service("analytics", max_slots=1), asyncio.Event() + + @analytics.event("later.happened") + async def on_it(ctx, params): + arrived.set() + + producer = Service("producer") + + @producer.once() + async def go(ctx): + await ctx.emit("later.happened", delay="0.4s") + + app = an_app() + app.include(analytics) + app.include(producer) + + async with serving(app, broker): + await asyncio.sleep(0.15) + assert not arrived.is_set(), "delivered before it was due" + await asyncio.wait_for(arrived.wait(), 5.0) + + +async def test_a_delayed_call_does_not_spend_its_budget_waiting_to_be_due(broker): + """The budget starts at the due time. If it started at send time this would + always time out, which is why the rule exists.""" + svc = Service("svc", max_slots=1) + + @svc.action("later") + async def later(ctx, params): + return "arrived" + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + seen["result"] = await ctx.call("later", delay="0.3s", ttl="1s") + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + assert seen["result"] == "arrived" + + +async def test_an_event_with_its_own_ttl_expires(broker): + """Events do not inherit a deadline, but they can carry one.""" + svc, ran = Service("svc", max_slots=1), [] + + @svc.event("stale.fact") + async def on_it(ctx, params): + ran.append(1) + + stream = event_stream("stale.fact") + await broker.ensure_group(stream, "svc", start="0") + envelope = make_envelope(kind="event", name="stale.fact", ttl_ms=1) + await broker.append( + stream, envelope.model_copy(update={"dl": now_ms() - 5_000}), {} + ) + + app = an_app() + app.include(svc) + async with serving(app, broker): + await asyncio.sleep(0.3) + + assert ran == [], "an expired event must not run" + + +async def test_meta_reaches_a_grandchild_handler(broker): + """ctx.meta is how a caller threads its own context — a tenant, a trace id — + through a chain it does not control.""" + svc, seen = Service("svc", max_slots=4), {} + + @svc.action("outer") + async def outer(ctx, params): + seen["outer"] = dict(ctx.meta) + return await ctx.call("inner", ttl="2s", meta={"step": 1}) + + @svc.action("inner") + async def inner(ctx, params): + seen["inner"] = dict(ctx.meta) + return "ok" + + caller, done = Service("caller"), asyncio.Event() + + @caller.once() + async def go(ctx): + await ctx.call("outer", ttl="4s", meta={"tenant": "acme"}) + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + assert seen["outer"] == {"tenant": "acme"} + assert seen["inner"] == {"tenant": "acme", "step": 1}, "shallow merge, child wins" + + +async def test_remaining_reports_the_budget_left(broker): + svc, seen, done = Service("svc", max_slots=1), {}, asyncio.Event() + + @svc.action("slow") + async def slow(ctx, params): + seen["at_start"] = ctx.remaining() + await asyncio.sleep(0.2) + seen["later"] = ctx.remaining() + done.set() + + producer = Service("producer") + + @producer.once() + async def go(ctx): + assert ctx.remaining() is None, "a producer handles nothing, so has no budget" + await ctx.dispatch("slow", ttl="5s") + + app = an_app() + app.include(svc) + app.include(producer) + await run_until(app, broker, done) + + assert 4.0 < seen["at_start"] <= 5.0 + assert seen["later"] < seen["at_start"], "the budget must shrink as time passes" + + +async def test_sweeper_false_means_this_process_delivers_no_delayed_messages(broker): + """The opt-out for fleets big enough that a few dedicated sweepers beat every + process polling. Unpinned, it silently either does nothing or drops delayed + delivery entirely.""" + svc, ran = Service("svc", max_slots=1), [] + + @svc.action("later") + async def later(ctx, params): + ran.append(1) + + producer, dispatched = Service("producer"), asyncio.Event() + + @producer.once() + async def go(ctx): + await ctx.dispatch("later", delay="0.1s") + dispatched.set() + + app = an_app(sweeper=False) + app.include(svc) + app.include(producer) + + async with serving(app, broker): + await asyncio.wait_for(dispatched.wait(), 5.0) + await asyncio.sleep(0.5) # well past its due time + assert ran == [], "with no sweeper nothing should promote the message" + still_due = await broker.sweep(now_ms(), limit=10, maxlen=None) + + assert still_due == 1, "it should have been waiting in the due set" + + +# ── switches ───────────────────────────────────────────────────────────────── + + +async def test_a_paused_action_stops_consuming_and_resumes_where_it_left_off(broker): + svc, handled = Service("svc", max_slots=1), [] + + @svc.action("work") + async def work(ctx, params): + handled.append(params["n"]) + + producer, seen = Service("producer"), {} + paused, resumed = asyncio.Event(), asyncio.Event() + + @producer.once() + async def go(ctx): + await ctx.set_enabled("work", False) + # Let any claim already parked in the broker expire first. Pausing stops + # a worker taking NEW claims; it cannot retract one already in flight, + # which Redis will wake the moment a message lands. + await asyncio.sleep(0.2) + seen["task_id"] = await ctx.dispatch("work", n=1) + await asyncio.sleep(0.3) + paused.set() + await resumed.wait() + await ctx.set_enabled("work", True) + + app = an_app() + app.include(svc) + app.include(producer) + + async with serving(app, broker): + await asyncio.wait_for(paused.wait(), 5.0) + assert handled == [], "a paused action must not consume" + record = await broker.get_task(task_key(seen["task_id"])) + assert record["state"] == "pending", "queued — not failed, not lost" + + resumed.set() + for _ in range(300): + if handled: + break + await asyncio.sleep(0.02) + + assert handled == [1], "the backlog must drain on resume" + + +async def test_calling_a_paused_action_fails_fast_instead_of_waiting(broker): + svc = Service("svc", max_slots=1) + + @svc.action("work") + async def work(ctx, params): + return "never reached" + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + await ctx.set_enabled("work", False) + started = asyncio.get_running_loop().time() + try: + await ctx.call("work", ttl="10s") + except Disabled as e: + seen["error"] = e + seen["elapsed"] = asyncio.get_running_loop().time() - started + done.set() + + app = an_app() + app.include(svc) + app.include(caller) + await run_until(app, broker, done) + + assert isinstance(seen["error"], Disabled) + assert seen["elapsed"] < 1.0, "it waited instead of failing fast" + + +async def test_a_paused_producer_skips_ticks_but_keeps_its_cadence(broker): + svc, ticks = Service("svc"), [] + + @svc.every("0.02s", id="beat") + async def beat(ctx): + ticks.append(1) + + app = an_app() + app.include(svc) + + async with serving(app, broker): + await asyncio.sleep(0.15) + assert ticks, "should have been ticking" + await broker.switch_set("beat", False) + await asyncio.sleep(0.1) # let the switchboard notice + paused_at = len(ticks) + await asyncio.sleep(0.15) + assert len(ticks) == paused_at, "a paused producer must not run its body" + + await broker.switch_set("beat", True) + await asyncio.sleep(0.2) + assert len(ticks) > paused_at, "resuming must be immediate" + + +async def test_a_paused_one_shot_waits_instead_of_being_discarded(broker): + """A one-shot has exactly one chance to run, so skipping it while paused + would make a switch discard work — the one thing pausing must never do.""" + svc, ran = Service("svc"), [] + + @svc.once(id="warmup") + async def warmup(ctx): + ran.append(1) + + await broker.switch_set("warmup", False) + app = an_app() + app.include(svc) + + async with serving(app, broker): + await asyncio.sleep(0.2) + assert ran == [], "paused — it must not have run yet" + + await broker.switch_set("warmup", True) + for _ in range(300): + if ran: + break + await asyncio.sleep(0.02) + + assert ran == [1], "it must run once the switch is back on" + + +async def test_max_depth_stops_an_action_that_calls_itself(broker): + svc, depths = Service("svc", max_slots=4), [] + + @svc.action("recurse") + async def recurse(ctx, params): + depths.append(ctx.level) + return await ctx.call("recurse", ttl="5s") + + caller, seen, done = Service("caller"), {}, asyncio.Event() + + @caller.once() + async def go(ctx): + try: + await ctx.call("recurse", ttl="8s") + except Exception as e: # noqa: BLE001 + seen["error"] = e + done.set() + + app = an_app(max_depth=3) + app.include(svc) + app.include(caller) + await run_until(app, broker, done, timeout=10.0) + + assert seen["error"].code == "MAX_DEPTH" + assert max(depths) <= 3 + + +@pytest.mark.parametrize("verb", ["dispatch", "emit"]) +async def test_dispatch_and_emit_still_queue_for_a_paused_consumer(broker, verb): + """Deferring work for something paused is the point; only `call` refuses.""" + producer, done = Service("producer"), asyncio.Event() + + @producer.once() + async def go(ctx): + await ctx.set_enabled("thing", False) + await getattr(ctx, verb)("thing", n=1) + done.set() + + app = an_app() + app.include(producer) + await run_until(app, broker, done) diff --git a/tests/test_worker.py b/tests/test_worker.py index 2ca99cd..35b2739 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -1,134 +1,398 @@ +"""slot_worker: the concurrency bound, and what happens to a message that +misbehaves. These are the guarantees everything else assumes.""" + import asyncio -import uuid -from runtime.broker import stream_name -from runtime.pool import slot_worker -from runtime.redis_io import GROUP, from_fields # raw PEL probe helpers +from pydantic import BaseModel + +from runtime import App, Service +from runtime.broker import WORKERS_GROUP, Delivery, action_stream, task_key +from runtime.context import Limits, RuntimeContext +from runtime.envelope import make_envelope, now_ms +from runtime.redis_io import META_FIELD +from runtime.service import BLOCK_MS, slot_worker +from tests.helpers import serving + + +async def drive(broker, reg, *, semaphore, consumer="c0", limits=None, ctx=None): + """Run one slot worker as a task until the caller cancels it.""" + return asyncio.create_task( + slot_worker( + broker, + reg, + service_name="svc", + consumer=consumer, + ctx=ctx or RuntimeContext(broker), + semaphore=semaphore, + limits=limits or Limits(), + block_ms=20, + ) + ) -async def _drain(*tasks): +async def stop(*tasks): for t in tasks: t.cancel() await asyncio.gather(*tasks, return_exceptions=True) -async def _pending_messages(redis, stream: str, consumer: str): - pending = await redis.xreadgroup( - groupname=GROUP, - consumername=consumer, - streams={stream: "0"}, - ) - if not pending: - return [] - return pending[0][1] +# ── the concurrency bound ──────────────────────────────────────────────────── -async def test_handler_runs_with_resources(broker, redis): - consumes = f"ev-{uuid.uuid4().hex}" - stream = stream_name(consumes) - await broker.ensure_stream(stream) +async def test_the_semaphore_bounds_handlers_in_flight(broker): + """`max_slots` is a memory bound — a RAM budget for browser contexts, a + connection cap for a database. If it does not actually hold, nothing else in + the design does either.""" + budget = 2 + peak, live = 0, 0 + svc = Service("svc", max_slots=budget) - seen: list[dict] = [] - done = asyncio.Event() + @svc.action("work") + async def work(ctx, params): + nonlocal peak, live + live += 1 + peak = max(peak, live) + await asyncio.sleep(0.05) + live -= 1 - async def handler(ctx, event): - assert ctx.resources["db"] == "DB" # lifespan resources reach the handler - seen.append(event) - done.set() + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + for _ in range(12): + await broker.append(reg.stream, make_envelope(kind="dispatch", name="work"), {}) - await broker.append(stream, {"sku": "X"}) - task = asyncio.create_task( - slot_worker( - broker, - handler, - consumes=consumes, - consumer="c0", - resources={"db": "DB"}, - config={}, - pool_sem=asyncio.Semaphore(4), + semaphore = asyncio.Semaphore(budget) + workers = [ + await drive(broker, reg, semaphore=semaphore, consumer=f"c{i}") + for i in range(6) + ] + await asyncio.sleep(0.5) + await stop(*workers) + + assert peak <= budget, f"{peak} handlers ran at once under a budget of {budget}" + assert peak > 1, "the test never actually overlapped; it proves nothing" + + +async def test_one_semaphore_is_shared_across_a_services_registrations(broker): + """Otherwise two registrations would each get the full budget and the + service's real ceiling would be twice what was asked for.""" + seen: list[asyncio.Semaphore] = [] + svc = Service("svc", max_slots=3) + + @svc.action("a") + async def a(ctx, params): ... + + @svc.action("b") + async def b(ctx, params): ... + + import runtime.app as app_module + + original = app_module.slot_worker + + async def spy(bk, reg, **kwargs): + seen.append(kwargs["semaphore"]) + await original(bk, reg, **kwargs) + + app_module.slot_worker = spy + try: + app = App(redis="unused://") + app.include(svc) + async with serving(app, broker): + await asyncio.sleep(0.1) + finally: + app_module.slot_worker = original + + assert len(seen) == 6, "3 slots x 2 registrations" + assert len({id(s) for s in seen}) == 1, "each registration got its own budget" + + +async def test_a_per_registration_cap_limits_its_worker_count(broker): + """The 'one expensive action must not starve the cheap ones' lever.""" + spawned: list[str] = [] + svc = Service("svc", max_slots=5) + + @svc.action("cheap") + async def cheap(ctx, params): ... + + @svc.action("expensive", max_slots=2) + async def expensive(ctx, params): ... + + import runtime.app as app_module + + original = app_module.slot_worker + + async def spy(bk, reg, **kwargs): + spawned.append(reg.name) + await asyncio.sleep(3600) + + app_module.slot_worker = spy + try: + app = App(redis="unused://") + app.include(svc) + async with serving(app, broker): + await asyncio.sleep(0.1) + finally: + app_module.slot_worker = original + + assert spawned.count("cheap") == 5 + assert spawned.count("expensive") == 2 + + +# ── every message is acked, whatever happened to it ────────────────────────── + + +async def pending_count(broker, stream): + return len( + await broker.claim( + stream, WORKERS_GROUP, consumer="probe", count=100, block_ms=20 ) ) - await asyncio.wait_for(done.wait(), 2.0) - await _drain(task) - assert seen == [{"sku": "X"}] - assert await _pending_messages(redis, stream, "c0") == [] - await redis.delete(stream) +async def test_a_successful_message_is_acked(broker): + svc = Service("svc", max_slots=1) -async def test_handler_exception_leaves_unacked(broker, redis): - consumes = f"ev-{uuid.uuid4().hex}" - stream = stream_name(consumes) - await broker.ensure_stream(stream) + @svc.action("ok") + async def ok(ctx, params): ... - attempted = asyncio.Event() + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + await broker.append(reg.stream, make_envelope(kind="dispatch", name="ok"), {}) - async def handler(ctx, event): - attempted.set() - raise RuntimeError("boom") + worker = await drive(broker, reg, semaphore=asyncio.Semaphore(1)) + await asyncio.sleep(0.2) + await stop(worker) - await broker.append(stream, {"x": 1}) - task = asyncio.create_task( - slot_worker( - broker, - handler, - consumes=consumes, - consumer="c0", - resources={}, - config={}, - pool_sem=asyncio.Semaphore(1), + assert await pending_count(broker, reg.stream) == 0 + + +async def test_a_raising_handler_is_still_acked(broker): + """v0.3 inverted v0.2's rule. With no reclaim, leaving it un-acked would not + mean 'retry later' — it would mean lost anyway, plus a pending entry that + nothing will ever claim.""" + svc = Service("svc", max_slots=1) + + @svc.action("boom") + async def boom(ctx, params): + raise ValueError("no") + + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + await broker.append(reg.stream, make_envelope(kind="dispatch", name="boom"), {}) + + worker = await drive(broker, reg, semaphore=asyncio.Semaphore(1)) + await asyncio.sleep(0.2) + await stop(worker) + + assert await pending_count(broker, reg.stream) == 0 + + +async def test_an_undecodable_message_is_acked_not_left_as_a_poison_pill( + broker, redis, namespace +): + svc = Service("svc", max_slots=1) + + @svc.action("x") + async def x(ctx, params): ... + + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + await redis.xadd(f"{namespace}:{reg.stream}", {META_FIELD: "not an envelope"}) + + worker = await drive(broker, reg, semaphore=asyncio.Semaphore(1)) + await asyncio.sleep(0.2) + await stop(worker) + + assert await pending_count(broker, reg.stream) == 0 + + +async def test_an_invalid_body_is_acked_and_never_reaches_the_handler(broker): + class Body(BaseModel): + n: int + + ran = [] + svc = Service("svc", max_slots=1) + + @svc.action("x", params=Body) + async def x(ctx, params): + ran.append(params) + + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + await broker.append( + reg.stream, make_envelope(kind="dispatch", name="x"), {"n": "not a number"} + ) + + worker = await drive(broker, reg, semaphore=asyncio.Semaphore(1)) + await asyncio.sleep(0.2) + await stop(worker) + + assert ran == [] + assert await pending_count(broker, reg.stream) == 0 + + +# ── outcomes ───────────────────────────────────────────────────────────────── + + +async def test_a_handler_may_return_a_pydantic_model(broker): + """The docstring says handlers routinely do this; nothing proved it worked.""" + + class Result(BaseModel): + total: float + + svc = Service("svc", max_slots=1) + + @svc.action("quote") + async def quote(ctx, params): + return Result(total=1.5) + + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + task_id = "t1" + await broker.append( + reg.stream, + make_envelope(kind="dispatch", name="quote", task=task_id), + {}, + ) + + worker = await drive(broker, reg, semaphore=asyncio.Semaphore(1)) + await asyncio.sleep(0.2) + await stop(worker) + + record = await broker.get_task(task_key(task_id)) + assert record["state"] == "done" + assert record["result"] == {"total": 1.5} + + +async def test_a_return_value_that_cannot_cross_the_wire_becomes_an_error(broker): + """Rather than an exception thrown while reporting, which would take the + worker — and with it the whole process — down.""" + svc = Service("svc", max_slots=1) + + @svc.action("bad") + async def bad(ctx, params): + return {"not_serialisable": object()} + + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + await broker.append( + reg.stream, make_envelope(kind="dispatch", name="bad", task="t2"), {} + ) + + worker = await drive(broker, reg, semaphore=asyncio.Semaphore(1)) + await asyncio.sleep(0.2) + alive = not worker.done() + await stop(worker) + + assert alive, "the worker died instead of reporting the failure" + record = await broker.get_task(task_key("t2")) + assert record["state"] == "failed" + + +async def test_a_worker_survives_a_broker_error_while_reporting(broker): + """One bad message must not end the worker: its task dying propagates out of + App's gather and stops the process.""" + failed_once = [] + + class FlakyReport: + def __init__(self, inner): + self._inner = inner + + def __getattr__(self, name): + return getattr(self._inner, name) + + async def put_task(self, *a, **kw): + if not failed_once: + failed_once.append(True) + raise RuntimeError("redis blip") + return await self._inner.put_task(*a, **kw) + + handled = [] + svc = Service("svc", max_slots=1) + + @svc.action("x") + async def x(ctx, params): + handled.append(params["n"]) + + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + for n in (1, 2): + await broker.append( + reg.stream, make_envelope(kind="dispatch", name="x", task=f"t{n}"), {"n": n} ) + + worker = await drive(FlakyReport(broker), reg, semaphore=asyncio.Semaphore(1)) + await asyncio.sleep(0.3) + alive = not worker.done() + await stop(worker) + + assert alive, "the worker died on a transient broker error" + assert handled == [1, 2], "it stopped consuming after the failure" + + +async def test_an_expired_dispatch_is_recorded_as_expired_not_failed(broker): + svc = Service("svc", max_slots=1) + + @svc.action("slow") + async def slow(ctx, params): ... + + reg = svc._actions[0] + await broker.ensure_group(reg.stream, reg.group, start="0") + envelope = make_envelope(kind="dispatch", name="slow", task="t3", ttl_ms=1) + await broker.append( + reg.stream, envelope.model_copy(update={"dl": now_ms() - 5_000}), {} ) - await asyncio.wait_for(attempted.wait(), 2.0) - await asyncio.sleep(0.05) # give it time to (not) ack and loop - await _drain(task) - - # message is still pending for c0 (unacked → stays in the PEL); raw probe - pending = await _pending_messages(redis, stream, "c0") - assert len(pending) == 1 - _msg_id, fields = pending[0] - assert from_fields(fields) == {"x": 1} - await redis.delete(stream) - - -async def test_pool_semaphore_serializes(broker, redis): - consumes = f"ev-{uuid.uuid4().hex}" - stream = stream_name(consumes) - await broker.ensure_stream(stream) - - n = 3 - state = {"cur": 0, "max": 0, "done": 0} - done = asyncio.Event() - - async def handler(ctx, event): - state["cur"] += 1 - state["max"] = max(state["max"], state["cur"]) - await asyncio.sleep(0.02) - state["cur"] -= 1 - state["done"] += 1 - if state["done"] == n: - done.set() - - for i in range(n): - await broker.append(stream, {"i": i}) - - sem = asyncio.Semaphore(1) # budget of 1 → strictly serial across all workers - tasks = [ - asyncio.create_task( - slot_worker( - broker, - handler, - consumes=consumes, - consumer=f"c{i}", - resources={}, - config={}, - pool_sem=sem, - ) + + worker = await drive(broker, reg, semaphore=asyncio.Semaphore(1)) + await asyncio.sleep(0.2) + await stop(worker) + + assert (await broker.get_task(task_key("t3")))["state"] == "expired" + + +# ── retention ──────────────────────────────────────────────────────────────── + + +async def test_streams_are_trimmed(broker, redis, namespace): + """XACK never deletes and durable fan-out keeps entries past every reader, so + without a cap every stream grows forever. + + Asserted as a bound, not an exact length: `MAXLEN ~` trims at radix-tree node + boundaries, so real Redis overshoots (200 appends at maxlen=5 leaves ~12) + while fakeredis trims exactly. Pinning the exact number would pass on one + backend and fail on the other. + """ + stream = action_stream("trimmed") + for _ in range(200): + await broker.append( + stream, make_envelope(kind="dispatch", name="trimmed"), {}, maxlen=5 ) - for i in range(n) - ] - await asyncio.wait_for(done.wait(), 3.0) - await _drain(*tasks) - assert state["max"] == 1 # never two handlers in flight under a budget of 1 - await redis.delete(stream) + length = await redis.xlen(f"{namespace}:{stream}") + assert length < 50, f"stream grew to {length}; trimming is not happening" + + +async def test_without_a_maxlen_a_stream_grows_unbounded(broker, redis, namespace): + """The other half — proving the previous test measures the cap and not some + incidental Redis behaviour.""" + stream = action_stream("untrimmed") + for _ in range(200): + await broker.append( + stream, make_envelope(kind="dispatch", name="untrimmed"), {}, maxlen=None + ) + + assert await redis.xlen(f"{namespace}:{stream}") == 200 + + +async def test_delivery_is_what_the_worker_sees(broker): + """Guards the claim contract the worker is written against.""" + stream = action_stream("shape") + await broker.ensure_group(stream, WORKERS_GROUP, start="0") + await broker.append(stream, make_envelope(kind="dispatch", name="shape"), {"a": 1}) + + [got] = await broker.claim( + stream, WORKERS_GROUP, consumer="c", count=1, block_ms=BLOCK_MS + ) + + assert isinstance(got, Delivery) + assert got.params == {"a": 1} + assert got.envelope.name == "shape" diff --git a/uv.lock b/uv.lock index 7d74123..9ec4642 100644 --- a/uv.lock +++ b/uv.lock @@ -6,10 +6,20 @@ resolution-markers = [ "python_full_version < '3.15'", ] +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + [[package]] name = "archipellabs-runtime" source = { editable = "." } dependencies = [ + { name = "pydantic" }, { name = "redis" }, ] @@ -19,6 +29,7 @@ dev = [ { name = "mypy" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-cov" }, { name = "pytest-timeout" }, { name = "ruff" }, ] @@ -31,8 +42,10 @@ requires-dist = [ { name = "fakeredis", marker = "extra == 'dev'", specifier = ">=2.21" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8" }, { name = "playwright", marker = "extra == 'playwright'", specifier = ">=1.40" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5" }, { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.2" }, { name = "redis", specifier = ">=5" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6" }, @@ -88,6 +101,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coverage" +version = "7.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d0/55fe630f4cf94e3fcba868240fad8c8cdd1f764e2a932f8926347e6ec4cd/coverage-7.15.2.tar.gz", hash = "sha256:3df60dc267f0a2ca23cb7a9ab1109c62b9335ffbf519fcfe167157c28c09b81d", size = 927741, upload-time = "2026-07-15T18:56:19.558Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/50/eb5bf42e531611a9f8d272556b1ed4de503f84a91413584094487cf69f8f/coverage-7.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1adac78e5abc7c5438f7a209c9ca69d06542f0bf481d728b6989ea80b813fdf9", size = 221587, upload-time = "2026-07-15T18:54:18.439Z" }, + { url = "https://files.pythonhosted.org/packages/06/d1/da99af464c335d4e023a6efcd7ec30f63b88a43c93745154ab74ffb31cea/coverage-7.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b868acc62aa5de3be7a9d05c2333bf8359ca987e43f9cb30ff8fbda6a024ab73", size = 221943, upload-time = "2026-07-15T18:54:20.062Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8a/13c42723d61ca447eafa18732e8141dd6a63f2732e1c7e1502c182dd88d7/coverage-7.15.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6f6966fc30e6f06ca8f98fb0ce51eda6b111b3ee8d066a8b1ec9e77fa06ab55d", size = 253450, upload-time = "2026-07-15T18:54:21.765Z" }, + { url = "https://files.pythonhosted.org/packages/d7/29/99021303f98fbdcb63504b4d07bea4cc025b9b2dd907c4f07c85d50a0dab/coverage-7.15.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:68af907f595ab01a78f794932ff3bdf929c316d3000810d38dbc247129e26f8b", size = 256187, upload-time = "2026-07-15T18:54:23.4Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a8/fd503715ed6ca9c5d742923aa5209257340b367a867b2ced0c7d4ba8a0b9/coverage-7.15.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afa29e2eff3d5729267e2cb2fd4ce9d61c952932fb2694e34ccb5d9540c6a296", size = 257301, upload-time = "2026-07-15T18:54:25.183Z" }, + { url = "https://files.pythonhosted.org/packages/da/40/3f4b8fb409810036ebc2857d36adc0498c6e957b5df0290c5036b2e143f1/coverage-7.15.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbf44513ceb1589e31948e20eafbde9deaface90e1a1afa5f5f77b4423d17ce6", size = 259562, upload-time = "2026-07-15T18:54:27.204Z" }, + { url = "https://files.pythonhosted.org/packages/0b/8a/9bdffbef47db77cce3d6b02a28f7e919b19f0106c4b080c2c2246040f885/coverage-7.15.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9deddf09eecb717b7f980414b43d90a5b22ff3967d2949ab29cb0aa83d9e9098", size = 253841, upload-time = "2026-07-15T18:54:29.134Z" }, + { url = "https://files.pythonhosted.org/packages/1b/1e/9031efde019d31a06646261fce6dfc5c3c74e951e27a71e5c9a424563178/coverage-7.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae901f7e55ba405c84ee1cab3d3e962e4e871e4a2bcb9c90911adbd69b42ac5a", size = 255221, upload-time = "2026-07-15T18:54:31.142Z" }, + { url = "https://files.pythonhosted.org/packages/56/db/787acde872389fc84a9ef9d8cd1ccc658e391ab4cb5b28092a714426a394/coverage-7.15.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a0f47002c6eeb7c280228467a4cb0cc15ca2103a8421b986b2d3ec04a0f9bd8b", size = 253366, upload-time = "2026-07-15T18:54:32.886Z" }, + { url = "https://files.pythonhosted.org/packages/2f/9b/6f57bc4b93c842eef1695f8cdaf2318e35e7ba54f5ba80d84be213ab7858/coverage-7.15.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1cd7a5beb7af3e864a13b1f0fb26efd3695da43ef0daf71e586adfffaf34d5b2", size = 257434, upload-time = "2026-07-15T18:54:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/88/26/b3186a21b2acc83e451118978905c81c7072c3333707804db09a78c096a2/coverage-7.15.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:97a5c5457a9fb1d6c4e06cfb5dc835871fbfb6a6a51addc9e925bdeff5ef7440", size = 252935, upload-time = "2026-07-15T18:54:36.548Z" }, + { url = "https://files.pythonhosted.org/packages/20/c2/c9f3376b2e717ea69ed7a6e9a5fcab968fb0b290db6cf4bd9a1fc7541b75/coverage-7.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0901cfe6c13bcd2302da4f83e884555d2a22bda6e4c476f09ef204ba20ca536e", size = 254807, upload-time = "2026-07-15T18:54:38.296Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e1/dfc15401f4a8aaeb486e1ba3e9e3c40522a6e38bd0ecf0b3f29cb8082957/coverage-7.15.2-cp312-cp312-win32.whl", hash = "sha256:b171bdd71cb7ff792bf32e376173b0ace7e7963e7e57c58dfc42063a6a7174cd", size = 223641, upload-time = "2026-07-15T18:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/91/40/81b6d809d320cd366ec5bdf8176575e897dcb8efe7fb4b489ef9e93e4d13/coverage-7.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:582edc45c2040543fef83341be23c43024a3ab3ae0c2d8bc498a06282905ad40", size = 224172, upload-time = "2026-07-15T18:54:41.882Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/9f14ec438149f7de557f45518f09b4a7917b795cc37083aa7db482693f8c/coverage-7.15.2-cp312-cp312-win_arm64.whl", hash = "sha256:a638db90c61cd219aeee65e83a24fdaa57269a741ae0cf773309208ac862cee3", size = 223556, upload-time = "2026-07-15T18:54:43.674Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d5/f8c838e6b7282976f7c918884b792df7a0c42c5bba5d99c60ad2d221d56d/coverage-7.15.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1121caa19159a38b5463eaae4b1e1fde81e525b15ecc5e000cd5b1a108f743a8", size = 221606, upload-time = "2026-07-15T18:54:45.448Z" }, + { url = "https://files.pythonhosted.org/packages/bf/37/97c926376364f66298cc44893b89cdf17b8bc406376497c4061ae4b8a8ff/coverage-7.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a300c6934e0989c327b9e8a1e110329da4641149f872bbe9f70168be66da76c1", size = 221982, upload-time = "2026-07-15T18:54:47.341Z" }, + { url = "https://files.pythonhosted.org/packages/b7/30/a36050a6e83c2135ee0776f452ca3948224befc6d7f26acecc082d0c106a/coverage-7.15.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2617f8799d268fabdeef42a7e89ac3a23e1deee9025427db2df970f99a89a578", size = 252972, upload-time = "2026-07-15T18:54:49.2Z" }, + { url = "https://files.pythonhosted.org/packages/31/d3/06b5f1daf95f0f15ab05bd75f26ba5f3c8b33d0bb72f3aaa3cf41d1bad3a/coverage-7.15.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7dc2950a2992cd676d35c20ae63522836deeb034f08874699d14068710af3dc1", size = 255569, upload-time = "2026-07-15T18:54:51.098Z" }, + { url = "https://files.pythonhosted.org/packages/81/1c/9afb3f8de2b8d36960391c48559a2e3ff96594b58099f115921549ea8d0d/coverage-7.15.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e36686f7a442185db2400b3df171aac520869faf9deb59df687d28659eda2a6", size = 256806, upload-time = "2026-07-15T18:54:53.145Z" }, + { url = "https://files.pythonhosted.org/packages/64/d8/b989f96061a5e32d82fddd1b1b9ff48a7c8f8ae7606f0e80fd9de54b1e33/coverage-7.15.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d29ca7bd67af6e12e74632d65f026eabc1364da5c254494cd914446a28a3ef7", size = 258936, upload-time = "2026-07-15T18:54:55.015Z" }, + { url = "https://files.pythonhosted.org/packages/b8/fa/f99771f5110457c7b511c1935ca49ddf288218eaa84322e028b9334146ae/coverage-7.15.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:db9c8438057e5b0f6a22a0af99c0c1d26b57fbbdbd1be5861ddb8f897fcc3a2d", size = 253178, upload-time = "2026-07-15T18:54:57.527Z" }, + { url = "https://files.pythonhosted.org/packages/f6/96/c098a6044d119c751ceede7be91035fa8310170ec24a6523aff72f0a5793/coverage-7.15.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:63022c4c8dec1d0342f05c3ede99842fe3d007689acc45e86f123a1746e4a026", size = 254934, upload-time = "2026-07-15T18:54:59.41Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a2/1457b3a7a50c8d77500103b97a046db863e2f59a1cf6d2f814595f349885/coverage-7.15.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6c0be82b4d4aa5b2704e08518e2252f3e3d110164bcca826816801052e48a7aa", size = 252898, upload-time = "2026-07-15T18:55:01.338Z" }, + { url = "https://files.pythonhosted.org/packages/6c/0e/76958874c471ecfcdde0d2b2747bb2c61bdbf34a40636f4ce9db9923e643/coverage-7.15.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4510fb9cdf6bb02dfa6af0be4a534b8102d086e22e4a33f8836df663da3d660d", size = 257056, upload-time = "2026-07-15T18:55:03.243Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7c/3d7c4e3bf58baa40327dc7edc2272b17cf02299366d52763db1b0ca1556a/coverage-7.15.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:42ec3d989421b174a2ab607c1539f24127ad362757b7f1c0c0d7a2993f7eb37b", size = 252718, upload-time = "2026-07-15T18:55:05.029Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b8/1cecffed9ce14fb25be9ba42d37b6bb61485c9a3ddd43cd3dde36b6087d8/coverage-7.15.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8f91bce78e32343af184c3b7fa28fcf5a9e2641f4b6623d392038f804939188", size = 254490, upload-time = "2026-07-15T18:55:06.889Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2c/42984561bc7f4c045dca67516a0c50ee5ef8d84352dbeb5559dc86c4823e/coverage-7.15.2-cp313-cp313-win32.whl", hash = "sha256:434e68d531858205895eb0d74b73d20b84260de426387d53c422a5acda2cf050", size = 223647, upload-time = "2026-07-15T18:55:08.941Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/39c7c9245efc583beddf89a87683574e663ed93637f3afb6cd7b88405676/coverage-7.15.2-cp313-cp313-win_amd64.whl", hash = "sha256:26c3b04a6377fd7c09800921fa934e3a17c0020439cd59df73e73ae1d4b6a78c", size = 224190, upload-time = "2026-07-15T18:55:10.789Z" }, + { url = "https://files.pythonhosted.org/packages/c7/de/3a2883cf8a213659280ef4b403059e17a9acaeb7fc7fd4105e1226ff2e6d/coverage-7.15.2-cp313-cp313-win_arm64.whl", hash = "sha256:3ed010aa1b69cda8e827aabfca9866216c980e2dca82ab9a78c5f83689964c8b", size = 223583, upload-time = "2026-07-15T18:55:12.678Z" }, + { url = "https://files.pythonhosted.org/packages/81/5f/aed265fd7a3551a394f36dfe41868aee709b7f95db4052205b4ad1563ac3/coverage-7.15.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:40f633c5c5fc783732f6312280122e859538fa24461235597c13d803ea9a108a", size = 221650, upload-time = "2026-07-15T18:55:14.527Z" }, + { url = "https://files.pythonhosted.org/packages/6b/2c/222ba12a545189017120f8eddfc1a0bd4616b47d5d4a8d99421edb2fe4c6/coverage-7.15.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:075560438765b7a2ef43bf7aa7758661b53d889df47f062a31bda6c1ade553a2", size = 221988, upload-time = "2026-07-15T18:55:16.674Z" }, + { url = "https://files.pythonhosted.org/packages/aa/38/304b5877ab46e6c290b4292cfcf3fe28245f0e5597cad7f6acc91fc7e0a4/coverage-7.15.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:25fd15dd40a0a2c51a500d664ca29053c09c3259d998407bf982b6e114696138", size = 253029, upload-time = "2026-07-15T18:55:18.856Z" }, + { url = "https://files.pythonhosted.org/packages/6c/58/821b533b8db9e44cf1d8a97bd525149ced40dde1d0093da02cb78e715244/coverage-7.15.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b9a6367e4aff723e8ee8190836836124284e8fcd4265e307c844010cfa074f3f", size = 255536, upload-time = "2026-07-15T18:55:21.027Z" }, + { url = "https://files.pythonhosted.org/packages/f1/f2/7aa06604c389d32ea7f0a6a988359a7eafc3cd3f8e7bc2e88cd2fdf0b877/coverage-7.15.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9854ca62c152874b2060772503535be2e8f53f70b8aaa7686b094888d872f984", size = 256881, upload-time = "2026-07-15T18:55:23.125Z" }, + { url = "https://files.pythonhosted.org/packages/a2/4f/1ef342339c7916d0096bc5888cc0f653882cc7bc8f897d5cb89143287c9b/coverage-7.15.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:913b6c56e110da40e035bbd168353bf7aaa2544a5eaccea5d98a4629aac156c7", size = 259196, upload-time = "2026-07-15T18:55:25.099Z" }, + { url = "https://files.pythonhosted.org/packages/fe/f4/7ed055d7a9c5ec13b161773a115a5ccc6b0081d568c31fad830806306cc7/coverage-7.15.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aaccad4129d735a8a4d526f26929894c9a4e8ef7034566f210b176749d6906e3", size = 253036, upload-time = "2026-07-15T18:55:27.018Z" }, + { url = "https://files.pythonhosted.org/packages/14/79/ea82cca18c242a3a38b6c017da39726aa62dcb64aa635abf79b92009975c/coverage-7.15.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a164b50081fc7357331c4024ef4d17b78ba325f8380d05f5a69599a7e05257ee", size = 254887, upload-time = "2026-07-15T18:55:29.084Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ba/a136db3c0d9562b00e10b72540dbf3a33cd3bc5b95060c9308e247494623/coverage-7.15.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:bfd341ccf78128e72c094bc70cc25b3ef309c33c7c2c66ba3ed4309549e02de1", size = 252852, upload-time = "2026-07-15T18:55:31.184Z" }, + { url = "https://files.pythonhosted.org/packages/17/17/ea334246b16b7d059953fad6fdefa11e33c68efbd3fe37b1098120a1fac2/coverage-7.15.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1473b3ba8e7ee0f076117b1a72c23f579a2b9e2bb742f48a8d86ea27ca93f91a", size = 257128, upload-time = "2026-07-15T18:55:33.163Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c3/074fb66d46d607855f710876b117cbda562c5ab08363528e78820449f937/coverage-7.15.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:17c432b5f73ad52ef46fb06019f6fa7c66ce381961cf0f7dfd1d3a4bd3a98145", size = 252668, upload-time = "2026-07-15T18:55:35.063Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c1/f620850ada9b36435921c9a3a8057013422b1d964eb4bf37fe138724d192/coverage-7.15.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:77f0ef5011df53a4bd1b35211ab122287f8d9b8d7aa1c4553e5c2deb24b1d446", size = 254325, upload-time = "2026-07-15T18:55:37.125Z" }, + { url = "https://files.pythonhosted.org/packages/cc/31/a729ca3689404493af82ef8e6ff70bd88bdda8da89aeef6ca9b387aeb2b4/coverage-7.15.2-cp314-cp314-win32.whl", hash = "sha256:f653e5d7248c1191ec988a85c72edeab46c3ff44f90639a4ed4874ec0be90243", size = 223844, upload-time = "2026-07-15T18:55:39.078Z" }, + { url = "https://files.pythonhosted.org/packages/c6/83/5d809dc808fb1698c671f3e372259bb9158e64b7ea526fc6ab7de64de9fe/coverage-7.15.2-cp314-cp314-win_amd64.whl", hash = "sha256:9911f31aad8906abe337c271343485cf20df5e70df5d2f57f9f136e7b55f26bc", size = 224331, upload-time = "2026-07-15T18:55:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/16/4e/35e488548e952795829e129995c4174df33bf432b591d1aa42c8d9e4e7ad/coverage-7.15.2-cp314-cp314-win_arm64.whl", hash = "sha256:e38def96ad59853824c97953fdcd2c320a84ba3ce99b417db78af8bb6c3db635", size = 223760, upload-time = "2026-07-15T18:55:43.518Z" }, + { url = "https://files.pythonhosted.org/packages/ed/49/dd2c86cd6374038f6e415fb5bfb86db5218553209c081384a020369dee79/coverage-7.15.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:835ec4e20b45f0a7f63ed78f94065aca00de033403df8377bfe8b9c6abc0a7be", size = 222384, upload-time = "2026-07-15T18:55:45.569Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/173ff17a1c0808e5a438f549f6f145d5ac7528f2791310b63523e3200ac7/coverage-7.15.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7466cc7ab6dc0db871d264bf99e8779f0917ee63d40730af0552f71535a6e072", size = 222647, upload-time = "2026-07-15T18:55:47.544Z" }, + { url = "https://files.pythonhosted.org/packages/84/f8/b8cba872162356fb44ac79c10309d987206a4461e32072fc29228dad7331/coverage-7.15.2-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e370c12133095ff18432de8c044962be85a5a96d90c6fcbce8e17e76236d2328", size = 264013, upload-time = "2026-07-15T18:55:49.768Z" }, + { url = "https://files.pythonhosted.org/packages/ee/67/a807a7586d0b8cae485308ddd55756f0806c92f8e0b411bacbf23c48edf3/coverage-7.15.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fe41909c9515c3bfdb5f02c4d1f857dba322d9a9a1178069b91eea77889df63a", size = 266135, upload-time = "2026-07-15T18:55:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/cd78771dc985f7e4ebdcc82b1a96d9a932af9e806f01f2f91a89f4c72e80/coverage-7.15.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6aa28cfb6488e5453b5b762d65f73aa586380f6693a04d58078ce228a29b06c0", size = 268555, upload-time = "2026-07-15T18:55:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/18/3e/10134cf81275188c58568f324fc74aedff32c63ca4d5bbc513a91944a6f0/coverage-7.15.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcc0aae933921d03096f53b0b03eeb702129fd406dee59f08d2efacc68681fa5", size = 269674, upload-time = "2026-07-15T18:55:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/75/4a/771b77de446cba985dc414bbc5844bd21604da05dbc044286df8318a48a7/coverage-7.15.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7c63387e21ab21f512c69c9756a8c7dadd322c7275edb064064433c9a09c3743", size = 263101, upload-time = "2026-07-15T18:55:58.107Z" }, + { url = "https://files.pythonhosted.org/packages/5f/b5/70a7011da15f4071943361183aefa27847f3e3aec4fd335f1cb3d3a622b1/coverage-7.15.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e55510bc98ae943cece9e667a6c0fe94c6a92913720dea34243657a17993d0c", size = 266007, upload-time = "2026-07-15T18:56:00.468Z" }, + { url = "https://files.pythonhosted.org/packages/b4/0d/f9547e804ce7ad49646ffeffac26699510efbe6c0f751b66fdc960c4e825/coverage-7.15.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:2ff08701be2d1556fc78b326c80a3e8042da09352ecb3819105f8e386c8a3071", size = 263611, upload-time = "2026-07-15T18:56:02.615Z" }, + { url = "https://files.pythonhosted.org/packages/ac/59/f576a396659c0efd351f5c1544f67c3560e89c7761cabf7f65e412beeda5/coverage-7.15.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:38c9518b7103826c403a461544e3c2e77151e8676d06eaed85911a97e962584a", size = 267344, upload-time = "2026-07-15T18:56:04.622Z" }, + { url = "https://files.pythonhosted.org/packages/7c/5d/c2e4fce3579c0cb635024293f1a32bbe26df101b3e3a69f22243d1352b6c/coverage-7.15.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:dee88b1ed88587abd8c0269a1fc1f4cc77f7750d1dfde2869e2a123af420e67d", size = 262456, upload-time = "2026-07-15T18:56:06.641Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/956287d69436b66094bc4b57ac2da71e43bfd2a5524e958900b9f582fcf8/coverage-7.15.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fbeeeecea279727f8ac16c8e1133ddfeee793e985c86ae343d6a5ce744eef8c", size = 264771, upload-time = "2026-07-15T18:56:08.795Z" }, + { url = "https://files.pythonhosted.org/packages/2c/5a/6f979530c2734c575de77cf58f5f28d51f7123a94b5030fd9156fe5f363c/coverage-7.15.2-cp314-cp314t-win32.whl", hash = "sha256:cb0fddaa6884be6aae36ced9544b5e90f7d5f03845a2853bf47a14953a4e8688", size = 224151, upload-time = "2026-07-15T18:56:10.856Z" }, + { url = "https://files.pythonhosted.org/packages/54/7e/27f6b2a74d484742f4017553e710b01e396b23d809df3e95ca0bb9a2824b/coverage-7.15.2-cp314-cp314t-win_amd64.whl", hash = "sha256:77f091ea3a9cc611cd29f433565476bc1936c084ac8eee00ea0e7e70c27e4199", size = 224981, upload-time = "2026-07-15T18:56:12.928Z" }, + { url = "https://files.pythonhosted.org/packages/b1/48/284863423aa474240f6842bd00d680da22f4e6ea2e466618ef7c9c9e69a9/coverage-7.15.2-cp314-cp314t-win_arm64.whl", hash = "sha256:6fc448c377d6eeb00a47c673494bd9bae29280ca53987e1869e67ebedfe20658", size = 224294, upload-time = "2026-07-15T18:56:15.156Z" }, + { url = "https://files.pythonhosted.org/packages/ec/82/32e3bd191d498e64f6f911ad55d14006a0861e54869d2d32452326399e65/coverage-7.15.2-py3-none-any.whl", hash = "sha256:eb6bcae8d1a9d305351ecb108232441d11c5cfe9de840a04388ba5d2db8d735c", size = 213375, upload-time = "2026-07-15T18:56:17.305Z" }, +] + [[package]] name = "fakeredis" version = "2.36.1" @@ -336,6 +418,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, +] + [[package]] name = "pyee" version = "13.0.1" @@ -386,6 +558,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "pytest-timeout" version = "2.4.0" @@ -449,3 +635,15 @@ sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac8 wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +]