diff --git a/.agents/skills/obol-stack-dev/references/llm-routing.md b/.agents/skills/obol-stack-dev/references/llm-routing.md index 60b5d530..d2abac58 100644 --- a/.agents/skills/obol-stack-dev/references/llm-routing.md +++ b/.agents/skills/obol-stack-dev/references/llm-routing.md @@ -24,10 +24,10 @@ Only routes published through Traefik are reachable at `http://obol.stack:8080/` |---|---| | Traefik ingress (frontend, eRPC, x402 routes) | `http://obol.stack:8080/...` | | LiteLLM | `obol kubectl port-forward svc/litellm 14000:4000 -n llm` then `http://127.0.0.1:14000` | -| x402-buyer sidecar (no Service — pod only) | `obol kubectl port-forward -n llm 18402:8402` then `http://127.0.0.1:18402` | +| x402-buyer (own Deployment + Service) | `obol kubectl port-forward -n llm svc/x402-buyer 18402:8402` then `http://127.0.0.1:18402` | | OpenClaw instance | `obol kubectl port-forward -n openclaw- svc/openclaw 18789:18789` | -`http://obol.stack:8080/v1/...` does **not** hit LiteLLM — Traefik has no `/v1` route and returns the frontend 404. The `x402-buyer` sidecar is **distroless** — no `wget`/`curl`/shell. Always port-forward, never `kubectl exec`. +`http://obol.stack:8080/v1/...` does **not** hit LiteLLM — Traefik has no `/v1` route and returns the frontend 404. The `x402-buyer` binary is **distroless** — no `wget`/`curl`/shell. Always port-forward, never `kubectl exec`. ## Auto-Configuration During `stack up` @@ -72,16 +72,16 @@ commands directly. LiteLLM has a static route added by the embedded config: ``` -paid/* → openai/* → http://127.0.0.1:8402/v1 +paid/* → openai/* → http://x402-buyer.llm.svc.cluster.local:8402/v1 ``` -The `x402-buyer` sidecar in the litellm pod listens on port 8402. +`x402-buyer` runs as its own Deployment + Service in the llm namespace, listening on port 8402 (split out of the litellm pod so LiteLLM rolls with maxUnavailable: 0 — issue #321). **Critical**: the trailing `/v1` is mandatory. LiteLLM's OpenAI provider does **not** append `/v1` to a bare `api_base`. Without it, LiteLLM calls `/chat/completions` on the buyer and the buyer mux returns Go's default `404 page not found`, surfaced as `OpenAIException - 404 page not found`. ### Buyer flow -`buy.py` (in the agent pod, `${OBOL_SKILLS_DIR:-/data/.openclaw/skills}/buy-x402/scripts/buy.py`) creates a `PurchaseRequest` CR with pre-signed ERC-3009 (USDC) or Permit2 (OBOL) auths. The serviceoffer-controller reconciles the CR, writes per-upstream buyer config/auth files into the buyer ConfigMaps, and hot-adds the `paid/` LiteLLM route. The sidecar spends one auth per paid request. +`buy.py` (in the agent pod, `${OBOL_SKILLS_DIR:-/data/.openclaw/skills}/buy-x402/scripts/buy.py`) creates a `PurchaseRequest` CR with pre-signed ERC-3009 (USDC) or Permit2 (OBOL) auths. The serviceoffer-controller reconciles the CR, writes per-upstream buyer config/auth files into the buyer ConfigMaps, and hot-adds the `paid/` LiteLLM route. The buyer spends one auth per paid request. ``` probe [--model ] diff --git a/.agents/skills/obol-stack-dev/references/troubleshooting.md b/.agents/skills/obol-stack-dev/references/troubleshooting.md index 2d24218c..00a4121b 100644 --- a/.agents/skills/obol-stack-dev/references/troubleshooting.md +++ b/.agents/skills/obol-stack-dev/references/troubleshooting.md @@ -61,7 +61,7 @@ obol kubectl create configmap ca-certificates -n x402 \ ### `OpenAIException - 404 page not found` on `paid/` -`api_base` for the buyer route must end in `/v1`. LiteLLM's OpenAI provider does **not** append `/v1` to a bare `api_base`. The buyer route must be `http://127.0.0.1:8402/v1`, not `http://127.0.0.1:8402`. +`api_base` for the buyer route must end in `/v1`. LiteLLM's OpenAI provider does **not** append `/v1` to a bare `api_base`. The buyer route must be `http://x402-buyer.llm.svc.cluster.local:8402/v1` (the standalone buyer Service), not the bare `:8402` address. ### `eRPC` `eth_call` cache lag diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 01ed8d71..b6ce3b0f 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -47,6 +47,27 @@ jobs: - name: Run chart-testing (install) run: ct install --target-branch ${{ github.event.repository.default_branch }} + go-test: + name: Go build, vet, and unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: 'go.mod' + + - name: Build + run: go build ./... + + - name: Vet + run: go vet ./... + + - name: Unit tests + run: go test ./... + generate-check: name: CRD generation up-to-date runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index 3eaf1db6..ef64a7f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,7 +118,7 @@ Payment-gated access to cluster services via x402 (HTTP 402 micropayments, Traef **Sell flow**: `obol sell http` -> ServiceOffer CR -> controller reconciles: ModelReady -> UpstreamHealthy -> PaymentGateReady (x402 Middleware) -> RoutePublished (HTTPRoute) -> Registered (RegistrationRequest + optional ERC-8004) -> Ready. Traefik routes `/services//*` through ForwardAuth to upstream. -**Buy flow**: `buy.py probe` sees 402 pricing -> `buy.py buy` validates on-chain token contract -> pre-signs payment auths (ERC-3009 for USDC, Permit2 for OBOL) into `PurchaseRequest` CR in agent ns -> serviceoffer-controller writes buyer config/auth into `llm` ns, publishes `paid/` -> in-pod `x402-buyer` sidecar spends one auth per paid request. Agent-managed refill: `buy.py process --all`, NOT the controller. +**Buy flow**: `buy.py probe` sees 402 pricing -> `buy.py buy` validates on-chain token contract -> pre-signs payment auths (ERC-3009 for USDC, Permit2 for OBOL) into `PurchaseRequest` CR in agent ns -> serviceoffer-controller writes buyer config/auth into `llm` ns, publishes `paid/` -> the `x402-buyer` Deployment spends one auth per paid request. Agent-managed refill: `buy.py process --all`, NOT the controller. **buy.py** at `${OBOL_SKILLS_DIR:-/data/.openclaw/skills}/buy-x402/scripts/buy.py` (skill: `buy-x402`, not `buy`): ``` @@ -273,14 +273,14 @@ Caveats: |---------|---------------------------| | Traefik ingress (frontend, eRPC, x402 routes) | `http://obol.stack:8080/...` | | LiteLLM (`llm` ns, port 4000) | `kubectl port-forward svc/litellm 14000:4000 -n llm` → `http://127.0.0.1:14000` | -| x402-buyer sidecar (port 8402, no Service — pod only) | `kubectl port-forward -n llm 18402:8402` → `http://127.0.0.1:18402` | +| x402-buyer (own Deployment + Service, port 8402) | `kubectl port-forward -n llm svc/x402-buyer 18402:8402` → `http://127.0.0.1:18402` | | OpenClaw instance | `kubectl port-forward -n openclaw- svc/openclaw 18789:18789` | **Never call `http://obol.stack:8080/v1/...`** — Traefik has no `/v1` route, returns frontend 404. -**x402-buyer sidecar is distroless** — no `wget`, `curl`, or shell. Use port-forward, not `kubectl exec`. +**x402-buyer is distroless** — no `wget`, `curl`, or shell. Use port-forward (`svc/x402-buyer`), not `kubectl exec`. -**LiteLLM gateway** (`llm` ns, port 4000): OpenAI-compatible proxy → Ollama/Anthropic/OpenAI. ConfigMap `litellm-config` (YAML config.yaml with model_list), Secret `litellm-secrets` (master key + API keys). Auto-configured with Ollama models during `obol stack up` (no manual `obol model setup`). `ConfigureLiteLLM()` patches config + Secret + restarts or hot-adds via LiteLLM model API. Paid remote inference: Obol LiteLLM fork + `x402-buyer` sidecar, with static `paid/*` → `openai/*` → `http://127.0.0.1:8402` route (wildcard catch-all, requires >=1 concrete `paid/` entry to be useful). Hermes uses provider `"custom"` pointed at `http://litellm.llm.svc.cluster.local:4000/v1`; optional OpenClaw instances reuse the `"openai"` provider slot (ollama slot disabled). Agent configs use `dangerouslyDisableDeviceAuth` for Traefik-proxied access. +**LiteLLM gateway** (`llm` ns, port 4000): OpenAI-compatible proxy → Ollama/Anthropic/OpenAI. ConfigMap `litellm-config` (YAML config.yaml with model_list), Secret `litellm-secrets` (master key + API keys). Auto-configured with Ollama models during `obol stack up` (no manual `obol model setup`). `ConfigureLiteLLM()` patches config + Secret + restarts or hot-adds via LiteLLM model API. Paid remote inference: Obol LiteLLM fork + standalone `x402-buyer` Deployment, with static `paid/*` → `openai/*` → `http://x402-buyer.llm.svc.cluster.local:8402/v1` route (wildcard catch-all, requires >=1 concrete `paid/` entry to be useful). Hermes uses provider `"custom"` pointed at `http://litellm.llm.svc.cluster.local:4000/v1`; optional OpenClaw instances reuse the `"openai"` provider slot (ollama slot disabled). Agent configs use `dangerouslyDisableDeviceAuth` for Traefik-proxied access. **Auto-configuration**: `obol stack up` → `autoConfigureLLM()` detects host Ollama models, patches LiteLLM config. `obolup.sh` → `check_agent_model_api_key()` reads `~/.openclaw/openclaw.json`, resolves API key from `ANTHROPIC_API_KEY` / `CLAUDE_CODE_OAUTH_TOKEN` (Anthropic) or `OPENAI_API_KEY` (OpenAI), exports for downstream. @@ -363,9 +363,9 @@ Key code: `internal/inference/{gateway,container,store}.go`, `internal/enclave/{ **Remote-signer wallet**: `GenerateWallet()` in `internal/openclaw/wallet.go`. secp256k1 → Web3 V3 keystore → remote-signer REST API at port 9000 in same ns. -## Buyer Sidecar +## Buyer (x402-buyer) -`x402-buyer` — lean Go sidecar for buy-side x402 payments using pre-signed ERC-3009 authorizations. Second container in `litellm` Deployment (no separate Service). Agent `buy.py` signs auths locally → `PurchaseRequest`; controller writes per-upstream buyer config/auth files into buyer ConfigMaps, keeps LiteLLM routes in sync. Endpoints: `/status`, `/healthz`, `/metrics`, `/admin/reload`; metrics scraped via `PodMonitor`. Zero signer access, bounded spending (max loss = N × price). +`x402-buyer` — lean Go service for buy-side x402 payments using pre-signed ERC-3009 authorizations. Standalone Deployment + ClusterIP Service `x402-buyer.llm.svc:8402` (was a litellm sidecar; split out so LiteLLM stays stateless and rolls with `maxUnavailable: 0` — issue #321). Agent `buy.py` signs auths locally → `PurchaseRequest`; controller writes per-upstream buyer config/auth files into buyer ConfigMaps, keeps LiteLLM routes in sync. Endpoints: `/status`, `/healthz`, `/metrics`, `/admin/reload`; metrics scraped via `PodMonitor`. Zero signer access, bounded spending (max loss = N × price). Settlement lifecycle (cluster-routed paid flow): - Traefik/x402-verifier stays on the verify-only path (`verifyOnly: true`). @@ -414,8 +414,8 @@ A registry digest pin instead of `:latest` on the verifier means your dev rewrit 3. **ConfigMap propagation** — ~60-120s for k3d file watcher; force restart for immediate effect 4. **ExternalName services** — do not work with Traefik Gateway API; use ClusterIP + Endpoints 5. **eRPC `eth_call` cache** — default TTL is 10s for unfinalized reads; `buy.py balance` can lag behind an already-settled paid request for a few seconds -6. **`/v1` required in `api_base` for `paid/*` route** — LiteLLM's OpenAI provider does NOT append `/v1` to a bare `api_base`. The buyer sidecar route must be `http://127.0.0.1:8402/v1`, not `http://127.0.0.1:8402`. Without `/v1`, LiteLLM calls `/chat/completions` on the buyer; the buyer's mux returns `404 page not found` (Go default), which LiteLLM surfaces as `OpenAIException - 404 page not found`. -7. **LiteLLM restart is fallback, not the default buy path** — the validated happy path is `buy.py buy`/`process --all`/same-name top-up without a manual LiteLLM restart. Controller hot-add/hot-delete + buyer reload is expected to make `paid/` appear and disappear in place. If a paid alias still fails after controller reconciliation and buyer sidecar reports the upstream, restart LiteLLM as a fallback investigation step. Since rc14, Reloader auto-restarts LiteLLM on `litellm-config` changes (e.g. a buy that adds a NEW provider); buyer CM writes (top-ups/refills) hot-reload via `/admin/reload` with no restart — do not add the buyer CMs to the Reloader annotation. +6. **`/v1` required in `api_base` for `paid/*` route** — LiteLLM's OpenAI provider does NOT append `/v1` to a bare `api_base`. The buyer route must be `http://x402-buyer.llm.svc.cluster.local:8402/v1`, not `...:8402`. Without `/v1`, LiteLLM calls `/chat/completions` on the buyer; the buyer's mux returns `404 page not found` (Go default), which LiteLLM surfaces as `OpenAIException - 404 page not found`. +7. **LiteLLM restart is fallback, not the default buy path** — the validated happy path is `buy.py buy`/`process --all`/same-name top-up without a manual LiteLLM restart. Controller hot-add/hot-delete + buyer reload is expected to make `paid/` appear and disappear in place. If a paid alias still fails after controller reconciliation and buyer sidecar reports the upstream, restart LiteLLM as a fallback investigation step. Reloader watches ONLY `litellm-secrets` (key rotation needs a pod replacement; it rolls gaplessly via RollingUpdate maxUnavailable:0). It must NOT watch `litellm-config` — model_list changes are hot-applied via /model/new//model/delete and a CM-triggered rollout would gap inference (issue #321); `obol model status` surfaces CM-vs-router drift if a hot call silently failed. Buyer CM writes (top-ups/refills) hot-reload via `/admin/reload` with no restart — do not add the buyer CMs to any Reloader annotation. 8. **x402-verifier CA bundle missing → TLS failure** — The `x402-verifier` image is distroless (no CA store). The `ca-certificates` ConfigMap in `x402` namespace must be populated from the host CA bundle or the verifier cannot TLS-verify calls to the facilitator (`https://x402.gcp.obol.tech`), causing `x509: certificate signed by unknown authority` on every payment. **Fixed**: `obol stack up` calls `x402verifier.PopulateCABundle` after infrastructure deployment; `obol sell http` calls it before creating the ServiceOffer. If `Payment verification failed` errors still occur, check verifier logs for the x509 error and repopulate manually: `kubectl create configmap ca-certificates -n x402 --from-file=ca-certificates.crt=/etc/ssl/cert.pem --dry-run=client -o yaml | kubectl replace -f -` 9. **`EnsureVerifier` overwrites helmfile's image pin under `OBOL_DEVELOPMENT=true`** — `internal/x402/setup.go` reads embedded `x402.yaml` (hard-coded image pin) and `kubectl apply`s it. Without an in-memory rewrite this overwrites the helmfile-managed `:latest` deployment with the embedded pin → every source change to the verifier silently bypassed. Fix shipped in `5a10fb8` (rewrites pins in-memory before apply); structural regression test: `internal/x402/setup_structure_test.go` (`TestEnsureVerifier_NoInlineRegex`). **If you add a new component installed via `kubectl apply` of an embedded manifest**, give it the same dev-rewrite treatment. 10. **CAIP-2 vs legacy chain id form mismatch** — `RouteRule.Network` is normalized to CAIP-2 (`eip155:84532`) at one boundary, but `internal/x402/chains.go::ResolveChainInfo` must know both that form and the legacy alias (`base-sepolia`). If only one is registered, `matchPaidRouteFull` returns 404 silently on every paid request. When adding a new chain, register both the legacy alias and `CAIP2Network` in every `case` arm. diff --git a/README.md b/README.md index ebc4ec5d..72a1dcc8 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,19 @@   -

The Obol Stack: Where agents deploy their infrastructure

+

The Obol Stack: Run an AI agent business from your own machine

## Overview -The [Obol Stack](https://obol.org/stack) is a framework for AI agents to run decentralised infrastructure locally. It provides an agent with the ability to sync blockchain networks (Ethereum, Aztec, etc.), interact with them via skills, and expose services to the public internet through Cloudflare [tunnels](https://developers.cloudflare.com/cloudflare-one/networks/connectors/cloudflare-tunnel/) and [x402](https://www.x402.org/) payment gateways. +The [Obol Stack](https://obol.org/stack) turns a laptop or home server into a self-sovereign AI agent business: + +- **Run an agent locally.** A [Hermes](https://github.com/NousResearch/hermes-agent) agent with its own crypto wallet, backed by your local models (Ollama) or any cloud provider, orchestrated in a local Kubernetes cluster. +- **Sell to buyers worldwide.** Put inference, agents, or any HTTP service up for sale behind [x402](https://www.x402.org/) micropayments (USDC or OBOL). Buyers pay per request; you get paid onchain, directly to your wallet — no platform in between. +- **Get discovered.** A Cloudflare [tunnel](https://developers.cloudflare.com/cloudflare-one/networks/connectors/cloudflare-tunnel/) gives you a public URL serving a storefront, a machine-readable service catalog, and optional onchain [ERC-8004](https://eips.ethereum.org/EIPS/eip-8004) agent registration. +- **Buy from other sellers.** Purchase paid inference from any x402 seller and route it to your agents. +- **Own your chain access.** Sync blockchain networks (Ethereum, Aztec) locally instead of trusting third-party RPCs. Built on [Kubernetes](https://kubernetes.io) with [Helm](https://helm.sh/) for package management. Read more in the [docs](https://docs.obol.org/next/obol-stack/obol-stack). @@ -28,13 +34,15 @@ Built on [Kubernetes](https://kubernetes.io) with [Helm](https://helm.sh/) for p - **Linux**: [Docker Engine installation guide](https://docs.docker.com/engine/install/) - **macOS/Windows**: [Docker Desktop](https://docs.docker.com/desktop/) +For local models, install [Ollama](https://ollama.com) and pull at least one chat-capable model (e.g. `ollama pull qwen3:8b`). Skip this if you'll use a cloud provider (see [Models](#models)). + ### Install ```bash -bash <(curl -s https://stack.obol.org) +bash <(curl -fsSL https://stack.obol.org) ``` -The installer will set up the `obol` CLI and all dependencies (`kubectl`, `helm`, `k3d`, `helmfile`, `k9s`) into `~/.local/bin/`, configure your PATH, and offer to start the cluster. +The installer sets up the `obol` CLI and all dependencies (`kubectl`, `helm`, `k3d`, `helmfile`, `k9s`) into `~/.local/bin/`, verifies release checksums, configures your PATH, and offers to start the cluster. Verify: @@ -52,112 +60,132 @@ obol stack up # Apply agent capabilities to the default stack-managed agent obol agent init -# Inspect the default Hermes agent +# Inspect the default Hermes agent and grab its dashboard token obol agent list obol agent auth obol-agent ``` -`obol stack up` provisions the default [Hermes Agent](https://github.com/NousResearch/hermes-agent) runtime behind LiteLLM. `obol agent init` applies the controller-based agent capabilities used for monetization and reconciliation. +`obol stack up` provisions the cluster, auto-detects your local Ollama models into the LiteLLM gateway, deploys the default Hermes agent (with its own wallet behind a remote signer), and starts a Cloudflare quick-tunnel. From here you can chat with your agent locally at `http://obol.stack:8080` — or go straight to selling. -## Blockchain Networks +## Sell: Your First Paid Service -Install and run blockchain networks as isolated deployments. Each installation gets a unique namespace so you can run multiple instances side-by-side. +The core loop: put a service on sale → get a public URL → get registered → buyers pay per request, settled onchain to your wallet. -```bash -# List available networks -obol network list +### 1. Put a model up for sale -# Install a network (defaults to network name as ID) -obol network install ethereum -# → ethereum/mainnet +```bash +# Sell local Ollama inference, priced per request (USDC on Base by default) +obol sell inference my-qwen --model qwen3:8b --price 0.001 --pay-to 0x... -# Deploy to the cluster (auto-selects if only one deployment exists) -obol network sync +# Or price per million tokens, or accept OBOL instead of USDC +obol sell inference my-qwen --model qwen3:8b --per-mtok 0.50 --token OBOL --pay-to 0x... +``` -# Or specify by type (auto-selects if only one ethereum deployment) -obol network sync ethereum +If you omit `--pay-to`, the agent's own wallet address is auto-detected from the remote signer — your agent earns for itself. Set defaults once with `obol sell pricing --pay-to 0x... --chain base`. -# Or by full identifier -obol network sync ethereum/mainnet +Check it's live: -# Sync all deployments at once -obol network sync --all +```bash +obol sell list +obol sell status my-qwen +obol sell info # buyer's-eye view of everything on sale ``` -**Available networks:** ethereum, aztec +### 2. Go public -**Ethereum options:** `--network` (mainnet, sepolia, hoodi), `--execution-client` (reth, geth, nethermind, besu, erigon, ethereumjs), `--consensus-client` (lighthouse, prysm, teku, nimbus, lodestar, grandine) +A tunnel gives buyers a permanent URL to reach you. Create a tunnel in the Cloudflare dashboard (Networks → Tunnels), route its Public Hostname to `http://traefik.traefik.svc.cluster.local:80`, then paste the connector token (you can paste the whole `cloudflared tunnel run --token …` line): ```bash -# View installed deployments -obol kubectl get namespaces | grep -E "ethereum|aztec" - -# Delete a deployment (auto-selects if only one exists) -obol network delete -obol network delete ethereum/mainnet --force +obol tunnel setup --hostname stack.example.com +obol tunnel status ``` -> [!TIP] -> Use `obol network install --help` to see all options. +This uses a least-privilege, single-tunnel connector token — no account-wide API key required. Need a domain? `obol domain search`, `obol domain check`, and `obol domain register` wrap Cloudflare Registrar. (Advanced: `obol tunnel setup --management local` uses a browser login instead, which needs `cloudflared` installed.) -## Applications +### 3. Get discovered + +Your public hostname now serves a full discovery surface for buyers, humans and agents alike: -Install arbitrary Helm charts as managed applications: +| Path | What buyers get | +|------|-----------------| +| `/` | Storefront landing page with your branding | +| `/skill.md` | Machine-readable service catalog with worked x402 payment examples | +| `/api/services.json` | JSON catalog: pricing, models, payment requirements | +| `/openapi.json` | OpenAPI spec for your paid endpoints (indexed by x402 scanners) | +| `/.well-known/agent-registration.json` | ERC-8004 agent registration document | +| `/services//*` | The paid services themselves (402 challenge → pay → response) | + +Brand your storefront and optionally register your agent identity onchain: ```bash -# Install from ArtifactHub -obol app install bitnami/redis +obol sell info set --display-name "Acme Labs" --tagline "Paid inference, no middlemen." --logo-url "https://..." +obol sell register --chain base # publish ERC-8004 registration +obol sell identity # inspect your onchain identity +``` -# With specific version -obol app install bitnami/postgresql@15.0.0 +For the full end-to-end walkthrough, see [docs/guides/monetize-inference.md](docs/guides/monetize-inference.md). -# Deploy to cluster (auto-selects if only one app is installed) -obol app sync +## Sell: Other Service Types -# Or specify by type (auto-selects if only one postgresql deployment) -obol app sync postgresql +Anything that speaks HTTP can be payment-gated: -# Or by full identifier -obol app sync postgresql/eager-fox +```bash +# Gate any in-cluster HTTP service (here: Ollama's raw API in the llm namespace) +obol sell http ollama-gated \ + --upstream ollama --port 11434 --namespace llm --health-path /api/tags \ + --per-request 0.001 --chain base --pay-to 0x... -# List and manage -obol app list -obol app delete postgresql/eager-fox --force +# Sell access to an agent itself (wraps an Agent created with `obol agent new`) +obol sell agent my-researcher --per-request 0.01 --pay-to 0x... + +# Run an x402-paid MCP server that proxies a backend API with your own key injected +obol sell mcp my-tool ``` -Find charts at [Artifact Hub](https://artifacthub.io). +> [!NOTE] +> `--namespace` sets both the offer's namespace and the upstream service's namespace. Pass the same `-n ` to follow-up commands (`sell status`, `sell stop`, `sell delete`) — the CLI prints the right invocation after creation. -## Model Providers +## Run Your Business -The stack runs [LiteLLM](https://github.com/BerriAI/litellm) as an in-cluster OpenAI-compatible gateway that proxies all LLM traffic. By default, Ollama on the host machine is used. +```bash +obol sell list # everything on sale +obol sell status # operator health and conditions +obol sell update --price 0.002 # change price or payout wallet in place +obol sell stop # take an offer off sale (keeps it) +obol sell delete # remove an offer +obol sell resume # replay all offers after a host reboot +``` -**Minimum local model size**: agents in this stack rely heavily on tool calling (skills are exposed as OpenAI-style tools, the agent's identity bootstrap reads files, etc.). Models below ~7B parameters tend to either ignore the structured tool-calling channel and return raw JSON in the assistant message, or hallucinate tool failures without actually invoking the tool. Recommended local minimums for reliable agent behaviour: +`obol stack up` re-publishes your offers automatically after a restart; `obol sell resume --install-boot-unit` adds a systemd user unit on Linux so offers come back on boot. -- `llama3.1:8b` — reference Ollama tool-calling implementation -- `qwen3:8b` — strong tool support, modern -- `qwen2.5:7b` (instruct) — works, slightly less disciplined under load +**Back up your business.** Wallets, agent memory, and offers live only on this machine: -Avoid the 1B–4B and `*-coder` variants for the agent role — they pass simple chats but break under multi-tool workflows. They remain fine for embeddings or single-turn completions exposed via `obol sell inference`. +```bash +obol stack export # full backup archive (wallets, brains, offers, config) +obol stack import # restore onto a fresh stack +obol agent wallet backup # wallet-only encrypted backup +``` -To switch to a cloud provider: +## Buy Services + +Buy paid inference from any x402 seller and route it to your agents: ```bash -# Interactive — prompts for provider and API key -obol model setup +# Walk a seller's catalog, preview cost, pre-sign payments, wire up the model +obol buy inference https://inference.example.com/ -# Or pass flags directly -obol model setup --provider anthropic --api-key sk-ant-... -obol model setup --provider openai --api-key sk-proj-... +# Pay from a specific agent's wallet and switch that agent onto the paid model +obol buy inference https://inference.example.com/ --agent research -# Check which providers are enabled -obol model status +# Promote the purchased model to the stack-wide default +obol buy inference https://inference.example.com/ --set-default ``` -`model setup` patches the LiteLLM config and Secret with your API key, adds the model to the gateway, restarts LiteLLM, and syncs the stack-managed Hermes default agent. +The CLI probes the seller's 402 pricing, prompts for how many requests to pre-authorize (with a cost preview), signs payment authorizations via your agent's remote signer, and publishes the model as `paid/` through the LiteLLM gateway. Agents can also buy autonomously — the embedded `buy-x402` skill gives them `probe`, `buy`, `pay`, `balance`, and auto-refill tooling. -## AI Agent Runtimes +## Agents -Hermes is the default AI agent runtime deployed by the stack as `obol-agent`. OpenClaw remains available as an optional manual runtime. Multiple Hermes and OpenClaw instances can run side-by-side. +Hermes is the default runtime, deployed by the stack as `obol-agent`. [OpenClaw](https://github.com/ObolNetwork/openclaw) remains available as an optional runtime. Multiple instances run side-by-side, each in its own namespace with its own wallet. ```bash # Default stack-managed Hermes agent @@ -165,29 +193,31 @@ obol agent list obol agent auth obol-agent obol hermes skills list -# Create and deploy an additional Hermes instance -obol agent new --id research - -# Create and deploy an optional OpenClaw instance -obol agent new --runtime openclaw +# Declare a new sub-agent with a model, skills, an objective, and its own wallet +obol agent new research --model qwen3:8b --skills ethereum-networks,buy-x402 \ + --objective "Research onchain data and sell reports" --create-wallet -# List optional OpenClaw instances -obol agent list --runtime openclaw +# Wallet management +obol agent wallet address +obol agent wallet backup -# Open the OpenClaw web dashboard +# Optional OpenClaw instance +obol agent new --runtime openclaw obol openclaw dashboard ``` -Use `obol agent` for Obol-managed lifecycle and auth flows. Use `obol hermes` for native Hermes CLI commands against the default instance, or pass `--agent ` for a non-default Hermes instance. +Use `obol agent` for Obol-managed lifecycle and auth flows. Use `obol hermes` for native Hermes CLI commands against the default instance, or pass `--agent ` for a non-default instance. An agent created with `agent new` can itself be put on sale with `obol sell agent `. ### Skills -The stack ships with embedded Obol skills that are installed automatically for the default Hermes agent and for OpenClaw instances. Skills give the agent domain-specific capabilities — from querying blockchains to understanding Ethereum development patterns. +The stack ships with embedded Obol skills installed automatically for the default Hermes agent and OpenClaw instances. Skills give agents domain-specific capabilities — from querying blockchains to buying and selling services. -#### Infrastructure +#### Infrastructure & Commerce | Skill | Purpose | |-------|---------| +| `buy-x402` | Buy paid services: probe pricing, pre-sign payments, auto-refill, check balances | +| `monetize` | Manage the agent's own sell offers (ServiceOffer CRUD) | | `ethereum-networks` | Read-only Ethereum queries via cast — blocks, balances, contract reads, ERC-20, ENS | | `ethereum-local-wallet` | Sign and send Ethereum transactions via the per-agent remote-signer | | `obol-stack` | Kubernetes cluster diagnostics — pods, logs, events, deployments | @@ -223,42 +253,114 @@ The stack ships with embedded Obol skills that are installed automatically for t Manage skills at runtime: ```bash -obol openclaw skills list # list installed skills -obol openclaw skills sync # re-inject embedded defaults +obol openclaw skills list # list installed skills +obol openclaw skills sync # re-inject embedded defaults obol openclaw skills sync --from ./my-skills # push custom skills from local dir -obol openclaw skills add # add via openclaw CLI in pod -obol openclaw skills remove # remove via openclaw CLI in pod +obol openclaw skills add # add via openclaw CLI in pod +obol openclaw skills remove # remove via openclaw CLI in pod ``` Skills are delivered via host-path PVC injection — no ConfigMap size limits, works before pod readiness, and survives pod restarts. -## Public Access (Cloudflare Tunnel) +## Models -A tunnel exposes your stack to the public internet so buyers can discover and -pay for the services you sell. You don't need it for local use — set one up once -you're ready to sell, to get a permanent URL. +The stack runs [LiteLLM](https://github.com/BerriAI/litellm) as an in-cluster OpenAI-compatible gateway that proxies all LLM traffic. By default, host Ollama models are auto-detected on `obol stack up`. + +To use a cloud provider instead (or as well): ```bash -# Check tunnel status (a temporary quick-tunnel URL is the default) -obol tunnel status +# Interactive — walks you through provider pick, key creation, and free-tier models +obol model setup -# Create a permanent URL. Create a tunnel in the Cloudflare dashboard -# (Networks → Tunnels), route its Public Hostname to -# http://traefik.traefik.svc.cluster.local:80, then paste the connector token — -# you can paste the whole `cloudflared tunnel run --token …` line: -obol tunnel setup --hostname stack.example.com +# Or scriptable +obol model setup --provider openrouter --api-key sk-or-... +obol model setup --provider anthropic --api-key sk-ant-... + +# Any OpenAI-compatible endpoint (vLLM, sglang, a remote GPU box) +obol model setup custom --endpoint http://192.168.1.20:8000/v1 --model my-model + +# Manage the roster +obol model list # what's routed, in priority order +obol model prefer # promote a model to the default slot +obol model status # provider state +``` + +**Minimum local model size**: agents rely heavily on tool calling. Models below ~7B parameters tend to ignore the structured tool-calling channel or hallucinate tool failures. Recommended local minimums for reliable agent behaviour: `llama3.1:8b`, `qwen3:8b`, or `qwen2.5:7b` (instruct). The 1B–4B and `*-coder` variants remain fine for embeddings or single-turn completions sold via `obol sell inference`. + +## Blockchain Networks + +Install and run blockchain networks as isolated deployments. Each installation gets a unique namespace so you can run multiple instances side-by-side. Local nodes are automatically registered as priority upstreams for the stack's RPC gateway. + +```bash +# List available networks +obol network list + +# Install a network (defaults to network name as ID) +obol network install ethereum +# → ethereum/mainnet + +# Deploy to the cluster (auto-selects if only one deployment exists) +obol network sync + +# Or by full identifier, or all at once +obol network sync ethereum/mainnet +obol network sync --all + +# Add a remote RPC instead of running a node +obol network add +``` + +**Available networks:** ethereum, aztec + +**Ethereum options:** `--network` (mainnet, sepolia, hoodi), `--execution-client` (reth, geth, nethermind, besu, erigon, ethereumjs), `--consensus-client` (lighthouse, prysm, teku, nimbus, lodestar, grandine), `--mode` (full, archive), `--since` (partial archive: `merge`, `365d`, a block number) + +```bash +# View installed deployments +obol kubectl get namespaces | grep -E "ethereum|aztec" + +# Delete a deployment +obol network delete ethereum/mainnet --force +``` + +> [!TIP] +> Use `obol network install --help` to see all options. + +## Applications + +Install arbitrary Helm charts as managed applications — useful for running upstreams you then put on sale with `obol sell http`: + +```bash +# Install from ArtifactHub +obol app install bitnami/redis + +# With specific version +obol app install bitnami/postgresql@15.0.0 + +# Customize values at install or sync time (persisted into the app's values.yaml) +obol app install bitnami/redis --set architecture=standalone --set auth.enabled=false +obol app sync redis --values ./my-overrides.yaml --set image.tag=7.2 + +# Deploy to cluster (auto-selects if only one app is installed) +obol app sync +obol app sync postgresql/eager-fox + +# List and manage +obol app list +obol app delete postgresql/eager-fox --force ``` -This uses a least-privilege, single-tunnel connector token — no account-wide API -key required. (Advanced: `obol tunnel setup --management local` uses a browser -login on this machine instead, which needs `cloudflared` installed.) +Installed apps are re-synced automatically on `obol stack up`, so they survive cluster recreation. After a sync, the CLI prints a ready-to-run `obol sell http` command for each service the app exposes. + +Find charts at [Artifact Hub](https://artifacthub.io). ## Managing the Stack ```bash -obol stack up # Start the cluster +obol stack up # Start the cluster (replays models, RPCs, agents, offers) obol stack down # Stop the cluster (preserves data) -obol stack purge -f # Remove everything (including data) +obol stack purge -f # Remove everything (offers a full export first) +obol update # Check for CLI and chart updates +obol upgrade # Apply chart upgrades obol k9s # Interactive cluster UI ``` @@ -281,20 +383,9 @@ obol stack down && obol stack up Access at http://obol.stack:8080 instead. -#### Direct `X-PAYMENT` Buyers - -Raw direct `X-PAYMENT` requests through the Traefik `ForwardAuth` route are not a supported production payment path. The verifier is intentionally `verifyOnly: true`, so Traefik can gate requests but is not the final settlement point. +#### Monetize Flow Preflight -Use: - -- `x402-buyer` for cluster-routed paid traffic -- `obol sell inference` for direct buyers that need to send raw `X-PAYMENT` - -If you call `x402-verifier /verify` directly for debugging, you must send `X-Forwarded-Uri` (and usually `X-Forwarded-Host`) like Traefik does, or verifier correctly returns `403 forbidden: missing forwarded URI`. - -#### Monetize Flow Preflight (Recommended) - -Before running sell/buy tests on a machine, verify: +If sell/buy flows misbehave, verify in order: ```bash # 1) Kubeconfig matches the currently running k3d cluster (ports can drift). @@ -312,14 +403,16 @@ obol kubectl exec -n hermes-obol-agent deploy/hermes -c hermes -- \ python3 /data/.hermes/obol-skills/buy-x402/scripts/buy.py balance ``` -Run the paid tests only after all four checks pass. +#### Direct `X-PAYMENT` Buyers + +Raw direct `X-PAYMENT` requests through the Traefik `ForwardAuth` route are not a supported production payment path. The verifier is intentionally `verifyOnly: true`, so Traefik can gate requests but is not the final settlement point. Use `x402-buyer` for cluster-routed paid traffic, or `obol sell inference` for direct buyers that need to send raw `X-PAYMENT`. + +If you call `x402-verifier /verify` directly for debugging, you must send `X-Forwarded-Uri` (and usually `X-Forwarded-Host`) like Traefik does, or the verifier correctly returns `403 forbidden: missing forwarded URI`. #### Known Limitations -- `PurchaseRequest.status` (`remaining`/`spent` and `conditions[].message`) is a reconciled snapshot, not a live per-request counter. -- For real-time auth pool state, use `x402-buyer` `GET /status` from the litellm pod. +- `PurchaseRequest.status` (`remaining`/`spent` and `conditions[].message`) is a reconciled snapshot, not a live per-request counter. For real-time auth pool state, use `x402-buyer` `GET /status` from the litellm pod. - Agent-managed refill is driven by `buy.py process --all`; use live sidecar status as the source of truth for refill decisions. -- Raw direct `X-PAYMENT` through Traefik is not a supported production path; use `obol sell inference` if you need a direct buyer flow. ## File Locations @@ -328,13 +421,13 @@ Follows the [XDG Base Directory](https://specifications.freedesktop.org/basedir- | Directory | Purpose | |-----------|---------| | `~/.config/obol/` | Cluster config, kubeconfig, network and app deployments | -| `~/.local/share/obol/` | Persistent volumes (blockchain data) | +| `~/.local/share/obol/` | Persistent volumes (blockchain data, agent state) | | `~/.local/bin/` | CLI binary and dependencies | ## Updating ```bash -bash <(curl -s https://stack.obol.org) +bash <(curl -fsSL https://stack.obol.org) ``` The installer detects your existing installation and upgrades safely. diff --git a/cmd/obol/main.go b/cmd/obol/main.go index abb988a7..d82d2f69 100644 --- a/cmd/obol/main.go +++ b/cmd/obol/main.go @@ -223,6 +223,14 @@ GLOBAL OPTIONS:{{template "visibleFlagTemplate" .}}{{end}} // Best-effort. No-op when `obol sell info set` was // never used. storefront.ReconcileRecorded(cfg, u) + // Re-sync installed app deployments BEFORE sell + // offers: `obol sell http` offers can gate an + // app's Service as their upstream, and the + // controller's upstream health check needs it + // present. App state is declarative on disk + // (helmfile.yaml + values.yaml) but its cluster + // resources live in etcd. Best-effort. + app.ResumeAll(cfg, u) // Re-apply cluster-side state for locally-persisted // `obol sell *` offers. ServiceOffer CRs and the // Service/Endpoints that route to the host gateway @@ -373,6 +381,14 @@ Find charts at https://artifacthub.io`, Aliases: []string{"f"}, Usage: "Overwrite existing deployment", }, + &cli.StringSliceFlag{ + Name: "values", + Usage: "Values file merged onto chart defaults (repeatable, merged in order)", + }, + &cli.StringSliceFlag{ + Name: "set", + Usage: "Override a value, e.g. --set image.tag=1.2.3 (repeatable, applied after --values)", + }, }, Action: func(ctx context.Context, cmd *cli.Command) error { if cmd.NArg() == 0 { @@ -387,10 +403,12 @@ Find charts at https://artifacthub.io`, chartRef := cmd.Args().First() opts := app.InstallOptions{ - Name: cmd.String("name"), - Version: cmd.String("version"), - ID: cmd.String("id"), - Force: cmd.Bool("force"), + Name: cmd.String("name"), + Version: cmd.String("version"), + ID: cmd.String("id"), + Force: cmd.Bool("force"), + ValuesFiles: cmd.StringSlice("values"), + Set: cmd.StringSlice("set"), } return app.Install(cfg, getUI(cmd), chartRef, opts) @@ -400,13 +418,37 @@ Find charts at https://artifacthub.io`, Name: "sync", Usage: "Deploy application to cluster", ArgsUsage: "[/]", + Flags: []cli.Flag{ + &cli.StringSliceFlag{ + Name: "values", + Usage: "Values file merged into the deployment's values.yaml before syncing (repeatable)", + }, + &cli.StringSliceFlag{ + Name: "set", + Usage: "Override a value before syncing, e.g. --set image.tag=1.2.3 (repeatable)", + }, + }, Action: func(ctx context.Context, cmd *cli.Command) error { identifier, _, err := app.ResolveInstance(cfg, cmd.Args().Slice()) if err != nil { return err } - return app.Sync(cfg, getUI(cmd), identifier) + u := getUI(cmd) + // Overrides are persisted into the deployment's + // values.yaml (not passed as ephemeral flags) so + // resume-on-stack-up replays them. + if err := app.ApplyOverrides(cfg, identifier, cmd.StringSlice("values"), cmd.StringSlice("set")); err != nil { + return err + } + + if err := app.Sync(cfg, u, identifier); err != nil { + return err + } + + app.PrintSellHint(cfg, u, identifier) + + return nil }, }, { diff --git a/cmd/obol/model.go b/cmd/obol/model.go index b9e97a5c..610bd957 100644 --- a/cmd/obol/model.go +++ b/cmd/obol/model.go @@ -465,6 +465,12 @@ func modelSetupCustomCommand(cfg *config.Config) *cli.Command { type modelStatusResult struct { Providers []modelStatusProvider `json:"providers"` Discovered []discoverProvider `json:"discovered,omitempty"` + // RouterDrift is set when the live LiteLLM router disagrees with the + // litellm-config ConfigMap (a hot-add/hot-delete failed or never ran). + // Omitted when the check passed; RouterDriftError explains when the + // check itself could not run (cluster or pod unavailable). + RouterDrift *model.RouterDrift `json:"router_drift,omitempty"` + RouterDriftError string `json:"router_drift_error,omitempty"` } type modelStatusProvider struct { @@ -498,10 +504,21 @@ func modelStatusCommand(cfg *config.Config) *cli.Command { defer cancel() discovered, _ := model.DiscoverLocalProviders(scanCtx) + // Drift safety net: Reloader no longer restarts LiteLLM on + // litellm-config changes, so a silently-failed hot-add/hot-delete + // is only visible here. Non-fatal — status must still render + // when the cluster or pod is unreachable. + drift, driftErr := model.CheckRouterDrift(cfg) + if u.IsJSON() { result := modelStatusResult{ Discovered: discoveredProvidersToJSON(discovered), } + if driftErr != nil { + result.RouterDriftError = driftErr.Error() + } else if !drift.Empty() { + result.RouterDrift = &drift + } for _, name := range providers { s := status[name] key := "n/a" @@ -548,6 +565,21 @@ func modelStatusCommand(cfg *config.Config) *cli.Command { } u.Blank() + switch { + case driftErr != nil: + u.Dim(fmt.Sprintf(" Live router check unavailable: %v", driftErr)) + u.Blank() + case !drift.Empty(): + u.Warnf("LiteLLM live router disagrees with litellm-config (hot update failed?):") + for _, m := range drift.Missing { + u.Printf(" missing from router: %s (requests 404 until next LiteLLM rollout)", m) + } + for _, m := range drift.Extra { + u.Printf(" extra in router: %s (disappears on next LiteLLM rollout)", m) + } + u.Dim(" Fix: `obol kubectl rollout restart deployment/litellm -n llm` reloads the ConfigMap.") + u.Blank() + } if printModelRanking(u, cfg, "section") { u.Blank() } diff --git a/cmd/obol/sell.go b/cmd/obol/sell.go index 036c8e78..ffc742b5 100644 --- a/cmd/obol/sell.go +++ b/cmd/obol/sell.go @@ -22,6 +22,7 @@ import ( "time" "github.com/ObolNetwork/obol-stack/internal/agentcrd" + "github.com/ObolNetwork/obol-stack/internal/app" "github.com/ObolNetwork/obol-stack/internal/config" "github.com/ObolNetwork/obol-stack/internal/erc8004" "github.com/ObolNetwork/obol-stack/internal/hermes" @@ -3943,6 +3944,10 @@ Examples: // Recorded Agent CRs first: agent-backed offers resolve // agent.ref and would dangle on a freshly-recreated cluster. agentcrd.ResumeAll(cfg, u) + // Installed apps next: http offers can gate an app's Service + // as their upstream, so the Service must exist before the + // offer republishes. Best-effort. + app.ResumeAll(cfg, u) if err := resumeSellOffers(ctx, cfg, u); err != nil { return err } diff --git a/cmd/obol/stackup_resume_guard_test.go b/cmd/obol/stackup_resume_guard_test.go index f87fd057..e37d3f9a 100644 --- a/cmd/obol/stackup_resume_guard_test.go +++ b/cmd/obol/stackup_resume_guard_test.go @@ -22,6 +22,7 @@ func TestStackUpAction_ReplaysRecordedState(t *testing.T) { upIdx := strings.Index(body, "stack.Up(cfg") rpcIdx := strings.Index(body, "network.ReconcileRecordedRPCs(") agentsIdx := strings.Index(body, "agentcrd.ResumeAll(") + appsIdx := strings.Index(body, "app.ResumeAll(") offersIdx := strings.Index(body, "resumeSellOffers(") if rpcIdx < 0 { @@ -30,15 +31,21 @@ func TestStackUpAction_ReplaysRecordedState(t *testing.T) { if agentsIdx < 0 { t.Fatal("cmd/obol/main.go must call agentcrd.ResumeAll — without it recorded Agent CRs never reach a freshly-recreated cluster") } + if appsIdx < 0 { + t.Fatal("cmd/obol/main.go must call app.ResumeAll — without it installed apps never reach a freshly-recreated cluster") + } if upIdx < 0 || offersIdx < 0 { t.Fatalf("expected stack.Up and resumeSellOffers in main.go; upIdx=%d offersIdx=%d", upIdx, offersIdx) } - if rpcIdx < upIdx || agentsIdx < upIdx { + if rpcIdx < upIdx || agentsIdx < upIdx || appsIdx < upIdx { t.Error("recorded-state replay must run AFTER stack.Up — before it there is no kubeconfig/cluster") } if agentsIdx > offersIdx { t.Error("agentcrd.ResumeAll must run BEFORE resumeSellOffers — agent-backed ServiceOffers need their Agent CR first") } + if appsIdx > offersIdx { + t.Error("app.ResumeAll must run BEFORE resumeSellOffers — http ServiceOffers can gate an app's Service as their upstream") + } } // TestSellResumeAction_ReplaysAgentsBeforeOffers extends the same guard to @@ -56,6 +63,10 @@ func TestSellResumeAction_ReplaysAgentsBeforeOffers(t *testing.T) { if agentsIdx < 0 { t.Fatal("cmd/obol/sell.go (sell resume action) must call agentcrd.ResumeAll before replaying offers") } + appsIdx := strings.Index(body, "app.ResumeAll(") + if appsIdx < 0 { + t.Fatal("cmd/obol/sell.go (sell resume action) must call app.ResumeAll before replaying offers — http offers can gate app upstreams") + } // The resume action's offer replay is the only call site that returns // the error (`if err := resumeSellOffers(...)`); main.go's stack-up // call warns instead. @@ -66,4 +77,7 @@ func TestSellResumeAction_ReplaysAgentsBeforeOffers(t *testing.T) { if agentsIdx > offersIdx { t.Error("agentcrd.ResumeAll must run BEFORE resumeSellOffers in the sell resume action") } + if appsIdx > offersIdx { + t.Error("app.ResumeAll must run BEFORE resumeSellOffers in the sell resume action") + } } diff --git a/docs/getting-started.md b/docs/getting-started.md index 171906b9..8823f3eb 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -20,7 +20,7 @@ This guide walks you through installing the Obol Stack, starting a local Kuberne Run the bootstrap installer: ```bash -bash <(curl -s https://stack.obol.org) +bash <(curl -fsSL https://stack.obol.org) ``` This installs the `obol` CLI and all required tools (kubectl, helm, k3d, helmfile, k9s) to `~/.local/bin/`. diff --git a/flows/buy-external.sh b/flows/buy-external.sh index e3d3f7d8..bd054edb 100755 --- a/flows/buy-external.sh +++ b/flows/buy-external.sh @@ -222,7 +222,7 @@ external_snapshot_on_fail() { python3 -c " import urllib.request, json try: - resp = urllib.request.urlopen('http://localhost:8402/status', timeout=5) + resp = urllib.request.urlopen('http://x402-buyer.llm.svc.cluster.local:8402/status', timeout=5) print(json.dumps(json.loads(resp.read()), indent=2)) except Exception as e: print(json.dumps({'error': repr(e)})) @@ -478,7 +478,7 @@ bob kubectl exec -n llm deployment/litellm -c litellm -- \ python3 -c " import urllib.request, json, sys try: - resp = urllib.request.urlopen('http://localhost:8402/status', timeout=5) + resp = urllib.request.urlopen('http://x402-buyer.llm.svc.cluster.local:8402/status', timeout=5) print(json.dumps(json.loads(resp.read()), indent=2)) except Exception as e: print(json.dumps({'error': repr(e)})) @@ -550,7 +550,7 @@ bob kubectl rollout status deployment/litellm -n llm --timeout=180s >/dev/null 2 sidecar_status_raw=$(bob kubectl exec -n llm deployment/litellm -c litellm -- \ python3 -c " import urllib.request -print(urllib.request.urlopen('http://localhost:8402/status', timeout=5).read().decode()) +print(urllib.request.urlopen('http://x402-buyer.llm.svc.cluster.local:8402/status', timeout=5).read().decode()) " 2>/dev/null || true) PAID_MODEL=$(printf '%s' "$sidecar_status_raw" | python3 -c " import json, sys @@ -682,7 +682,7 @@ bob kubectl exec -n llm deployment/litellm -c litellm -- \ python3 -c " import urllib.request, json try: - resp = urllib.request.urlopen('http://localhost:8402/status', timeout=5) + resp = urllib.request.urlopen('http://x402-buyer.llm.svc.cluster.local:8402/status', timeout=5) print(json.dumps(json.loads(resp.read()), indent=2)) except Exception as e: print(json.dumps({'error': repr(e)})) diff --git a/flows/flow-06-sell-setup.sh b/flows/flow-06-sell-setup.sh index 7cba7d61..e9ab454d 100755 --- a/flows/flow-06-sell-setup.sh +++ b/flows/flow-06-sell-setup.sh @@ -119,19 +119,21 @@ if [ "$litellm_port" = "4000" ]; then else fail "LiteLLM service port unexpected: $litellm_port (expected 4000)" fi -# Verify LiteLLM pod has 2 containers (litellm + x402-buyer sidecar) -step "LiteLLM pod has 2 containers (litellm + x402-buyer sidecar)" -container_count=$("$OBOL" kubectl get pods -n llm --no-headers 2>&1 | awk '{print $2}' | head -1) -if [ "$container_count" = "2/2" ]; then - pass "LiteLLM pod has 2/2 containers (litellm + x402-buyer sidecar)" +# x402-buyer runs as its own Deployment since the buyer split (litellm pod +# is single-container and stateless for zero-downtime rollouts). +step "x402-buyer deployment ready (standalone, no longer a litellm sidecar)" +buyer_ready=$("$OBOL" kubectl get deploy x402-buyer -n llm \ + -o jsonpath='{.status.readyReplicas}' 2>&1) || true +if [ "$buyer_ready" = "1" ]; then + pass "x402-buyer deployment has 1 ready replica" else - fail "LiteLLM pod container count unexpected: $container_count (expected 2/2)" + fail "x402-buyer deployment not ready: readyReplicas=$buyer_ready (expected 1)" fi -# Verify x402-buyer sidecar health (serves /healthz at port 8402 in litellm pod) -step "x402-buyer sidecar healthy (buy-side payment handler)" +# Verify x402-buyer health (serves /healthz at port 8402 via its Service) +step "x402-buyer healthy (buy-side payment handler)" kill $(lsof -ti:8402) 2>/dev/null || true -"$OBOL" kubectl port-forward -n llm deployment/litellm 8402:8402 &>/dev/null & +"$OBOL" kubectl port-forward -n llm svc/x402-buyer 8402:8402 &>/dev/null & PF_BUYER_PID=$! for i in $(seq 1 8); do if curl -sf --max-time 2 http://localhost:8402/healthz >/dev/null 2>&1; then @@ -142,9 +144,9 @@ done buyer_health=$(curl -sf --max-time 5 http://localhost:8402/healthz 2>&1) || true cleanup_pid "$PF_BUYER_PID" if echo "$buyer_health" | grep -q "ok"; then - pass "x402-buyer sidecar healthy: $buyer_health" + pass "x402-buyer healthy: $buyer_health" else - fail "x402-buyer sidecar health check failed — ${buyer_health:0:100}" + fail "x402-buyer health check failed — ${buyer_health:0:100}" fi if [ "$SELL_UPSTREAM_SERVICE" = "ollama" ]; then run_step_grep "Ollama reachable" "models" curl -sf http://localhost:11434/api/tags diff --git a/flows/flow-08-buy.sh b/flows/flow-08-buy.sh index eed7c82d..1de2a63f 100755 --- a/flows/flow-08-buy.sh +++ b/flows/flow-08-buy.sh @@ -68,7 +68,7 @@ buyer_sidecar_status() { python3 -c " import urllib.request, json try: - resp = urllib.request.urlopen('http://localhost:8402/status', timeout=5) + resp = urllib.request.urlopen('http://x402-buyer.llm.svc.cluster.local:8402/status', timeout=5) d = json.loads(resp.read()) for name, info in d.items(): print('%s: remaining=%d spent=%d model=%s' % (name, info['remaining'], info['spent'], info['public_model'])) diff --git a/flows/flow-11-dual-stack.sh b/flows/flow-11-dual-stack.sh index 19b61c2b..fe092760 100755 --- a/flows/flow-11-dual-stack.sh +++ b/flows/flow-11-dual-stack.sh @@ -308,7 +308,7 @@ buyer_sidecar_status() { python3 -c " import urllib.request, json try: - resp = urllib.request.urlopen('http://localhost:8402/status', timeout=5) + resp = urllib.request.urlopen('http://x402-buyer.llm.svc.cluster.local:8402/status', timeout=5) d = json.loads(resp.read()) for name, info in d.items(): print('%s: remaining=%d spent=%d model=%s' % (name, info['remaining'], info['spent'], info['public_model'])) diff --git a/flows/lib-dual-stack.sh b/flows/lib-dual-stack.sh index b949f37e..b8a72d76 100644 --- a/flows/lib-dual-stack.sh +++ b/flows/lib-dual-stack.sh @@ -400,7 +400,7 @@ buyer_sidecar_status() { python3 -c " import urllib.request, json try: - resp = urllib.request.urlopen('http://localhost:8402/status', timeout=5) + resp = urllib.request.urlopen('http://x402-buyer.llm.svc.cluster.local:8402/status', timeout=5) d = json.loads(resp.read()) for name, info in d.items(): print('%s: remaining=%d spent=%d model=%s' % (name, info['remaining'], info['spent'], info['public_model'])) diff --git a/internal/agentruntime/charts.go b/internal/agentruntime/charts.go index 0cb03414..ae774862 100644 --- a/internal/agentruntime/charts.go +++ b/internal/agentruntime/charts.go @@ -5,7 +5,7 @@ package agentruntime // deployments. It MUST be updated as a single edit; bumping it here // updates every consumer in lockstep. // -// Chart 0.3.3 ships remote-signer image `v0.4.0`, which honours +// Chart 0.4.0 ships remote-signer image `v0.4.0`, which honours // `SIGNER__AUTH__TOKEN` (the bearer token the controller mints into the // keystore Secret and injects via env). Canonical Ethereum recovery-id // signatures (`v=27/28`) from `/sign/.../message`, `/sign/.../typed-data`, @@ -14,5 +14,11 @@ package agentruntime // the buy.py caller to renormalize for EIP-712 / ERC-3009 verifiers like // USDC `transferWithAuthorization`. // +// 0.4.0 also renders `spec.strategy` from `.Values.strategy` (defaulting to +// `Recreate`), so the RWO-keystore singleton no longer needs an imperative +// post-sync `kubectl patch` to stay off RollingUpdate. `values-remote-signer.yaml` +// pins `strategy.type: Recreate` explicitly so the intent is visible at the +// obol-stack layer and survives any future change to the chart default. +// // renovate: datasource=helm depName=remote-signer registryUrl=https://obolnetwork.github.io/helm-charts/ -const RemoteSignerChartVersion = "0.3.3" +const RemoteSignerChartVersion = "0.4.0" diff --git a/internal/app/app.go b/internal/app/app.go index e2f899d2..a98d29cd 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -18,10 +18,12 @@ import ( // InstallOptions contains options for the install command type InstallOptions struct { - Name string // Optional app name override - Version string // Chart version (empty = latest for repo/chart, extracted for URL) - ID string // Deployment ID (empty = generate petname) - Force bool // Overwrite existing deployment + Name string // Optional app name override + Version string // Chart version (empty = latest for repo/chart, extracted for URL) + ID string // Deployment ID (empty = generate petname) + Force bool // Overwrite existing deployment + ValuesFiles []string // Values files merged onto chart defaults (in order) + Set []string // key.path=value overrides applied after values files } // ListOptions contains options for the list command @@ -117,13 +119,20 @@ func Install(cfg *config.Config, u *ui.UI, chartRef string, opts InstallOptions) return fmt.Errorf("failed to write values.yaml: %w", err) } - // 9. Generate helmfile.yaml (references chart remotely) + // 9. Merge user overrides (--values files, then --set) into values.yaml + // so the deployment directory records exactly what will deploy. + if err := applyOverridesToFile(valuesPath, opts.ValuesFiles, opts.Set); err != nil { + os.RemoveAll(deploymentDir) + return fmt.Errorf("failed to apply value overrides: %w", err) + } + + // 10. Generate helmfile.yaml (references chart remotely) if err := generateRemoteHelmfile(deploymentDir, chart, appName, id); err != nil { os.RemoveAll(deploymentDir) return fmt.Errorf("failed to generate helmfile: %w", err) } - // 10. Print success message + // 11. Print success message u.Blank() u.Successf("Application installed successfully!") u.Detail("Deployment", fmt.Sprintf("%s/%s", appName, id)) diff --git a/internal/app/resume.go b/internal/app/resume.go new file mode 100644 index 00000000..3f499bdd --- /dev/null +++ b/internal/app/resume.go @@ -0,0 +1,36 @@ +package app + +import ( + "github.com/ObolNetwork/obol-stack/internal/config" + "github.com/ObolNetwork/obol-stack/internal/ui" +) + +// ResumeAll re-syncs every installed app deployment into the (possibly +// freshly recreated) cluster. App deployments are declarative on disk +// (helmfile.yaml + values.yaml under applications///) but their +// cluster resources live in etcd, which `obol stack down` + recreate +// destroys — without this replay a fresh `stack up` comes back with the +// deployment directories still on disk but no matching cluster resources, +// and any `obol sell http` offer gating an app upstream dangles. +// +// Best-effort, mirroring agentcrd.ResumeAll: a failed app warns and does +// not block stack-up. +func ResumeAll(cfg *config.Config, u *ui.UI) { + ids, err := ListInstanceIDs(cfg) + if err != nil { + u.Warnf("Could not list installed apps: %v", err) + return + } + + if len(ids) == 0 { + return + } + + u.Infof("Re-syncing %d installed app(s)...", len(ids)) + + for _, id := range ids { + if err := Sync(cfg, u, id); err != nil { + u.Warnf("Could not re-sync app %s (run 'obol app sync %s' to retry): %v", id, id, err) + } + } +} diff --git a/internal/app/resume_test.go b/internal/app/resume_test.go new file mode 100644 index 00000000..db8cb196 --- /dev/null +++ b/internal/app/resume_test.go @@ -0,0 +1,63 @@ +package app + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ObolNetwork/obol-stack/internal/config" + "github.com/ObolNetwork/obol-stack/internal/ui" +) + +// TestResumeAll_NoApps ensures a config dir with no app deployments is a +// silent no-op — stack up must not print resume noise for users who never +// installed an app. +func TestResumeAll_NoApps(t *testing.T) { + cfg := &config.Config{ConfigDir: t.TempDir()} + + var stdout, stderr bytes.Buffer + ResumeAll(cfg, ui.NewForTest(&stdout, &stderr)) + + if out := stdout.String() + stderr.String(); out != "" { + t.Errorf("expected no output with zero apps, got: %q", out) + } +} + +// TestResumeAll_SkipsNonAppStateDirs ensures resume only replays real app +// deployments (identified by values.yaml) and never touches sibling +// subsystem state dirs like applications/hermes/, which use +// values-.yaml naming and are NOT helmfile-syncable apps. +func TestResumeAll_SkipsNonAppStateDirs(t *testing.T) { + configDir := t.TempDir() + cfg := &config.Config{ConfigDir: configDir} + + appDir := filepath.Join(configDir, "applications", "redis", "eager-fox") + if err := os.MkdirAll(appDir, 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, appDir, "values.yaml", "a: 1\n") + writeFile(t, appDir, "helmfile.yaml", "releases: []\n") + + hermesDir := filepath.Join(configDir, "applications", "hermes", "obol-agent") + if err := os.MkdirAll(hermesDir, 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, hermesDir, "values-hermes.yaml", "b: 2\n") + writeFile(t, hermesDir, "helmfile.yaml", "releases: []\n") + + var stdout, stderr bytes.Buffer + // No kubeconfig.yaml in configDir, so Sync fails fast with "cluster + // not running" — the point is WHICH deployments resume attempts. + ResumeAll(cfg, ui.NewForTest(&stdout, &stderr)) + + out := stdout.String() + stderr.String() + + if !strings.Contains(out, "redis/eager-fox") { + t.Errorf("expected resume attempt for redis/eager-fox, got: %q", out) + } + if strings.Contains(out, "hermes") { + t.Errorf("resume must not touch the hermes state dir, got: %q", out) + } +} diff --git a/internal/app/sellhint.go b/internal/app/sellhint.go new file mode 100644 index 00000000..c17b57c3 --- /dev/null +++ b/internal/app/sellhint.go @@ -0,0 +1,75 @@ +package app + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/ObolNetwork/obol-stack/internal/config" + "github.com/ObolNetwork/obol-stack/internal/ui" +) + +// maxSellHints caps how many per-service sell suggestions are printed +// after a sync; charts with many Services (umbrella charts) would +// otherwise drown the success output. +const maxSellHints = 3 + +// PrintSellHint discovers the Services a just-synced app exposes and +// prints a ready-to-run `obol sell http` command for each, connecting app +// installs to the monetization flow. Best-effort: any discovery failure +// (no kubeconfig, kubectl missing, no Services yet) prints nothing. +func PrintSellHint(cfg *config.Config, u *ui.UI, deploymentIdentifier string) { + appName, id, err := parseDeploymentIdentifier(deploymentIdentifier) + if err != nil { + return + } + + namespace := fmt.Sprintf("%s-%s", appName, id) + + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + if _, err := os.Stat(kubeconfigPath); err != nil { + return + } + + kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") + if _, err := os.Stat(kubectlBinary); err != nil { + return + } + + cmd := exec.Command(kubectlBinary, "get", "services", "-n", namespace, + "-o", `jsonpath={range .items[*]}{.metadata.name} {.spec.ports[0].port}{"\n"}{end}`) + cmd.Env = append(os.Environ(), "KUBECONFIG="+kubeconfigPath) + + out, err := cmd.Output() + if err != nil { + return + } + + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + + printed := 0 + + for _, line := range lines { + name, port, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok || name == "" || port == "" { + continue + } + + if printed == 0 { + u.Blank() + u.Print("To put this app on sale behind x402 payments:") + } + + if printed == maxSellHints { + u.Printf(" (more services: obol kubectl get svc -n %s)", namespace) + break + } + + u.Printf(" obol sell http %s --upstream %s --port %s --namespace %s --per-request 0.001 --pay-to 0x", + name, name, port, namespace) + + printed++ + } +} diff --git a/internal/app/values.go b/internal/app/values.go new file mode 100644 index 00000000..0b6bfa9b --- /dev/null +++ b/internal/app/values.go @@ -0,0 +1,137 @@ +package app + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ObolNetwork/obol-stack/internal/config" + "gopkg.in/yaml.v3" +) + +// ApplyOverrides merges values files and --set overrides into a +// deployment's values.yaml. Overrides are persisted to disk rather than +// passed as ephemeral helmfile flags so the deployment directory stays the +// single declarative record — `obol app sync` and stack-up resume replay +// exactly what was last applied. +// +// Files are merged in order, then set expressions on top (later wins). +func ApplyOverrides(cfg *config.Config, deploymentIdentifier string, valuesFiles, sets []string) error { + if len(valuesFiles) == 0 && len(sets) == 0 { + return nil + } + + appName, id, err := parseDeploymentIdentifier(deploymentIdentifier) + if err != nil { + return err + } + + valuesPath := filepath.Join(cfg.ConfigDir, "applications", appName, id, "values.yaml") + + return applyOverridesToFile(valuesPath, valuesFiles, sets) +} + +// applyOverridesToFile does the actual load → merge → write on a +// values.yaml path. Split out so Install can call it on a freshly written +// file without going through identifier resolution. +func applyOverridesToFile(valuesPath string, valuesFiles, sets []string) error { + base, err := loadValuesMap(valuesPath) + if err != nil { + return err + } + + for _, file := range valuesFiles { + overlay, err := loadValuesMap(file) + if err != nil { + return err + } + + base = mergeMaps(base, overlay) + } + + for _, expr := range sets { + if err := applySetExpression(base, expr); err != nil { + return err + } + } + + out, err := yaml.Marshal(base) + if err != nil { + return fmt.Errorf("failed to marshal merged values: %w", err) + } + + return os.WriteFile(valuesPath, out, 0o600) +} + +// loadValuesMap reads a YAML file into a string-keyed map. A file that is +// empty or contains only comments (e.g. a chart with no default values) +// yields an empty map. +func loadValuesMap(path string) (map[string]any, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read values file: %w", err) + } + + var values map[string]any + if err := yaml.Unmarshal(data, &values); err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", path, err) + } + + if values == nil { + values = map[string]any{} + } + + return values, nil +} + +// mergeMaps deep-merges src into dst: nested maps merge recursively, +// everything else (scalars, lists) is replaced by src. Returns dst. +func mergeMaps(dst, src map[string]any) map[string]any { + for key, srcVal := range src { + if dstMap, ok := dst[key].(map[string]any); ok { + if srcMap, ok := srcVal.(map[string]any); ok { + dst[key] = mergeMaps(dstMap, srcMap) + continue + } + } + + dst[key] = srcVal + } + + return dst +} + +// applySetExpression applies a single "path.to.key=value" override onto +// values, creating intermediate maps as needed. The value is parsed as a +// YAML scalar so `true`, `3`, and quoted strings get their natural types. +// Existing non-map intermediates are overwritten with maps, matching helm +// --set semantics. +func applySetExpression(values map[string]any, expr string) error { + key, rawValue, ok := strings.Cut(expr, "=") + if !ok || key == "" { + return fmt.Errorf("invalid --set expression %q: expected key.path=value", expr) + } + + var value any + if err := yaml.Unmarshal([]byte(rawValue), &value); err != nil { + return fmt.Errorf("invalid --set value in %q: %w", expr, err) + } + + segments := strings.Split(key, ".") + current := values + + for _, segment := range segments[:len(segments)-1] { + next, ok := current[segment].(map[string]any) + if !ok { + next = map[string]any{} + current[segment] = next + } + + current = next + } + + current[segments[len(segments)-1]] = value + + return nil +} diff --git a/internal/app/values_test.go b/internal/app/values_test.go new file mode 100644 index 00000000..92ff7d59 --- /dev/null +++ b/internal/app/values_test.go @@ -0,0 +1,120 @@ +package app + +import ( + "os" + "path/filepath" + "testing" + + "gopkg.in/yaml.v3" +) + +func writeFile(t *testing.T, dir, name, content string) string { + t.Helper() + + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + + return path +} + +func readValues(t *testing.T, path string) map[string]any { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read values: %v", err) + } + + var values map[string]any + if err := yaml.Unmarshal(data, &values); err != nil { + t.Fatalf("parse values: %v", err) + } + + return values +} + +func TestApplyOverridesToFile_SetExpressions(t *testing.T) { + dir := t.TempDir() + valuesPath := writeFile(t, dir, "values.yaml", "image:\n tag: latest\nreplicas: 1\n") + + err := applyOverridesToFile(valuesPath, nil, []string{ + "image.tag=1.2.3", // overwrite nested scalar + "service.port=8080", // create intermediate map + "persistence=true", // typed bool + "replicas=3", // typed int + "name=my-app", // plain string + }) + if err != nil { + t.Fatalf("applyOverridesToFile: %v", err) + } + + values := readValues(t, valuesPath) + + if got := values["image"].(map[string]any)["tag"]; got != "1.2.3" { + t.Errorf("image.tag = %v, want 1.2.3", got) + } + if got := values["service"].(map[string]any)["port"]; got != 8080 { + t.Errorf("service.port = %v (%T), want int 8080", got, got) + } + if got := values["persistence"]; got != true { + t.Errorf("persistence = %v (%T), want bool true", got, got) + } + if got := values["replicas"]; got != 3 { + t.Errorf("replicas = %v (%T), want int 3", got, got) + } + if got := values["name"]; got != "my-app" { + t.Errorf("name = %v, want my-app", got) + } +} + +func TestApplyOverridesToFile_ValuesFilesMergeInOrder(t *testing.T) { + dir := t.TempDir() + valuesPath := writeFile(t, dir, "values.yaml", "image:\n tag: latest\n pullPolicy: IfNotPresent\n") + first := writeFile(t, dir, "first.yaml", "image:\n tag: from-first\nextra: 1\n") + second := writeFile(t, dir, "second.yaml", "image:\n tag: from-second\n") + + if err := applyOverridesToFile(valuesPath, []string{first, second}, []string{"extra=2"}); err != nil { + t.Fatalf("applyOverridesToFile: %v", err) + } + + values := readValues(t, valuesPath) + image := values["image"].(map[string]any) + + if image["tag"] != "from-second" { + t.Errorf("image.tag = %v, want from-second (later file wins)", image["tag"]) + } + if image["pullPolicy"] != "IfNotPresent" { + t.Errorf("image.pullPolicy = %v, want IfNotPresent (untouched keys survive merge)", image["pullPolicy"]) + } + if values["extra"] != 2 { + t.Errorf("extra = %v, want 2 (--set applies after --values)", values["extra"]) + } +} + +func TestApplyOverridesToFile_CommentOnlyValues(t *testing.T) { + dir := t.TempDir() + valuesPath := writeFile(t, dir, "values.yaml", "# No default values in chart\n") + + if err := applyOverridesToFile(valuesPath, nil, []string{"a.b=c"}); err != nil { + t.Fatalf("applyOverridesToFile on comment-only file: %v", err) + } + + values := readValues(t, valuesPath) + if got := values["a"].(map[string]any)["b"]; got != "c" { + t.Errorf("a.b = %v, want c", got) + } +} + +func TestApplyOverridesToFile_InvalidSetExpression(t *testing.T) { + dir := t.TempDir() + valuesPath := writeFile(t, dir, "values.yaml", "a: 1\n") + + if err := applyOverridesToFile(valuesPath, nil, []string{"no-equals-sign"}); err == nil { + t.Error("expected error for --set expression without '='") + } + if err := applyOverridesToFile(valuesPath, nil, []string{"=value"}); err == nil { + t.Error("expected error for --set expression with empty key") + } +} diff --git a/internal/embed/embed_buyer_state_test.go b/internal/embed/embed_buyer_state_test.go index db99ef81..38b841ed 100644 --- a/internal/embed/embed_buyer_state_test.go +++ b/internal/embed/embed_buyer_state_test.go @@ -3,13 +3,18 @@ package embed import "testing" // TestBuyerStatePVC asserts that x402-buyer's /state is backed by a PVC -// (not an emptyDir), and that the litellm Deployment uses the Recreate -// strategy so the RWO PVC can be remounted without overlap. +// (not an emptyDir), that the buyer runs as its own Recreate Deployment so +// the RWO PVC is remounted without overlap, and that the litellm Deployment +// is stateless with RollingUpdate maxUnavailable: 0 (zero-downtime rollouts, +// issue #321). // -// Regression: emptyDir lost consumed.json on every pod restart, causing +// Regression (emptyDir): lost consumed.json on every pod restart, causing // the buyer to re-spend already-consumed auths from the ConfigMap pool // and cascading into facilitator 400s ("nonce already used") until a // manual `buy.py process --all` reseeded. +// Regression (sidecar coupling): while the buyer lived in the litellm pod, +// the RWO PVC forced litellm onto Recreate — every rollout (Reloader secret +// change, image bump) was a full inference gap. func TestBuyerStatePVC(t *testing.T) { data, err := ReadInfrastructureFile("base/templates/llm.yaml") if err != nil { @@ -41,15 +46,15 @@ func TestBuyerStatePVC(t *testing.T) { t.Error("PVC missing spec.resources.requests.storage") } - // litellm Deployment volume entry must reference the PVC, not emptyDir. - dep := findDocByName(docs, "Deployment", "litellm") - if dep == nil { - t.Fatal("litellm Deployment missing from llm.yaml") + // The x402-buyer Deployment owns the PVC mount; litellm must not. + buyerDep := findDocByName(docs, "Deployment", "x402-buyer") + if buyerDep == nil { + t.Fatal("x402-buyer Deployment missing from llm.yaml (buyer split, issue #321)") } - volumes, ok := nested(dep, "spec", "template", "spec", "volumes").([]any) + volumes, ok := nested(buyerDep, "spec", "template", "spec", "volumes").([]any) if !ok { - t.Fatal("litellm Deployment has no volumes") + t.Fatal("x402-buyer Deployment has no volumes") } var stateVolume map[string]any @@ -65,7 +70,7 @@ func TestBuyerStatePVC(t *testing.T) { } if stateVolume == nil { - t.Fatal("litellm Deployment missing 'x402-buyer-state' volume") + t.Fatal("x402-buyer Deployment missing 'x402-buyer-state' volume") } if _, isEmptyDir := stateVolume["emptyDir"]; isEmptyDir { @@ -81,45 +86,67 @@ func TestBuyerStatePVC(t *testing.T) { t.Errorf("persistentVolumeClaim.claimName = %v, want x402-buyer-state", claim) } - // Strategy must be Recreate so the new pod waits for the old pod to - // release the RWO PVC before mounting. RollingUpdate with maxSurge>0 - // would block indefinitely. - if strat := nested(dep, "spec", "strategy", "type"); strat != "Recreate" { - t.Errorf("litellm Deployment strategy.type = %v, want Recreate (RWO PVC cannot be co-mounted during surge)", strat) + // Buyer strategy must be Recreate: the new pod waits for the old pod to + // release the RWO PVC before mounting, and consumed-auth state must + // have exactly one writer (double-spend protection). + if strat := nested(buyerDep, "spec", "strategy", "type"); strat != "Recreate" { + t.Errorf("x402-buyer Deployment strategy.type = %v, want Recreate (RWO PVC + single-writer auth state)", strat) } - if policy := nested(dep, "spec", "template", "spec", "securityContext", "fsGroupChangePolicy"); policy != "OnRootMismatch" { - t.Errorf("litellm pod fsGroupChangePolicy = %v, want OnRootMismatch", policy) + if policy := nested(buyerDep, "spec", "template", "spec", "securityContext", "fsGroupChangePolicy"); policy != "OnRootMismatch" { + t.Errorf("x402-buyer pod fsGroupChangePolicy = %v, want OnRootMismatch", policy) } - // x402-buyer must keep container-level UID/GID 1000 while hostPath PVs - // from <= v0.10.0-rc12 clusters remain in support: those PVs ignore the - // pod fsGroup (kubelet skips ownership management on hostPath) and hold - // a consumed.json written 0600 by UID 1000 — a 65532 sidecar crashloops + // The buyer pod must keep UID/GID 1000 while hostPath PVs from + // <= v0.10.0-rc12 clusters remain in support: those PVs ignore fsGroup + // (kubelet skips ownership management on hostPath) and hold a + // consumed.json written 0600 by UID 1000 — a 65532 buyer crashloops // on `load state` and takes every paid/* route down. On fresh local-type - // PVs the explicit UID is harmless (fsGroup 65532 grants group access). + // PVs the explicit UID is harmless. + if u := nested(buyerDep, "spec", "template", "spec", "securityContext", "runAsUser"); u != 1000 { + t.Errorf("x402-buyer pod securityContext.runAsUser = %v, want 1000 (legacy hostPath-PV state compat)", u) + } + if g := nested(buyerDep, "spec", "template", "spec", "securityContext", "runAsGroup"); g != 1000 { + t.Errorf("x402-buyer pod securityContext.runAsGroup = %v, want 1000 (legacy hostPath-PV state compat)", g) + } + if nr := nested(buyerDep, "spec", "template", "spec", "securityContext", "runAsNonRoot"); nr != true { + t.Errorf("x402-buyer pod securityContext.runAsNonRoot = %v, want true (restricted PSS)", nr) + } + + // litellm must be stateless: no buyer container, no PVC volume, and + // RollingUpdate with maxUnavailable: 0 so rollouts never gap inference. + dep := findDocByName(docs, "Deployment", "litellm") + if dep == nil { + t.Fatal("litellm Deployment missing from llm.yaml") + } + if liteVols, ok := nested(dep, "spec", "template", "spec", "volumes").([]any); ok { + for _, v := range liteVols { + vm, ok := v.(map[string]any) + if !ok { + continue + } + if _, isPVC := vm["persistentVolumeClaim"]; isPVC { + t.Errorf("litellm Deployment mounts PVC volume %v — litellm must stay stateless for RollingUpdate maxUnavailable:0", vm["name"]) + } + } + } containers, ok := nested(dep, "spec", "template", "spec", "containers").([]any) if !ok { t.Fatal("litellm Deployment has no containers") } - var buyer map[string]any for _, c := range containers { cm, ok := c.(map[string]any) if ok && cm["name"] == "x402-buyer" { - buyer = cm - break + t.Error("x402-buyer container found in litellm Deployment — the buyer must run as its own Deployment (issue #321)") } } - if buyer == nil { - t.Fatal("x402-buyer container missing from litellm Deployment") - } - if u := nested(buyer, "securityContext", "runAsUser"); u != 1000 { - t.Errorf("x402-buyer securityContext.runAsUser = %v, want 1000 (legacy hostPath-PV state compat)", u) + if strat := nested(dep, "spec", "strategy", "type"); strat != "RollingUpdate" { + t.Errorf("litellm Deployment strategy.type = %v, want RollingUpdate (zero-downtime rollouts)", strat) } - if g := nested(buyer, "securityContext", "runAsGroup"); g != 1000 { - t.Errorf("x402-buyer securityContext.runAsGroup = %v, want 1000 (legacy hostPath-PV state compat)", g) + if mu := nested(dep, "spec", "strategy", "rollingUpdate", "maxUnavailable"); mu != 0 { + t.Errorf("litellm Deployment rollingUpdate.maxUnavailable = %v, want 0 (a new pod must be Ready before the old one terminates)", mu) } - if nr := nested(buyer, "securityContext", "runAsNonRoot"); nr != true { - t.Errorf("x402-buyer securityContext.runAsNonRoot = %v, want true (restricted PSS)", nr) + if ms := nested(dep, "spec", "strategy", "rollingUpdate", "maxSurge"); ms != 1 { + t.Errorf("litellm Deployment rollingUpdate.maxSurge = %v, want 1", ms) } } diff --git a/internal/embed/embed_pdb_test.go b/internal/embed/embed_pdb_test.go new file mode 100644 index 00000000..41180c1e --- /dev/null +++ b/internal/embed/embed_pdb_test.go @@ -0,0 +1,103 @@ +package embed + +import ( + "strings" + "testing" +) + +// TestPodDisruptionBudgets pins the PDB stance for the singleton services on +// user-facing paths. On multi-node clusters a `kubectl drain` would otherwise +// silently evict the only pod of the payment gate (x402-verifier → every +// /services/* route 5xxs) or the RPC front door (eRPC). minAvailable: 1 on a +// single replica deliberately blocks voluntary eviction until the operator +// deletes the pod explicitly — the same stance llm.yaml takes for litellm. +// +// Rollouts are NOT affected by PDBs; both deployments surge gaplessly via +// RollingUpdate maxUnavailable 0 (default rounding at 1 replica). +func TestPodDisruptionBudgets(t *testing.T) { + tests := []struct { + name string + file string + pdbName string + namespace string + wantSelector map[string]string + }{ + { + name: "x402-verifier payment gate", + file: "base/templates/x402.yaml", + pdbName: "x402-verifier", + namespace: "x402", + wantSelector: map[string]string{ + "app": "x402-verifier", + }, + }, + { + name: "erpc RPC front door", + file: "base/templates/erpc.yaml", + pdbName: "erpc", + namespace: "erpc", + // Must match the ethereum/erpc chart's selectorLabels, which the + // base chart does not control — verified against the live + // deployment's spec.selector.matchLabels. + wantSelector: map[string]string{ + "app.kubernetes.io/name": "erpc", + "app.kubernetes.io/instance": "erpc", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := ReadInfrastructureFile(tt.file) + if err != nil { + t.Fatalf("ReadInfrastructureFile(%s): %v", tt.file, err) + } + + pdb := findDocByName(multiDoc(data), "PodDisruptionBudget", tt.pdbName) + if pdb == nil { + t.Fatalf("PodDisruptionBudget %q missing from %s", tt.pdbName, tt.file) + } + if ns := nested(pdb, "metadata", "namespace"); ns != tt.namespace { + t.Errorf("namespace = %v, want %s", ns, tt.namespace) + } + if ma := nested(pdb, "spec", "minAvailable"); ma != 1 { + t.Errorf("minAvailable = %v, want 1", ma) + } + sel, ok := nested(pdb, "spec", "selector", "matchLabels").(map[string]any) + if !ok { + t.Fatal("PDB missing spec.selector.matchLabels") + } + for k, v := range tt.wantSelector { + if sel[k] != v { + t.Errorf("selector[%s] = %v, want %s", k, sel[k], v) + } + } + if len(sel) != len(tt.wantSelector) { + t.Errorf("selector has %d labels %v, want exactly %v — extra labels risk matching no pods", len(sel), sel, tt.wantSelector) + } + }) + } +} + +// TestCloudflaredPDBTemplate pins the pre-existing tunnel-connector PDB: it +// must stay gated on an active tunnel (local/remote mode) with >1 replica, +// where minAvailable: 1 lets drains roll one connector at a time without +// dropping the public path. +func TestCloudflaredPDBTemplate(t *testing.T) { + data, err := ReadInfrastructureFile("cloudflared/templates/pdb.yaml") + if err != nil { + t.Fatalf("ReadInfrastructureFile: %v", err) + } + raw := string(data) + + for _, want := range []string{ + "kind: PodDisruptionBudget", + "minAvailable: 1", + "app.kubernetes.io/name: cloudflared", + "gt $persistentReplicas 1", + } { + if !strings.Contains(raw, want) { + t.Errorf("cloudflared pdb.yaml missing %q", want) + } + } +} diff --git a/internal/embed/infrastructure/base/templates/erpc.yaml b/internal/embed/infrastructure/base/templates/erpc.yaml index 635665d3..e80f92ac 100644 --- a/internal/embed/infrastructure/base/templates/erpc.yaml +++ b/internal/embed/infrastructure/base/templates/erpc.yaml @@ -98,3 +98,24 @@ data: } } } + +--- +# PodDisruptionBudget: eRPC is the single RPC front door for agents and the +# frontend. The upstream ethereum/erpc chart (0.0.4) has no PDB template of +# its own (its `podDisruptionBudget` value is unwired), so it lives here with +# the other erpc-namespace resources. With one replica, minAvailable: 1 makes +# voluntary evictions (multi-node drains, upgrades) require explicit operator +# intent instead of silently gapping local RPC. Rollouts are unaffected — the +# chart's RollingUpdate at 1 replica surges 0-unavailable behind its +# readiness probe (`network add/remove` restarts stay gapless). +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: erpc + namespace: erpc +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: erpc + app.kubernetes.io/instance: erpc diff --git a/internal/embed/infrastructure/base/templates/llm.yaml b/internal/embed/infrastructure/base/templates/llm.yaml index 917d2bfd..cd50bdf2 100644 --- a/internal/embed/infrastructure/base/templates/llm.yaml +++ b/internal/embed/infrastructure/base/templates/llm.yaml @@ -23,7 +23,7 @@ metadata: name: llm labels: # Pod Security Standards: Restricted profile enforced at admission. - # The litellm pod (litellm + x402-buyer sidecar) runs as non-root with + # The litellm and x402-buyer pods run as non-root with # all caps dropped, seccomp=RuntimeDefault, and readOnlyRootFilesystem; # write paths are routed to named emptyDir mounts. pod-security.kubernetes.io/enforce: restricted @@ -82,7 +82,11 @@ data: - model_name: "paid/*" litellm_params: model: "openai/*" - api_base: "http://127.0.0.1:8402/v1" + # x402-buyer runs as its own Deployment/Service (not a sidecar) so + # LiteLLM stays stateless and can roll with maxUnavailable: 0. + # The /v1 suffix is required — LiteLLM's OpenAI provider does not + # append it (CLAUDE.md pitfall 6). + api_base: "http://x402-buyer.llm.svc.cluster.local:8402/v1" api_key: "unused" general_settings: master_key: os.environ/LITELLM_MASTER_KEY @@ -133,10 +137,6 @@ stringData: # facilitator's nonce protection until a manual buy.py process --all. # PVC backed by local-path (single-node k3d default storage class) # gives crash-safety without conversion to StatefulSet. -# -# Deployment strategy: Recreate — RWO PVC can't be mounted by two -# pods, so RollingUpdate's surge would block. Recreate accepts a -# brief gap during rollout (litellm is replicas:1 anyway). apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -158,16 +158,23 @@ metadata: labels: app: litellm spec: - # Keep a single LiteLLM + x402-buyer pod while buyer auth consumption state - # is local to the sidecar pod. Scale this back out only after consumed auth - # state is shared or auth pools are sharded per replica. + # Single replica is enough locally: with RollingUpdate maxUnavailable: 0 + # below, every rollout (image bump, secret rotation via Reloader) surges a + # new pod to Ready before the old one terminates — no inference gap. + # LiteLLM is stateless here (config is copied from the ConfigMap at pod + # start; x402-buyer runs as its own Deployment with the stateful PVC), so + # scaling out is safe if ever needed — but note the hot model APIs must + # then be fanned out per pod (the CLI already does; see litellmAPICall). replicas: 1 - # Recreate (not RollingUpdate) because the x402-buyer-state PVC is RWO and - # cannot be co-mounted by an overlapping new pod during surge. Litellm is - # replicas: 1 so this just trades the (currently maxSurge:1) overlap for a - # short gap during rollout — acceptable, and unavoidable with RWO storage. + # RollingUpdate with zero unavailability is the whole point of keeping + # this pod stateless (issue #321). The old Recreate strategy was forced by + # the buyer sidecar's RWO PVC, which now lives in the x402-buyer + # Deployment below. strategy: - type: Recreate + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 selector: matchLabels: app: litellm @@ -176,15 +183,23 @@ spec: labels: app: litellm annotations: - # litellm-config only: model-list changes must roll the pod so paid - # routes pick up new entries. The buyer ConfigMaps are deliberately - # NOT listed — x402-buyer hot-reloads them via /admin/reload (driven - # by the serviceoffer-controller on every buy/refill), and this - # Deployment is strategy Recreate with one replica: reloading on - # x402-buyer-config/x402-buyer-auths would take the whole inference - # gateway down on every purchase event (CLAUDE.md pitfall 7: restart - # is the fallback, not the default buy path). - configmap.reloader.stakater.com/reload: "litellm-config" + # litellm-secrets ONLY. Deliberately narrow: + # - litellm-config is NOT watched: every model_list change is + # hot-applied to the live router via /model/new + /model/delete by + # both the CLI (internal/model/model.go hotAddModels/hotDeleteModel) + # and the serviceoffer-controller (purchase_helpers.go). The + # ConfigMap is persistence for the next pod start, nothing more. + # Watching it here would roll the pod on every model add/remove/ + # prefer and on every first-time purchase — reintroducing the + # inference gap the hot APIs exist to remove (issue #321). + # - x402-buyer-config/x402-buyer-auths are NOT watched: x402-buyer + # hot-reloads them via /admin/reload driven by the controller on + # every buy/refill (CLAUDE.md pitfall 7: restart is the fallback, + # not the default buy path). + # - litellm-secrets IS watched: os.environ/ API-key references are + # resolved once at config load, so key changes genuinely need a + # pod replacement. With RollingUpdate maxUnavailable:0 below this + # is a gapless surge, not an outage. secret.reloader.stakater.com/reload: "litellm-secrets" spec: terminationGracePeriodSeconds: 60 @@ -300,9 +315,81 @@ spec: limits: cpu: "1" memory: 2Gi + volumes: + - name: litellm-config-source + configMap: + name: litellm-config + items: + - key: config.yaml + path: config.yaml + # Runtime copy of litellm-config. ConfigMap volumes are read-only, but + # LiteLLM's config-backed /model/new path persists to config.yaml after + # updating the live router. Keep the source ConfigMap immutable and give + # LiteLLM a pod-local writable working file instead. + - name: litellm-config-work + emptyDir: + sizeLimit: 1Mi + # Writable /tmp for Python tempfile / multipart uploads. Sized + # modestly — LiteLLM streams responses rather than buffering them. + - name: litellm-tmp + emptyDir: + sizeLimit: 128Mi + # Writable HOME for LiteLLM's pip/HF/XDG cache lookups so the + # container can run with readOnlyRootFilesystem=true. + - name: litellm-home + emptyDir: + sizeLimit: 256Mi + +--- +# x402-buyer as its own Deployment (was: sidecar in the litellm pod). +# Separating it makes LiteLLM stateless so LiteLLM can use RollingUpdate +# maxUnavailable: 0 (zero-downtime rollouts on image bumps and secret +# rotation, issue #321). The buyer keeps the RWO PVC and stays single +# replica + Recreate: consumed-auth state must have exactly one writer or +# pre-signed auths could be double-spent. Buyer rollouts briefly gap paid/* +# routes only — rare (buyer image bumps; buyer CM changes hot-reload via +# /admin/reload, driven by the serviceoffer-controller). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: x402-buyer + namespace: llm + labels: + app: x402-buyer +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: x402-buyer + template: + metadata: + labels: + app: x402-buyer + # No Reloader annotations: x402-buyer-config/x402-buyer-auths changes + # are hot-reloaded via /admin/reload (CLAUDE.md pitfall 7). + spec: + # PSS Restricted. UID/GID 1000 (not the 65532 distroless default): on + # fresh clusters the StorageClass provisions local PVs where kubelet + # applies fsGroup and any UID works, but clusters UPGRADED in place + # from <= v0.10.0-rc12 have a hostPath-typed x402-buyer-state PV + # (kubelet ignores fsGroup there) chowned 1000:1000 with consumed.json + # written 0600 by UID 1000 — a 65532 buyer cannot read it and + # crashloops on startup (`load state` Fatalf), taking every paid/* + # route down. Keep 1000 until hostPath PVs are out of support. + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: - name: x402-buyer # Pinned by sha256 digest (multi-arch manifest list, amd64+arm64) - # so the deployed sidecar is byte-for-byte identical across QA + # so the deployed buyer is byte-for-byte identical across QA # hosts. The :62f11c9 tag is preserved for human readability; the # digest is authoritative. # Previous tag-only pin allowed the local-build path to silently @@ -312,20 +399,7 @@ spec: # across flow-08/11/14/13. See internal/embed/embed_image_pin_test.go. image: ghcr.io/obolnetwork/x402-buyer:62f11c9@sha256:c58f7a20303538311d7e207e816e699c038bd8ded6339697fc0a01ed187abcda imagePullPolicy: IfNotPresent - # PSS Restricted + writable PVC. On fresh clusters the StorageClass - # asks local-path-provisioner for local PVs, so kubelet applies the - # pod-level fsGroup to /state on mount and any UID works. The - # explicit UID/GID 1000 below is for clusters UPGRADED in place - # from <= v0.10.0-rc12: their x402-buyer-state PV is hostPath-typed - # (kubelet ignores fsGroup there) and the old provisioner chowned - # the dir 1000:1000 with consumed.json written 0600 by UID 1000 — - # a 65532 sidecar cannot even read it and crashloops on startup - # (`load state` Fatalf), taking every paid/* route down. Keep 1000 - # until hostPath PVs are out of support. securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: @@ -368,29 +442,6 @@ spec: - name: x402-buyer-state mountPath: /state volumes: - - name: litellm-config-source - configMap: - name: litellm-config - items: - - key: config.yaml - path: config.yaml - # Runtime copy of litellm-config. ConfigMap volumes are read-only, but - # LiteLLM's config-backed /model/new path persists to config.yaml after - # updating the live router. Keep the source ConfigMap immutable and give - # LiteLLM a pod-local writable working file instead. - - name: litellm-config-work - emptyDir: - sizeLimit: 1Mi - # Writable /tmp for Python tempfile / multipart uploads. Sized - # modestly — LiteLLM streams responses rather than buffering them. - - name: litellm-tmp - emptyDir: - sizeLimit: 128Mi - # Writable HOME for LiteLLM's pip/HF/XDG cache lookups so the - # container can run with readOnlyRootFilesystem=true. - - name: litellm-home - emptyDir: - sizeLimit: 256Mi - name: buyer-config configMap: name: x402-buyer-config @@ -403,6 +454,30 @@ spec: persistentVolumeClaim: claimName: x402-buyer-state +--- +# Stable in-cluster address for the buyer: LiteLLM's paid/* routes point +# api_base at this Service, and agents' buy.py reaches /status through it. +# Exposure note: pod IP :8402 was already cluster-reachable when the buyer +# was a sidecar (no NetworkPolicy in this namespace); the Service changes +# addressing, not the security posture. Auth spend is bounded by the +# pre-signed pool (max loss = N x price). +apiVersion: v1 +kind: Service +metadata: + name: x402-buyer + namespace: llm + labels: + app: x402-buyer +spec: + type: ClusterIP + selector: + app: x402-buyer + ports: + - name: buyer-http + port: 8402 + targetPort: buyer-http + protocol: TCP + --- apiVersion: policy/v1 kind: PodDisruptionBudget @@ -435,15 +510,16 @@ spec: --- # Relocated from helmfile.yaml `llm-buyer-podmonitor` bedag/raw release. -# Lives alongside its workload (litellm + x402-buyer sidecar) instead of +# Lives alongside its workload (the x402-buyer Deployment) instead of # inlined in helmfile so the chart layout is the single source of truth # for what ships in the llm namespace. The PodMonitor CRD comes from the # monitoring release (kube-prometheus-stack), so `base` now declares a # `needs: [monitoring/monitoring]` in helmfile.yaml to guarantee CRD # presence before this template applies. # -# PodMonitor (not ServiceMonitor) because the sidecar listens on a per-pod -# port (8402) that is NOT exposed via the litellm Service. +# PodMonitor kept under its historical name (litellm-x402-buyer) so +# helmfile ordering notes and tests stay stable; it now targets the +# standalone x402-buyer pods (the buyer is no longer a litellm sidecar). # Picked up by kube-prometheus-stack via the `release: monitoring` label. apiVersion: monitoring.coreos.com/v1 kind: PodMonitor @@ -452,11 +528,11 @@ metadata: namespace: llm labels: release: monitoring - app: litellm + app: x402-buyer spec: selector: matchLabels: - app: litellm + app: x402-buyer podMetricsEndpoints: - port: buyer-http path: /metrics diff --git a/internal/embed/infrastructure/base/templates/local-path.yaml b/internal/embed/infrastructure/base/templates/local-path.yaml index b7ed8408..1f857ff0 100644 --- a/internal/embed/infrastructure/base/templates/local-path.yaml +++ b/internal/embed/infrastructure/base/templates/local-path.yaml @@ -121,6 +121,13 @@ spec: - rancher.io/local-path - --helper-image - rancher/mirrored-library-busybox:1.36.1 + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi volumeMounts: - name: config-volume mountPath: /etc/config/ diff --git a/internal/embed/infrastructure/base/templates/x402.yaml b/internal/embed/infrastructure/base/templates/x402.yaml index a1504973..6a053213 100644 --- a/internal/embed/infrastructure/base/templates/x402.yaml +++ b/internal/embed/infrastructure/base/templates/x402.yaml @@ -411,6 +411,26 @@ spec: targetPort: http protocol: TCP +--- +# PodDisruptionBudget: the verifier is the ForwardAuth gate for every paid +# route — evicting its only pod takes all /services/* traffic down. With the +# deliberate single replica (see the Deployment comment on per-pod metric +# registries), minAvailable: 1 makes voluntary evictions (multi-node drains, +# upgrades) require explicit operator intent instead of silently gapping the +# payment path. Same stance as the litellm PDB in llm.yaml. Rollouts are +# unaffected — the default RollingUpdate surge at 1 replica is 0-unavailable, +# gated by /readyz (config + routes loaded). +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: x402-verifier + namespace: x402 +spec: + minAvailable: 1 + selector: + matchLabels: + app: x402-verifier + --- # ServiceMonitor for x402-verifier — scrapes the stable Service endpoint # rather than per-pod IPs (which is what a PodMonitor would do). Lives diff --git a/internal/embed/infrastructure/cloudflared/values.yaml b/internal/embed/infrastructure/cloudflared/values.yaml index 22b5db60..2777a434 100644 --- a/internal/embed/infrastructure/cloudflared/values.yaml +++ b/internal/embed/infrastructure/cloudflared/values.yaml @@ -14,7 +14,7 @@ quickTunnel: url: "http://traefik.traefik.svc.cluster.local:80" persistent: - replicaCount: 2 + replicaCount: 1 management: configMapName: "cloudflared-management" diff --git a/internal/embed/infrastructure/helmfile.yaml b/internal/embed/infrastructure/helmfile.yaml index f4d9c828..f9440a6b 100644 --- a/internal/embed/infrastructure/helmfile.yaml +++ b/internal/embed/infrastructure/helmfile.yaml @@ -150,6 +150,16 @@ releases: createNamespace: true chart: stakater/reloader version: 2.2.7 + values: + - reloader: + deployment: + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 100m + memory: 256Mi # eRPC - name: erpc diff --git a/internal/embed/infrastructure/values/obol-frontend.yaml.gotmpl b/internal/embed/infrastructure/values/obol-frontend.yaml.gotmpl index e46a51f5..35006b99 100644 --- a/internal/embed/infrastructure/values/obol-frontend.yaml.gotmpl +++ b/internal/embed/infrastructure/values/obol-frontend.yaml.gotmpl @@ -53,6 +53,14 @@ service: type: ClusterIP port: 3000 +resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + podSecurityContext: fsGroup: 1001 diff --git a/internal/embed/skills/buy-x402/SKILL.md b/internal/embed/skills/buy-x402/SKILL.md index 71a9e8de..cda04167 100644 --- a/internal/embed/skills/buy-x402/SKILL.md +++ b/internal/embed/skills/buy-x402/SKILL.md @@ -357,9 +357,9 @@ flowchart LR 4. **Reconcile**: The controller validates pricing, writes per-upstream buyer config/auth files into the `x402-buyer-config` and `x402-buyer-auths` ConfigMaps in `llm`, and keeps the paid model route available in LiteLLM. -5. **Runtime mount**: A lean Go sidecar (`x402-buyer`) already runs inside the existing `litellm` pod in the `llm` namespace. It mounts both ConfigMaps and serves as an OpenAI-compatible reverse proxy on `127.0.0.1:8402`. +5. **Runtime mount**: A lean Go service (`x402-buyer`) runs as its own Deployment in the `llm` namespace. It mounts both ConfigMaps and serves as an OpenAI-compatible reverse proxy at `x402-buyer.llm.svc.cluster.local:8402`. -6. **Wire**: LiteLLM keeps one static wildcard route: `paid/* -> openai/* -> 127.0.0.1:8402/v1`. The controller also adds explicit paid-model entries when required so models with colons resolve reliably. The public model name is always `paid/`. +6. **Wire**: LiteLLM keeps one static wildcard route: `paid/* -> openai/* -> http://x402-buyer.llm.svc.cluster.local:8402/v1`. The controller also adds explicit paid-model entries when required so models with colons resolve reliably. The public model name is always `paid/`. 7. **Runtime**: On each request through the sidecar: - Sidecar forwards to upstream seller diff --git a/internal/embed/skills/buy-x402/references/x402-buyer-api.md b/internal/embed/skills/buy-x402/references/x402-buyer-api.md index 2f705ad5..37bec510 100644 --- a/internal/embed/skills/buy-x402/references/x402-buyer-api.md +++ b/internal/embed/skills/buy-x402/references/x402-buyer-api.md @@ -228,12 +228,12 @@ model_list: - model_name: "paid/*" litellm_params: model: "openai/*" - api_base: "http://127.0.0.1:8402/v1" + api_base: "http://x402-buyer.llm.svc.cluster.local:8402/v1" api_key: "unused" - model_name: "paid/qwen3.5:9b" litellm_params: model: "openai/paid/qwen3.5:9b" - api_base: "http://127.0.0.1:8402/v1" + api_base: "http://x402-buyer.llm.svc.cluster.local:8402/v1" api_key: "unused" ``` diff --git a/internal/embed/skills/buy-x402/scripts/buy.py b/internal/embed/skills/buy-x402/scripts/buy.py index 8b0b571f..dc1d6cf0 100644 --- a/internal/embed/skills/buy-x402/scripts/buy.py +++ b/internal/embed/skills/buy-x402/scripts/buy.py @@ -62,7 +62,10 @@ DEFAULT_CHAIN = os.environ.get("ERPC_NETWORK", "base-sepolia") BUYER_NS = "llm" -LITELLM_DEPLOY = "litellm" +# x402-buyer runs as its own Deployment (label app=x402-buyer) since the +# litellm pod went stateless for zero-downtime rollouts; it is no longer a +# sidecar in the litellm pod. +BUYER_DEPLOY = "x402-buyer" BUYER_PORT = 8402 # Some sellers sit behind a Cloudflare WAF that blocks the default @@ -335,13 +338,13 @@ def _parse_money_amount(value, asset, extra=None): # --------------------------------------------------------------------------- -# Buyer sidecar status helpers +# Buyer status helpers # --------------------------------------------------------------------------- -def _get_litellm_pod(token, ssl_ctx): - """Return the current LiteLLM pod object, or None if unavailable.""" +def _get_buyer_pod(token, ssl_ctx): + """Return the current x402-buyer pod object, or None if unavailable.""" pods = api_get( - f"/api/v1/namespaces/{BUYER_NS}/pods?labelSelector=app={LITELLM_DEPLOY}", + f"/api/v1/namespaces/{BUYER_NS}/pods?labelSelector=app={BUYER_DEPLOY}", token, ssl_ctx, ) for item in pods.get("items", []): @@ -353,10 +356,10 @@ def _get_litellm_pod(token, ssl_ctx): def _buyer_status(): - """Return live sidecar status JSON, or None if the sidecar is unavailable.""" + """Return live buyer status JSON, or None if the buyer is unavailable.""" token, _ = load_sa() ssl_ctx = make_ssl_context() - pod = _get_litellm_pod(token, ssl_ctx) + pod = _get_buyer_pod(token, ssl_ctx) if not pod: return None @@ -2344,11 +2347,11 @@ def cmd_status(name): print(f"Auths spent: {live_status.get('spent', status.get('spent', 0))}") print() - pod = _get_litellm_pod(token, ssl_ctx) + pod = _get_buyer_pod(token, ssl_ctx) if not pod: - print("Sidecar: NOT RUNNING (LiteLLM pod unavailable)") + print("Buyer: NOT RUNNING (x402-buyer pod unavailable)") else: - print(f"Sidecar: {pod.get('status', {}).get('phase', 'Unknown')}") + print(f"Buyer: {pod.get('status', {}).get('phase', 'Unknown')}") # --------------------------------------------------------------------------- diff --git a/internal/model/drift.go b/internal/model/drift.go new file mode 100644 index 00000000..a063df26 --- /dev/null +++ b/internal/model/drift.go @@ -0,0 +1,147 @@ +package model + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ObolNetwork/obol-stack/internal/config" + "github.com/ObolNetwork/obol-stack/internal/kubectl" + "gopkg.in/yaml.v3" +) + +// RouterDrift describes a divergence between the litellm-config ConfigMap +// (the persistence source of truth) and the live LiteLLM router. +// +// Since Reloader no longer watches litellm-config (issue #321: a +// ConfigMap-triggered rollout would gap inference on every model change), +// hot-add/hot-delete API calls are the only thing keeping the live router in +// sync between pod restarts. This check is the replacement safety net: it +// makes a silently-failed hot call visible in `obol model status` instead of +// leaving the operator with a router that quietly disagrees with the config. +type RouterDrift struct { + // Missing entries exist in the ConfigMap model_list but not in the live + // router — a hot-add failed or never ran. They will appear after the next + // pod restart; until then requests to them 404. + Missing []string `json:"missing,omitempty"` + // Extra entries are served by the live router but absent from the + // ConfigMap — a hot-delete failed or never ran. They disappear on the + // next pod restart. + Extra []string `json:"extra,omitempty"` +} + +// Empty reports whether the live router and the ConfigMap agree. +func (d RouterDrift) Empty() bool { + return len(d.Missing) == 0 && len(d.Extra) == 0 +} + +// DiffRouterModels is the pure core of the drift check. configured is the +// raw model_name list from the ConfigMap; live is the model id list reported +// by the running router (/v1/models). +// +// Wildcard entries (trailing "/*") are configuration for LiteLLM's router, +// not concrete serving targets: a configured wildcard is never "missing", +// and any live model matched by a configured wildcard prefix is not "extra". +func DiffRouterModels(configured, live []string) RouterDrift { + liveSet := make(map[string]struct{}, len(live)) + for _, m := range live { + liveSet[m] = struct{}{} + } + + var wildcardPrefixes []string + configuredSet := make(map[string]struct{}, len(configured)) + var drift RouterDrift + + for _, name := range configured { + if prefix, ok := strings.CutSuffix(name, "*"); ok { + wildcardPrefixes = append(wildcardPrefixes, prefix) + configuredSet[name] = struct{}{} + continue + } + configuredSet[name] = struct{}{} + if _, ok := liveSet[name]; !ok { + drift.Missing = append(drift.Missing, name) + } + } + + for _, name := range live { + if _, ok := configuredSet[name]; ok { + continue + } + if strings.HasSuffix(name, "*") { + // The live router lists wildcard groups verbatim; they are + // router config, not drift. + continue + } + matched := false + for _, prefix := range wildcardPrefixes { + if strings.HasPrefix(name, prefix) { + matched = true + break + } + } + if !matched { + drift.Extra = append(drift.Extra, name) + } + } + + return drift +} + +// CheckRouterDrift compares the litellm-config ConfigMap model_list against +// the live router's /v1/models. It returns an error when either side cannot +// be read (cluster down, pod not running) — callers should treat that as +// "check unavailable", not as drift. +func CheckRouterDrift(cfg *config.Config) (RouterDrift, error) { + kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + + if _, err := os.Stat(kubeconfigPath); os.IsNotExist(err) { + return RouterDrift{}, errors.New("cluster not running") + } + + raw, err := kubectl.Output(kubectlBinary, kubeconfigPath, + "get", "configmap", configMapName, "-n", namespace, "-o", "jsonpath={.data.config\\.yaml}") + if err != nil { + return RouterDrift{}, fmt.Errorf("read LiteLLM config: %w", err) + } + + var litellmConfig LiteLLMConfig + if err := yaml.Unmarshal([]byte(raw), &litellmConfig); err != nil { + return RouterDrift{}, fmt.Errorf("parse LiteLLM config: %w", err) + } + + configured := make([]string, 0, len(litellmConfig.ModelList)) + for _, entry := range litellmConfig.ModelList { + configured = append(configured, entry.ModelName) + } + + masterKey, err := GetMasterKey(cfg) + if err != nil { + return RouterDrift{}, fmt.Errorf("read master key: %w", err) + } + + body, err := litellmGETViaPortForward(kubectlBinary, kubeconfigPath, masterKey, "/v1/models") + if err != nil { + return RouterDrift{}, fmt.Errorf("query live router: %w", err) + } + + var resp struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return RouterDrift{}, fmt.Errorf("parse /v1/models response: %w", err) + } + + live := make([]string, 0, len(resp.Data)) + for _, m := range resp.Data { + live = append(live, m.ID) + } + + return DiffRouterModels(configured, live), nil +} diff --git a/internal/model/drift_test.go b/internal/model/drift_test.go new file mode 100644 index 00000000..aaeab1e7 --- /dev/null +++ b/internal/model/drift_test.go @@ -0,0 +1,89 @@ +package model + +import ( + "reflect" + "testing" +) + +// DiffRouterModels is the drift safety net that replaced the Reloader +// annotation on litellm-config (issue #321): with hot-add/hot-delete as the +// only live-apply path, a silently-failed hot call must surface as drift in +// `obol model status` instead of a router that quietly disagrees with the +// ConfigMap. +func TestDiffRouterModels(t *testing.T) { + tests := []struct { + name string + configured []string + live []string + wantMissing []string + wantExtra []string + }{ + { + name: "in sync", + configured: []string{"anthropic/claude-sonnet-5", "qwen3.5:9b"}, + live: []string{"anthropic/claude-sonnet-5", "qwen3.5:9b"}, + }, + { + name: "hot-add failed: configured model absent from router", + configured: []string{"qwen3.5:9b", "qwen3.5:4b"}, + live: []string{"qwen3.5:9b"}, + wantMissing: []string{"qwen3.5:4b"}, + }, + { + name: "hot-delete failed: removed model still served", + configured: []string{"qwen3.5:9b"}, + live: []string{"qwen3.5:9b", "qwen3.5:4b"}, + wantExtra: []string{"qwen3.5:4b"}, + }, + { + name: "configured wildcard is never missing", + configured: []string{"paid/*"}, + live: []string{}, + }, + { + name: "live model matched by configured wildcard is not extra", + configured: []string{"paid/*"}, + live: []string{"paid/qwen36-deep"}, + }, + { + name: "live wildcard group listed verbatim is not extra", + configured: []string{"paid/*"}, + live: []string{"paid/*"}, + }, + { + name: "wildcard does not cover other prefixes", + configured: []string{"paid/*", "qwen3.5:9b"}, + live: []string{"paid/qwen36-deep", "stale-model"}, + wantMissing: []string{"qwen3.5:9b"}, + wantExtra: []string{"stale-model"}, + }, + { + name: "both empty", + configured: nil, + live: nil, + }, + { + name: "empty config with live models", + live: []string{"orphan"}, + wantExtra: []string{"orphan"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := DiffRouterModels(tt.configured, tt.live) + + if !reflect.DeepEqual(got.Missing, tt.wantMissing) { + t.Errorf("Missing = %v; want %v", got.Missing, tt.wantMissing) + } + if !reflect.DeepEqual(got.Extra, tt.wantExtra) { + t.Errorf("Extra = %v; want %v", got.Extra, tt.wantExtra) + } + + wantEmpty := len(tt.wantMissing) == 0 && len(tt.wantExtra) == 0 + if got.Empty() != wantEmpty { + t.Errorf("Empty() = %v; want %v", got.Empty(), wantEmpty) + } + }) + } +} diff --git a/internal/model/model.go b/internal/model/model.go index de2737ef..553c5347 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -466,6 +466,10 @@ func PatchLiteLLMEntries(cfg *config.Config, u *ui.UI, entries []ModelEntry) err } // RestartLiteLLM restarts the LiteLLM deployment and waits for rollout. +// A rollout that does not converge within the timeout is an error — callers +// must not report success while LiteLLM may be down or serving stale config +// (issue #321: the old warn-and-succeed behavior silently left LiteLLM +// broken after failed rollouts). func RestartLiteLLM(cfg *config.Config, u *ui.UI, provider string) error { kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") @@ -480,12 +484,10 @@ func RestartLiteLLM(cfg *config.Config, u *ui.UI, provider string) error { if err := kubectl.Run(kubectlBinary, kubeconfigPath, "rollout", "status", "deployment/"+deployName, "-n", namespace, "--timeout=90s"); err != nil { - u.Warnf("LiteLLM rollout not confirmed: %v", err) - u.Print("The deployment may still be rolling out.") - } else { - u.Successf("LiteLLM configured with %s provider", provider) + return fmt.Errorf("LiteLLM rollout not confirmed within 90s: %w (check `obol kubectl get pods -n %s`)", err, namespace) } + u.Successf("LiteLLM configured with %s provider", provider) return nil } @@ -838,10 +840,10 @@ func reorderModelList(entries []ModelEntry, names []string) ([]ModelEntry, bool, // Returns an error if any of the requested names is not present in the // current model_list — typos should be loud, not silent no-ops. // -// LiteLLM has no model_list reorder API, so after the ConfigMap patch this -// rolls the LiteLLM Deployment so the new order takes effect (the -// /v1/models listing follows model_list order, and hermes/openclaw read -// the ConfigMap directly via GetConfiguredModels for the agent primary). +// ConfigMap-only operation: hermes/openclaw read the agent primary from the +// ConfigMap via GetConfiguredModels, and LiteLLM's router does not use +// model_list order for routing, so no pod restart is needed. The live +// /v1/models listing keeps the old order until the next natural restart. func PreferModels(cfg *config.Config, u *ui.UI, names []string) error { if len(names) == 0 { return errors.New("at least one model name is required") @@ -892,15 +894,12 @@ func PreferModels(cfg *config.Config, u *ui.UI, names []string) error { return fmt.Errorf("failed to patch ConfigMap: %w", err) } - // LiteLLM has no reorder API; restart the deployment so the new order - // takes effect (mostly cosmetic for /v1/models listings — agent primary - // is read from the ConfigMap directly via GetConfiguredModels, which is - // already correct after the patch above). - if err := RestartLiteLLM(cfg, u, "prefer"); err != nil { - u.Warnf("LiteLLM rollout failed: %v", err) - u.Dim(" The ConfigMap is updated; agent will pick up the new primary on next sync.") - } - + // No LiteLLM restart: model_list order is an obol convention, not a + // LiteLLM routing input. The agent primary is read from the ConfigMap + // directly via GetConfiguredModels, which is already correct after the + // patch above. The live router's /v1/models listing keeps the old order + // until the next natural pod replacement — cosmetic only, and not worth + // an inference gap (issue #321). return nil } diff --git a/internal/model/record.go b/internal/model/record.go index 0f0b71f9..f9d1d360 100644 --- a/internal/model/record.go +++ b/internal/model/record.go @@ -130,9 +130,15 @@ func ReconcileRecorded(cfg *config.Config, u *ui.UI) { u.Warnf("LiteLLM restart after model reconcile failed: %v", err) } case configChanged: - // Reloader restarts LiteLLM on litellm-config changes since rc14; - // no manual rollout needed for a ConfigMap-only change. + // Reloader deliberately does NOT watch litellm-config (issue #321: + // a CM-triggered rollout would gap inference on every model change), + // so a ConfigMap-only reconcile must roll the pod explicitly for the + // live router to pick up the recorded model_list. With RollingUpdate + // maxUnavailable:0 on the litellm Deployment this is a gapless surge. u.Infof("Restored recorded model list (%d models)", len(state.ModelList)) + if err := RestartLiteLLM(cfg, u, "recorded model config"); err != nil { + u.Warnf("LiteLLM restart after model reconcile failed: %v", err) + } } } diff --git a/internal/openclaw/wallet.go b/internal/openclaw/wallet.go index af096a73..0320bb2f 100644 --- a/internal/openclaw/wallet.go +++ b/internal/openclaw/wallet.go @@ -463,6 +463,14 @@ func generateRemoteSignerValues(wallet *WalletInfo) string { keystorePassword: value: %q +# The signer is a singleton over a ReadWriteOnce keystore PVC: a RollingUpdate +# surge pod can wedge on the volume attach on multi-node clusters and briefly +# double-runs the signer over the same keystore on any cluster. Recreate is the +# chart default as of remote-signer 0.4.0; pinned explicitly here so the intent +# is visible and robust to a future chart-default change. +strategy: + type: Recreate + persistence: enabled: true size: 100Mi diff --git a/internal/schemas/service-catalog.schema.json b/internal/schemas/service-catalog.schema.json index a7b0f046..2d1d1320 100644 --- a/internal/schemas/service-catalog.schema.json +++ b/internal/schemas/service-catalog.schema.json @@ -6,12 +6,17 @@ "type": "object", "additionalProperties": false, "required": [ + "schemaVersion", "displayName", "tagline", "logoUrl", "services" ], "properties": { + "schemaVersion": { + "const": "1", + "description": "Envelope wire version. Bumped only on breaking changes; additive fields do not bump it." + }, "displayName": { "type": "string", "minLength": 1 diff --git a/internal/schemas/service_catalog.go b/internal/schemas/service_catalog.go index 1fb21a43..b48a337d 100644 --- a/internal/schemas/service_catalog.go +++ b/internal/schemas/service_catalog.go @@ -137,10 +137,20 @@ type StorefrontProfile struct { ContactEmail string `json:"contactEmail,omitempty"` } +// ServiceCatalogSchemaVersion is the current version of the +// /api/services.json envelope. Bump only on a breaking change to the wire +// shape; additive fields do not bump the version. Integrators/indexers use +// it to detect incompatible catalogs instead of guessing from field shapes. +const ServiceCatalogSchemaVersion = "1" + // ServiceCatalog is the public /api/services.json envelope. type ServiceCatalog struct { - DisplayName string `json:"displayName"` - Tagline string `json:"tagline"` - LogoURL string `json:"logoUrl"` - Services []ServiceCatalogEntry `json:"services"` + // SchemaVersion identifies the envelope wire version (currently "1"). + // Always populated by the controller; consumers should treat an absent + // field as a legacy pre-versioning catalog. + SchemaVersion string `json:"schemaVersion"` + DisplayName string `json:"displayName"` + Tagline string `json:"tagline"` + LogoURL string `json:"logoUrl"` + Services []ServiceCatalogEntry `json:"services"` } diff --git a/internal/serviceoffercontroller/controller.go b/internal/serviceoffercontroller/controller.go index f94f7c6b..48145f8b 100644 --- a/internal/serviceoffercontroller/controller.go +++ b/internal/serviceoffercontroller/controller.go @@ -248,6 +248,11 @@ func (c *Controller) Run(ctx context.Context, workers int) error { log.Printf("serviceoffer-controller: ensure default AgentIdentity: %v", err) } + // Heal paid-route entries written before the x402-buyer split (they + // point at the removed litellm-pod sidecar address). One-shot, + // idempotent, no-op on fresh clusters. + c.migrateLegacyBuyerAPIBases(ctx, "llm") + if workers < 1 { workers = 1 } @@ -1231,6 +1236,11 @@ func (c *Controller) reconcileSkillCatalog(ctx context.Context, override *moneti if err := c.applyObject(ctx, c.services.Namespace(skillCatalogNamespace), buildSkillCatalogService()); err != nil { return err } + // Headers Middleware must exist before the routes that reference it, or + // Traefik drops the routes for a dangling ExtensionRef. + if err := c.applyObject(ctx, c.middlewares.Namespace(skillCatalogNamespace), buildCatalogHeadersMiddleware()); err != nil { + return err + } if err := c.applyObject(ctx, c.httpRoutes.Namespace(skillCatalogNamespace), buildSkillCatalogHTTPRoute()); err != nil { return err } diff --git a/internal/serviceoffercontroller/purchase_helpers.go b/internal/serviceoffercontroller/purchase_helpers.go index 7e8b0491..0bdf422e 100644 --- a/internal/serviceoffercontroller/purchase_helpers.go +++ b/internal/serviceoffercontroller/purchase_helpers.go @@ -59,9 +59,9 @@ func (c *Controller) getLiteLLMMasterKey(ctx context.Context, ns string) (string } // hotAddLiteLLMModel adds a model via the LiteLLM /model/new HTTP API. The -// in-memory router is updated without a pod restart, preserving the x402-buyer -// sidecar's consumed-auth state (which lives in a pod-local emptyDir and would -// be wiped by a rollout). +// in-memory router is updated without a pod restart, so paid routes appear +// with zero inference downtime (issue #321 — Reloader deliberately does not +// watch litellm-config, making this hot call the primary apply path). // // Returns an error if the API call fails; callers fall back to a pod restart // only as a last resort — see addLiteLLMModelEntry. @@ -275,14 +275,33 @@ func (c *Controller) removeBuyerUpstream(ctx context.Context, ns, name string) { } } +// buyerAPIBase returns the LiteLLM api_base for paid routes: the standalone +// x402-buyer Deployment's Service. The /v1 suffix is required — LiteLLM's +// OpenAI provider does not append it (CLAUDE.md pitfall 6). +func buyerAPIBase(ns string) string { + return fmt.Sprintf("http://x402-buyer.%s.svc.cluster.local:8402/v1", ns) +} + +// legacyBuyerAPIBase is the pre-split address from when x402-buyer ran as a +// sidecar in the litellm pod. Entries still carrying it are dead upstreams +// (nothing listens on the litellm pod's localhost:8402 anymore) and are +// migrated in place by addLiteLLMModelEntry and +// migrateLegacyBuyerAPIBases. +const legacyBuyerAPIBase = "http://127.0.0.1:8402/v1" + // addLiteLLMModelEntry adds a paid/ route to LiteLLM. Writes the // ConfigMap (persistence across restarts) and then hot-adds via /model/new -// (no pod restart — preserves the buyer sidecar's consumed-auth state). +// (no pod restart — paid routes appear with zero inference downtime). +// +// If the hot-add API call fails, we do NOT fall back to a pod restart. +// Instead we log and rely on the ConfigMap being reloaded on the next +// natural rollout (litellm rolls gaplessly via RollingUpdate +// maxUnavailable: 0, but a restart here would still churn in-flight +// requests for no reason). // -// If the hot-add API call fails, we do NOT fall back to a pod restart: that -// would wipe the sidecar's emptyDir /state/consumed.json and cause the -// facilitator to reject previously-spent auths as double-spends. Instead we -// log and rely on the ConfigMap being reloaded on the next natural restart. +// An existing entry that still points at the legacy sidecar address is +// migrated to the x402-buyer Service in place (ConfigMap + hot +// delete/re-add) — it was a dead upstream anyway. func (c *Controller) addLiteLLMModelEntry(ctx context.Context, ns, modelName string) { cm, err := c.kubeClient.CoreV1().ConfigMaps(ns).Get(ctx, "litellm-config", metav1.GetOptions{}) if err != nil { @@ -299,20 +318,43 @@ func (c *Controller) addLiteLLMModelEntry(ctx context.Context, ns, modelName str return } - for _, entry := range cfg.ModelList { - if entry.ModelName == modelName { - return - } - } - entry := model.ModelEntry{ ModelName: modelName, LiteLLMParams: model.LiteLLMParams{ Model: "openai/" + modelName, - APIBase: "http://127.0.0.1:8402/v1", + APIBase: buyerAPIBase(ns), APIKey: "unused", }, } + + for i := range cfg.ModelList { + if cfg.ModelList[i].ModelName != modelName { + continue + } + if cfg.ModelList[i].LiteLLMParams.APIBase != legacyBuyerAPIBase { + return + } + // Legacy sidecar address: rewrite in place, persist, hot-sync. + cfg.ModelList[i].LiteLLMParams.APIBase = buyerAPIBase(ns) + rendered, err := yaml.Marshal(&cfg) + if err != nil { + log.Printf("purchase: failed to serialize litellm-config: %v", err) + return + } + cm.Data["config.yaml"] = string(rendered) + if _, err := c.kubeClient.CoreV1().ConfigMaps(ns).Update(ctx, cm, metav1.UpdateOptions{FieldManager: litellmConfigFieldManager}); err != nil { + log.Printf("purchase: failed to update litellm-config: %v", err) + return + } + if err := c.hotDeleteLiteLLMModel(ctx, ns, modelName); err != nil { + log.Printf("purchase: migrate %s: hot-delete failed: %v", modelName, err) + } + if err := c.hotAddLiteLLMModel(ctx, ns, cfg.ModelList[i]); err != nil { + log.Printf("purchase: migrate %s: hot-add failed: %v; relying on ConfigMap reload", modelName, err) + } + return + } + cfg.ModelList = append(cfg.ModelList, entry) rendered, err := yaml.Marshal(&cfg) @@ -443,12 +485,72 @@ func (c *Controller) removeLiteLLMModelEntry(ctx context.Context, ns, modelName } } -// triggerBuyerReload sends POST /admin/reload to the x402-buyer sidecar -// on all running litellm pods. Best-effort — the sidecar reloads on its -// own 5-second timer anyway. +// migrateLegacyBuyerAPIBases rewrites model_list entries that still point at +// the pre-split buyer sidecar address (127.0.0.1:8402 inside the litellm +// pod) to the standalone x402-buyer Service. One-shot at controller startup: +// this is what heals clusters upgraded in place, whose litellm-config still +// carries paid/ entries written before the buyer split. Those entries +// are dead upstreams until migrated, so the delete/re-add window is not a +// regression. Best-effort: failures are logged and the ConfigMap rewrite +// alone still fixes the next LiteLLM rollout. +func (c *Controller) migrateLegacyBuyerAPIBases(ctx context.Context, ns string) { + cm, err := c.kubeClient.CoreV1().ConfigMaps(ns).Get(ctx, "litellm-config", metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + log.Printf("buyer-migrate: read litellm-config: %v", err) + } + return + } + + var cfg model.LiteLLMConfig + if err := yaml.Unmarshal([]byte(cm.Data["config.yaml"]), &cfg); err != nil { + log.Printf("buyer-migrate: parse litellm-config: %v", err) + return + } + + var migrated []model.ModelEntry + for i := range cfg.ModelList { + if cfg.ModelList[i].LiteLLMParams.APIBase != legacyBuyerAPIBase { + continue + } + cfg.ModelList[i].LiteLLMParams.APIBase = buyerAPIBase(ns) + migrated = append(migrated, cfg.ModelList[i]) + } + if len(migrated) == 0 { + return + } + + rendered, err := yaml.Marshal(&cfg) + if err != nil { + log.Printf("buyer-migrate: serialize litellm-config: %v", err) + return + } + if cm.Data == nil { + cm.Data = make(map[string]string) + } + cm.Data["config.yaml"] = string(rendered) + if _, err := c.kubeClient.CoreV1().ConfigMaps(ns).Update(ctx, cm, metav1.UpdateOptions{FieldManager: litellmConfigFieldManager}); err != nil { + log.Printf("buyer-migrate: update litellm-config: %v", err) + return + } + + for _, entry := range migrated { + if err := c.hotDeleteLiteLLMModel(ctx, ns, entry.ModelName); err != nil { + log.Printf("buyer-migrate: hot-delete %s: %v", entry.ModelName, err) + } + if err := c.hotAddLiteLLMModel(ctx, ns, entry); err != nil { + log.Printf("buyer-migrate: hot-add %s: %v; entry loads on next LiteLLM rollout", entry.ModelName, err) + } + } + log.Printf("buyer-migrate: rewrote %d legacy buyer api_base entries in %s/litellm-config", len(migrated), ns) +} + +// triggerBuyerReload sends POST /admin/reload to all running x402-buyer +// pods (a standalone Deployment since the litellm pod went stateless). +// Best-effort — the buyer reloads on its own 5-second timer anyway. func (c *Controller) triggerBuyerReload(ctx context.Context, ns string) { pods, err := c.kubeClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ - LabelSelector: "app=litellm", + LabelSelector: "app=x402-buyer", }) if err != nil || len(pods.Items) == 0 { return @@ -471,7 +573,7 @@ func (c *Controller) triggerBuyerReload(ctx context.Context, ns string) { func (c *Controller) triggerBuyerRemove(ctx context.Context, ns, name string) { pods, err := c.kubeClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ - LabelSelector: "app=litellm", + LabelSelector: "app=x402-buyer", }) if err != nil || len(pods.Items) == 0 { return @@ -495,12 +597,12 @@ func (c *Controller) triggerBuyerRemove(ctx context.Context, ns, name string) { // ── Sidecar status check ──────────────────────────────────────────────────── func (c *Controller) checkBuyerStatus(ctx context.Context, ns, name string) (remaining, spent int, err error) { - // List LiteLLM pods to get a pod IP. + // List x402-buyer pods to get a pod IP. pods, err := c.kubeClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ - LabelSelector: "app=litellm", + LabelSelector: "app=x402-buyer", }) if err != nil || len(pods.Items) == 0 { - return 0, 0, fmt.Errorf("no litellm pods in %s", ns) + return 0, 0, fmt.Errorf("no x402-buyer pods in %s", ns) } for _, pod := range pods.Items { diff --git a/internal/serviceoffercontroller/purchase_helpers_test.go b/internal/serviceoffercontroller/purchase_helpers_test.go index 21de7deb..c1732640 100644 --- a/internal/serviceoffercontroller/purchase_helpers_test.go +++ b/internal/serviceoffercontroller/purchase_helpers_test.go @@ -114,8 +114,8 @@ func TestAddLiteLLMModelEntryUpdatesConfigMapAndHotAdds(t *testing.T) { if entry.LiteLLMParams.Model != "openai/paid/qwen3.5:9b" { t.Fatalf("litellm_params.model = %q", entry.LiteLLMParams.Model) } - if entry.LiteLLMParams.APIBase != "http://127.0.0.1:8402/v1" { - t.Fatalf("litellm_params.api_base = %q", entry.LiteLLMParams.APIBase) + if entry.LiteLLMParams.APIBase != "http://x402-buyer.llm.svc.cluster.local:8402/v1" { + t.Fatalf("litellm_params.api_base = %q, want the x402-buyer Service (the buyer is no longer a litellm sidecar)", entry.LiteLLMParams.APIBase) } if got := fakeAPI.addCalls.Load(); got != 1 { @@ -348,18 +348,18 @@ func TestCheckBuyerStatusSkipsDeletingPods(t *testing.T) { kubeClient := fake.NewSimpleClientset( &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "litellm-old", + Name: "x402-buyer-old", Namespace: "llm", - Labels: map[string]string{"app": "litellm"}, + Labels: map[string]string{"app": "x402-buyer"}, DeletionTimestamp: &now, }, Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.1"}, }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "litellm-new", + Name: "x402-buyer-new", Namespace: "llm", - Labels: map[string]string{"app": "litellm"}, + Labels: map[string]string{"app": "x402-buyer"}, }, Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.2"}, }, diff --git a/internal/serviceoffercontroller/purchase_lifecycle_test.go b/internal/serviceoffercontroller/purchase_lifecycle_test.go index 65f96bd2..6ee31061 100644 --- a/internal/serviceoffercontroller/purchase_lifecycle_test.go +++ b/internal/serviceoffercontroller/purchase_lifecycle_test.go @@ -156,9 +156,9 @@ func newPurchaseLifecycleController(t *testing.T, purchases ...monetizeapi.Purch }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "litellm-0", + Name: "x402-buyer-0", Namespace: "llm", - Labels: map[string]string{"app": "litellm"}, + Labels: map[string]string{"app": "x402-buyer"}, }, Status: corev1.PodStatus{ Phase: corev1.PodRunning, diff --git a/internal/serviceoffercontroller/purchase_migration_test.go b/internal/serviceoffercontroller/purchase_migration_test.go new file mode 100644 index 00000000..4df7eed5 --- /dev/null +++ b/internal/serviceoffercontroller/purchase_migration_test.go @@ -0,0 +1,201 @@ +package serviceoffercontroller + +import ( + "context" + "testing" + + "github.com/ObolNetwork/obol-stack/internal/model" + "gopkg.in/yaml.v3" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// These tests pin the upgrade path for the x402-buyer split (issue #321): +// clusters upgraded in place carry paid-route entries whose api_base still +// points at the removed litellm-pod sidecar (127.0.0.1:8402). Both the +// startup migration and the per-purchase reconcile must rewrite them to the +// x402-buyer Service and hot-sync the live router. + +func newMigrationController(t *testing.T, configYAML string) (*Controller, *litellmFake) { + t.Helper() + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "litellm-config", Namespace: "llm"}, + Data: map[string]string{"config.yaml": configYAML}, + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "litellm-secrets", Namespace: "llm"}, + Data: map[string][]byte{"LITELLM_MASTER_KEY": []byte("sk-obol-test")}, + } + fakeAPI := newLiteLLMFake() + t.Cleanup(fakeAPI.close) + + return &Controller{ + kubeClient: fake.NewSimpleClientset(cm, secret), + httpClient: fakeAPI.server.Client(), + litellmURLOverride: fakeAPI.server.URL, + }, fakeAPI +} + +func configMapModelList(t *testing.T, c *Controller) []model.ModelEntry { + t.Helper() + + cm, err := c.kubeClient.CoreV1().ConfigMaps("llm").Get(context.Background(), "litellm-config", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get litellm-config: %v", err) + } + var cfg model.LiteLLMConfig + if err := yaml.Unmarshal([]byte(cm.Data["config.yaml"]), &cfg); err != nil { + t.Fatalf("parse config.yaml: %v", err) + } + return cfg.ModelList +} + +func TestBuyerAPIBase(t *testing.T) { + // The /v1 suffix is load-bearing: LiteLLM's OpenAI provider does not + // append it, and without it the buyer's mux returns 404 (CLAUDE.md + // pitfall 6). + if got, want := buyerAPIBase("llm"), "http://x402-buyer.llm.svc.cluster.local:8402/v1"; got != want { + t.Fatalf("buyerAPIBase(llm) = %q; want %q", got, want) + } +} + +func TestAddLiteLLMModelEntryMigratesLegacySidecarAPIBase(t *testing.T) { + const legacyConfig = `model_list: + - model_name: "paid/qwen36-deep" + litellm_params: + model: "openai/paid/qwen36-deep" + api_base: "http://127.0.0.1:8402/v1" + api_key: "unused" +` + c, fakeAPI := newMigrationController(t, legacyConfig) + fakeAPI.infoResp = []map[string]any{ + {"model_name": "paid/qwen36-deep", "model_info": map[string]any{"id": "id-1"}}, + } + + c.addLiteLLMModelEntry(context.Background(), "llm", "paid/qwen36-deep") + + entries := configMapModelList(t, c) + if len(entries) != 1 { + t.Fatalf("expected 1 entry after migration, got %d", len(entries)) + } + if got, want := entries[0].LiteLLMParams.APIBase, buyerAPIBase("llm"); got != want { + t.Errorf("api_base = %q; want migrated %q", got, want) + } + if got := fakeAPI.delCalls.Load(); got != 1 { + t.Errorf("delCalls = %d; want 1 (stale legacy deployment removed from live router)", got) + } + if got := fakeAPI.addCalls.Load(); got != 1 { + t.Errorf("addCalls = %d; want 1 (migrated entry hot-added)", got) + } +} + +func TestAddLiteLLMModelEntryLeavesNonLegacyEntryAlone(t *testing.T) { + const currentConfig = `model_list: + - model_name: "paid/qwen36-deep" + litellm_params: + model: "openai/paid/qwen36-deep" + api_base: "http://x402-buyer.llm.svc.cluster.local:8402/v1" + api_key: "unused" +` + c, fakeAPI := newMigrationController(t, currentConfig) + + c.addLiteLLMModelEntry(context.Background(), "llm", "paid/qwen36-deep") + + if got := fakeAPI.addCalls.Load() + fakeAPI.delCalls.Load(); got != 0 { + t.Errorf("expected no hot API calls for an up-to-date entry, got %d", got) + } + entries := configMapModelList(t, c) + if got, want := entries[0].LiteLLMParams.APIBase, buyerAPIBase("llm"); got != want { + t.Errorf("api_base = %q; want unchanged %q", got, want) + } +} + +func TestMigrateLegacyBuyerAPIBases(t *testing.T) { + // Wildcard and concrete legacy entries are both rewritten; entries + // pointing elsewhere (cloud providers, custom endpoints) are untouched. + const mixedConfig = `model_list: + - model_name: "paid/*" + litellm_params: + model: "openai/*" + api_base: "http://127.0.0.1:8402/v1" + api_key: "unused" + - model_name: "paid/qwen36-deep" + litellm_params: + model: "openai/paid/qwen36-deep" + api_base: "http://127.0.0.1:8402/v1" + api_key: "unused" + - model_name: "qwen36-deep" + litellm_params: + model: "openai/qwen36-deep" + api_base: "http://192.168.18.23:8000/v1" + api_key: "os.environ/CUSTOM_KEY" +` + c, fakeAPI := newMigrationController(t, mixedConfig) + fakeAPI.infoResp = []map[string]any{ + {"model_name": "paid/*", "model_info": map[string]any{"id": "id-wild"}}, + {"model_name": "paid/qwen36-deep", "model_info": map[string]any{"id": "id-1"}}, + } + + c.migrateLegacyBuyerAPIBases(context.Background(), "llm") + + entries := configMapModelList(t, c) + if len(entries) != 3 { + t.Fatalf("expected 3 entries, got %d", len(entries)) + } + byName := make(map[string]model.ModelEntry, len(entries)) + for _, e := range entries { + byName[e.ModelName] = e + } + for _, name := range []string{"paid/*", "paid/qwen36-deep"} { + if got, want := byName[name].LiteLLMParams.APIBase, buyerAPIBase("llm"); got != want { + t.Errorf("%s api_base = %q; want migrated %q", name, got, want) + } + } + if got, want := byName["qwen36-deep"].LiteLLMParams.APIBase, "http://192.168.18.23:8000/v1"; got != want { + t.Errorf("custom endpoint api_base = %q; want untouched %q", got, want) + } + + if got := fakeAPI.delCalls.Load(); got != 2 { + t.Errorf("delCalls = %d; want 2 (both legacy deployments removed live)", got) + } + if got := fakeAPI.addCalls.Load(); got != 2 { + t.Errorf("addCalls = %d; want 2 (both migrated entries hot-added)", got) + } +} + +func TestMigrateLegacyBuyerAPIBasesNoopWhenClean(t *testing.T) { + const cleanConfig = `model_list: + - model_name: "paid/*" + litellm_params: + model: "openai/*" + api_base: "http://x402-buyer.llm.svc.cluster.local:8402/v1" + api_key: "unused" +` + c, fakeAPI := newMigrationController(t, cleanConfig) + + c.migrateLegacyBuyerAPIBases(context.Background(), "llm") + + if got := fakeAPI.addCalls.Load() + fakeAPI.delCalls.Load(); got != 0 { + t.Errorf("expected no hot API calls on a clean config, got %d", got) + } +} + +func TestMigrateLegacyBuyerAPIBasesMissingConfigMap(t *testing.T) { + fakeAPI := newLiteLLMFake() + t.Cleanup(fakeAPI.close) + c := &Controller{ + kubeClient: fake.NewSimpleClientset(), + httpClient: fakeAPI.server.Client(), + litellmURLOverride: fakeAPI.server.URL, + } + + // Must not panic or call the API when litellm-config does not exist + // (fresh cluster where the controller starts before the chart applies). + c.migrateLegacyBuyerAPIBases(context.Background(), "llm") + + if got := fakeAPI.addCalls.Load() + fakeAPI.delCalls.Load(); got != 0 { + t.Errorf("expected no hot API calls without a ConfigMap, got %d", got) + } +} diff --git a/internal/serviceoffercontroller/render.go b/internal/serviceoffercontroller/render.go index bf0bde54..ee8be8a1 100644 --- a/internal/serviceoffercontroller/render.go +++ b/internal/serviceoffercontroller/render.go @@ -28,6 +28,12 @@ const ( servicesJSONRouteName = "obol-services-json-route" openAPIRouteName = "obol-openapi-route" apiDocsRouteName = "obol-api-docs-route" + + // catalogHeadersMiddlewareName is the Traefik headers Middleware attached + // to the public catalog HTTPRoutes (/skill.md, /openapi.json, /api, + // /api/services.json). The busybox httpd serving those files cannot set + // custom response headers, so CORS + caching are applied at the gateway. + catalogHeadersMiddlewareName = "obol-catalog-headers" ) // restrictedPodSecurityContext returns a Pod-level securityContext that @@ -377,6 +383,59 @@ func buildSkillCatalogService() *unstructured.Unstructured { } } +// buildCatalogHeadersMiddleware renders the Traefik headers Middleware for +// the public discovery surfaces. These are read-only public documents served +// without credentials, so a wildcard CORS origin is correct — browser-based +// buyers, dashboards, and aggregators must be able to fetch them cross-origin. +// Cache-Control keeps CDN/browser refetch pressure off the busybox httpd +// while staying short enough (5 min) that catalog updates propagate quickly. +// +// Deliberately NOT attached to the /services/* paid routes (the 402/paid +// path has its own header semantics) nor to the ERC-8004 +// /.well-known/agent-registration.json routes (those live in per-agent +// namespaces, where an ExtensionRef cannot reference this x402-namespace +// Middleware). +func buildCatalogHeadersMiddleware() *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": "traefik.io/v1alpha1", + "kind": "Middleware", + "metadata": map[string]any{ + "name": catalogHeadersMiddlewareName, + "namespace": skillCatalogNamespace, + "labels": map[string]any{ + "obol.org/managed-by": "serviceoffer-controller", + }, + }, + "spec": map[string]any{ + "headers": map[string]any{ + "accessControlAllowOriginList": []any{"*"}, + "accessControlAllowMethods": []any{"GET", "OPTIONS"}, + "customResponseHeaders": map[string]any{ + "Cache-Control": "public, max-age=300", + }, + }, + }, + }, + } +} + +// catalogHeadersFilters is the HTTPRoute rule filter list that attaches the +// catalog headers Middleware — same ExtensionRef mechanism the x402 +// ForwardAuth Middleware uses on gated routes. +func catalogHeadersFilters() []any { + return []any{ + map[string]any{ + "type": "ExtensionRef", + "extensionRef": map[string]any{ + "group": "traefik.io", + "kind": "Middleware", + "name": catalogHeadersMiddlewareName, + }, + }, + } +} + func buildSkillCatalogHTTPRoute() *unstructured.Unstructured { return &unstructured.Unstructured{ Object: map[string]any{ @@ -407,6 +466,7 @@ func buildSkillCatalogHTTPRoute() *unstructured.Unstructured { }, }, }, + "filters": catalogHeadersFilters(), "backendRefs": []any{ map[string]any{ "name": skillCatalogConfigMapName, @@ -459,6 +519,7 @@ func buildOpenAPIHTTPRoute() *unstructured.Unstructured { }, }, }, + "filters": catalogHeadersFilters(), "backendRefs": []any{ map[string]any{ "name": skillCatalogConfigMapName, @@ -520,6 +581,7 @@ func buildAPIDocsHTTPRoute() *unstructured.Unstructured { }, }, }, + "filters": catalogHeadersFilters(), "backendRefs": []any{ map[string]any{ "name": skillCatalogConfigMapName, @@ -564,6 +626,7 @@ func buildServicesJSONHTTPRoute() *unstructured.Unstructured { }, }, }, + "filters": catalogHeadersFilters(), "backendRefs": []any{ map[string]any{ "name": skillCatalogConfigMapName, @@ -1013,6 +1076,7 @@ func buildSkillCatalogMarkdown(offers []*monetizeapi.ServiceOffer, baseURL strin description = fmt.Sprintf("x402 payment-gated %s service", fallbackOfferType(offer)) } lines = append(lines, fmt.Sprintf("- **Description**: %s", description), "") + lines = append(lines, skillCatalogTryIt(offer, endpoint)...) } return strings.Join(lines, "\n") @@ -1056,6 +1120,69 @@ func skillCatalogHowToPay(baseURL string) []string { } } +// catalogModelName resolves the model id a buyer should put in a paid +// chat-completions body. type=agent offers leave spec.model empty by design +// (the model lives on the linked Agent), so fall back to the controller's +// resolved view. Shared by /api/services.json and the /skill.md worked +// examples so both surfaces advertise the same id. +func catalogModelName(offer *monetizeapi.ServiceOffer) string { + if offer == nil { + return "" + } + if offer.Spec.Model.Name != "" { + return offer.Spec.Model.Name + } + if offer.Status.AgentResolution != nil { + return offer.Status.AgentResolution.Model + } + return "" +} + +// skillCatalogTryIt renders the per-offer "Try it" subsection: one curl that +// probes the 402 pricing, and one worked paid request. The paid example for +// chat-shaped offers is buyprompts.Build's Example — the exact same bytes +// /api/services.json publishes in the entry's buy.example — so the two +// surfaces cannot drift. Agent buyers convert off copy-paste, not prose. +func skillCatalogTryIt(offer *monetizeapi.ServiceOffer, endpoint string) []string { + lines := []string{ + "#### Try it", + "", + "Probe the price (no payment; the `402` body carries the signable `accepts[]` requirements):", + "", + "```bash", + fmt.Sprintf("curl -i %s", endpoint), + "```", + "", + } + block := buyprompts.Build(buyprompts.Input{ + Type: fallbackOfferType(offer), + URL: endpoint, + Model: catalogModelName(offer), + }) + if block.Example != "" { + // inference/agent: OpenAI-style chat-completions with the real model id. + lines = append(lines, + "Then sign one `accepts[]` entry — see [How to pay (x402)](#how-to-pay-x402) — and send a paid request:", + "", + "```", + block.Example, + "```", + "", + ) + } else { + // http/fine-tuning: retry the gated path itself with the payment attached. + lines = append(lines, + "Then sign one `accepts[]` entry — see [How to pay (x402)](#how-to-pay-x402) — and retry the identical request with the payment attached:", + "", + "```bash", + fmt.Sprintf("curl -i %s -H \"X-PAYMENT: \"", endpoint), + "```", + "", + ) + } + return lines +} + // offerOperationallyReady reports whether an offer is usable for x402 // payments today. This is intentionally LOOSER than the controller's // Ready=True condition: ModelReady + UpstreamHealthy + PaymentGateReady @@ -1175,13 +1302,7 @@ func buildServiceCatalogJSON(offers []*monetizeapi.ServiceOffer, baseURL string, if desc == "" { desc = fmt.Sprintf("x402 payment-gated %s service", fallbackOfferType(offer)) } - // type=agent offers leave spec.model empty by design (the model - // lives on the linked Agent). Fall back to the controller's - // resolved view so the storefront can display it. - modelName := offer.Spec.Model.Name - if modelName == "" && offer.Status.AgentResolution != nil { - modelName = offer.Status.AgentResolution.Model - } + modelName := catalogModelName(offer) drainEndsAt := "" if offer.IsDraining() { @@ -1256,10 +1377,11 @@ func buildServiceCatalogJSON(offers []*monetizeapi.ServiceOffer, baseURL string, } catalog := schemas.ServiceCatalog{ - DisplayName: profile.DisplayName, - Tagline: profile.Tagline, - LogoURL: profile.LogoURL, - Services: services, + SchemaVersion: schemas.ServiceCatalogSchemaVersion, + DisplayName: profile.DisplayName, + Tagline: profile.Tagline, + LogoURL: profile.LogoURL, + Services: services, } if catalog.Services == nil { catalog.Services = []schemas.ServiceCatalogEntry{} @@ -1275,14 +1397,15 @@ func buildServiceCatalogJSON(offers []*monetizeapi.ServiceOffer, baseURL string, func fallbackServiceCatalogJSON(baseURL string) string { profile := storefront.ResolvePublished(nil, baseURL) catalog := schemas.ServiceCatalog{ - DisplayName: profile.DisplayName, - Tagline: profile.Tagline, - LogoURL: profile.LogoURL, - Services: []schemas.ServiceCatalogEntry{}, + SchemaVersion: schemas.ServiceCatalogSchemaVersion, + DisplayName: profile.DisplayName, + Tagline: profile.Tagline, + LogoURL: profile.LogoURL, + Services: []schemas.ServiceCatalogEntry{}, } out, err := json.MarshalIndent(catalog, "", " ") if err != nil { - return `{"displayName":"Obol Stack","tagline":"Unlock Agent and API services with digital payments.","logoUrl":"/obol-stack-logo.png","services":[]}` + return `{"schemaVersion":"1","displayName":"Obol Stack","tagline":"Unlock Agent and API services with digital payments.","logoUrl":"/obol-stack-logo.png","services":[]}` } return string(out) } diff --git a/internal/serviceoffercontroller/render_test.go b/internal/serviceoffercontroller/render_test.go index c1b023f1..58e4dc36 100644 --- a/internal/serviceoffercontroller/render_test.go +++ b/internal/serviceoffercontroller/render_test.go @@ -11,6 +11,7 @@ import ( "github.com/ObolNetwork/obol-stack/internal/schemas" "github.com/santhosh-tekuri/jsonschema/v6" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" ) @@ -1569,3 +1570,185 @@ func TestBuildServiceCatalogJSON_RegistrationPendingFalseForFullyReady(t *testin t.Error("RegistrationPending must be false for fully Ready=True offers") } } + +// TestBuildCatalogHeadersMiddleware locks the CORS + cache posture on the +// public catalog surfaces: wildcard origin (read-only public data, no +// credentials), GET/OPTIONS only, and a 5-minute public cache. +func TestBuildCatalogHeadersMiddleware(t *testing.T) { + mw := buildCatalogHeadersMiddleware() + if got := mw.GetAPIVersion(); got != "traefik.io/v1alpha1" { + t.Fatalf("apiVersion = %q, want traefik.io/v1alpha1", got) + } + if got := mw.GetKind(); got != "Middleware" { + t.Fatalf("kind = %q, want Middleware", got) + } + if mw.GetName() != catalogHeadersMiddlewareName { + t.Fatalf("name = %q, want %q", mw.GetName(), catalogHeadersMiddlewareName) + } + if mw.GetNamespace() != skillCatalogNamespace { + t.Fatalf("namespace = %q, want %q", mw.GetNamespace(), skillCatalogNamespace) + } + + headers := mw.Object["spec"].(map[string]any)["headers"].(map[string]any) + origins := headers["accessControlAllowOriginList"].([]any) + if len(origins) != 1 || origins[0] != "*" { + t.Errorf("accessControlAllowOriginList = %v, want [*]", origins) + } + methods := headers["accessControlAllowMethods"].([]any) + if len(methods) != 2 || methods[0] != "GET" || methods[1] != "OPTIONS" { + t.Errorf("accessControlAllowMethods = %v, want [GET OPTIONS]", methods) + } + custom := headers["customResponseHeaders"].(map[string]any) + if custom["Cache-Control"] != "public, max-age=300" { + t.Errorf("Cache-Control = %v, want 'public, max-age=300'", custom["Cache-Control"]) + } +} + +// TestCatalogRoutesCarryHeadersFilter asserts every catalog HTTPRoute +// (/skill.md, /api/services.json, /openapi.json, /api) attaches the headers +// Middleware via an ExtensionRef filter — the same mechanism the x402 +// middleware uses on paid routes. Paid /services/* routes must NOT carry it +// (locked separately by TestBuildHTTPRoute's no-filters assertion). +func TestCatalogRoutesCarryHeadersFilter(t *testing.T) { + routes := map[string]*unstructured.Unstructured{ + "/skill.md": buildSkillCatalogHTTPRoute(), + "/api/services.json": buildServicesJSONHTTPRoute(), + "/openapi.json": buildOpenAPIHTTPRoute(), + "/api": buildAPIDocsHTTPRoute(), + } + for path, route := range routes { + rules := route.Object["spec"].(map[string]any)["rules"].([]any) + firstRule := rules[0].(map[string]any) + filters, found := firstRule["filters"].([]any) + if !found || len(filters) != 1 { + t.Errorf("%s route: filters = %v, want exactly one ExtensionRef filter", path, firstRule["filters"]) + continue + } + filter := filters[0].(map[string]any) + if filter["type"] != "ExtensionRef" { + t.Errorf("%s route: filter type = %v, want ExtensionRef", path, filter["type"]) + } + ref := filter["extensionRef"].(map[string]any) + if ref["group"] != "traefik.io" || ref["kind"] != "Middleware" || ref["name"] != catalogHeadersMiddlewareName { + t.Errorf("%s route: extensionRef = %v, want traefik.io Middleware %q", path, ref, catalogHeadersMiddlewareName) + } + } +} + +// TestBuildServiceCatalogJSON_SchemaVersion locks the envelope version +// marker: always present, always "1" until a breaking wire change bumps it. +func TestBuildServiceCatalogJSON_SchemaVersion(t *testing.T) { + jsonStr := buildServiceCatalogJSON(nil, "https://example.com", nil) + assertServiceCatalogSchema(t, jsonStr) + + if got := decodeServiceCatalog(t, jsonStr).SchemaVersion; got != schemas.ServiceCatalogSchemaVersion { + t.Errorf("schemaVersion = %q, want %q", got, schemas.ServiceCatalogSchemaVersion) + } + // Assert on the raw wire too, so a struct/tag rename can't silently + // satisfy the decode-side check. + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(jsonStr), &raw); err != nil { + t.Fatalf("unmarshal raw catalog: %v", err) + } + if string(raw["schemaVersion"]) != `"1"` { + t.Errorf(`raw schemaVersion = %s, want "1"`, raw["schemaVersion"]) + } + + // The fallback envelope must carry the same version. + fallback := fallbackServiceCatalogJSON("https://example.com") + assertServiceCatalogSchema(t, fallback) + if got := decodeServiceCatalog(t, fallback).SchemaVersion; got != schemas.ServiceCatalogSchemaVersion { + t.Errorf("fallback schemaVersion = %q, want %q", got, schemas.ServiceCatalogSchemaVersion) + } +} + +// TestBuildSkillCatalogMarkdown_TryIt asserts every offer detail block ends +// with a copy-paste "Try it" section: a 402 probe curl plus a worked paid +// request. Chat-shaped offers must show the REAL model id (including the +// AgentResolution fallback for type=agent) — the same id services.json +// publishes — and http offers a curl of the gated path. +func TestBuildSkillCatalogMarkdown_TryIt(t *testing.T) { + readyCond := []monetizeapi.Condition{{Type: "Ready", Status: "True"}} + inferenceOffer := &monetizeapi.ServiceOffer{ + ObjectMeta: metav1.ObjectMeta{Name: "flow-qwen", Namespace: "llm"}, + Spec: monetizeapi.ServiceOfferSpec{ + Type: "inference", + Model: monetizeapi.ServiceOfferModel{Name: "qwen36-deep"}, + Payment: monetizeapi.ServiceOfferPayment{ + Network: "base", + PayTo: "0x1111111111111111111111111111111111111111", + Price: monetizeapi.ServiceOfferPriceTable{PerMTok: "0.10"}, + }, + }, + Status: monetizeapi.ServiceOfferStatus{Conditions: readyCond}, + } + agentOffer := &monetizeapi.ServiceOffer{ + ObjectMeta: metav1.ObjectMeta{Name: "quant", Namespace: "agent-quant"}, + Spec: monetizeapi.ServiceOfferSpec{ + Type: "agent", + Payment: monetizeapi.ServiceOfferPayment{ + Network: "base", + PayTo: "0x2222222222222222222222222222222222222222", + Price: monetizeapi.ServiceOfferPriceTable{PerRequest: "0.05"}, + }, + }, + Status: monetizeapi.ServiceOfferStatus{ + AgentResolution: &monetizeapi.ServiceOfferAgentResolution{Model: "qwen3.5:9b", Runtime: "hermes"}, + Conditions: readyCond, + }, + } + httpOffer := &monetizeapi.ServiceOffer{ + ObjectMeta: metav1.ObjectMeta{Name: "echo", Namespace: "demo"}, + Spec: monetizeapi.ServiceOfferSpec{ + Type: "http", + Payment: monetizeapi.ServiceOfferPayment{ + Network: "base", + PayTo: "0x3333333333333333333333333333333333333333", + Price: monetizeapi.ServiceOfferPriceTable{PerRequest: "0.001"}, + }, + }, + Status: monetizeapi.ServiceOfferStatus{Conditions: readyCond}, + } + + content := buildSkillCatalogMarkdown( + []*monetizeapi.ServiceOffer{inferenceOffer, agentOffer, httpOffer}, + "https://seller.example", nil, + ) + + if got := strings.Count(content, "#### Try it"); got != 3 { + t.Fatalf("Try it sections = %d, want 3:\n%s", got, content) + } + // 402 probe curl per offer. + for _, probe := range []string{ + "curl -i https://seller.example/services/flow-qwen", + "curl -i https://seller.example/services/quant", + "curl -i https://seller.example/services/echo", + } { + if !strings.Contains(content, probe) { + t.Errorf("missing 402 probe %q:\n%s", probe, content) + } + } + // Chat-shaped offers: worked chat-completions example with the real model + // id (identical bytes to services.json's buy.example — buyprompts is the + // single authoring point). + if !strings.Contains(content, "POST https://seller.example/services/flow-qwen/v1/chat/completions") { + t.Errorf("inference Try it missing chat-completions example:\n%s", content) + } + if !strings.Contains(content, `"model": "qwen36-deep"`) { + t.Errorf("inference Try it missing real model id qwen36-deep:\n%s", content) + } + if !strings.Contains(content, `"model": "qwen3.5:9b"`) { + t.Errorf("agent Try it missing AgentResolution model qwen3.5:9b:\n%s", content) + } + if !strings.Contains(content, "X-PAYMENT: ") { + t.Errorf("Try it examples missing X-PAYMENT placeholder:\n%s", content) + } + // http offer: paid retry is a curl of the gated path. + if !strings.Contains(content, `curl -i https://seller.example/services/echo -H "X-PAYMENT: "`) { + t.Errorf("http Try it missing paid curl:\n%s", content) + } + // Payment mechanics are referenced, not duplicated. + if !strings.Contains(content, "[How to pay (x402)](#how-to-pay-x402)") { + t.Errorf("Try it missing How-to-pay anchor link:\n%s", content) + } +} diff --git a/internal/stack/stack_test.go b/internal/stack/stack_test.go index 0854fda3..4244e16c 100644 --- a/internal/stack/stack_test.go +++ b/internal/stack/stack_test.go @@ -485,17 +485,19 @@ func TestLLMTemplate_IncludesPaidRouteAndBuyerSidecar(t *testing.T) { for _, want := range []string{ `model_name: "paid/*"`, `model: "openai/*"`, - `api_base: "http://127.0.0.1:8402/v1"`, + // Paid routes go through the standalone x402-buyer Service — the + // buyer is no longer a litellm-pod sidecar (issue #321: LiteLLM must + // be stateless so RollingUpdate maxUnavailable:0 gives zero-downtime + // rollouts). + `api_base: "http://x402-buyer.llm.svc.cluster.local:8402/v1"`, `name: x402-buyer`, `containerPort: 8402`, `name: buyer-http`, `name: x402-buyer-config`, `name: x402-buyer-auths`, - // litellm-config ONLY: the buyer ConfigMaps must stay out of the - // Reloader annotation — x402-buyer hot-reloads them via /admin/reload - // and the Recreate strategy would otherwise bounce the whole gateway - // on every buy/refill (CLAUDE.md pitfall 7). - `configmap.reloader.stakater.com/reload: "litellm-config"`, + // Key rotation is the one remaining case that needs a pod + // replacement (os.environ/ refs resolve at config load). + `secret.reloader.stakater.com/reload: "litellm-secrets"`, `emptyDir:`, } { if !strings.Contains(out, want) { @@ -503,8 +505,13 @@ func TestLLMTemplate_IncludesPaidRouteAndBuyerSidecar(t *testing.T) { } } - if strings.Contains(out, `configmap.reloader.stakater.com/reload: "litellm-config,`) { - t.Fatal("llm template reload annotation must list litellm-config only — buyer CM writes happen per purchase and would Recreate-bounce the gateway") + // Reloader must NOT watch litellm-config: every model_list change is + // hot-applied via /model/new + /model/delete (CLI and controller), and a + // ConfigMap-triggered rollout would reintroduce an inference gap on + // every model add/remove/prefer and first-time purchase (issue #321). + // The buyer ConfigMaps likewise hot-reload via /admin/reload. + if strings.Contains(out, "configmap.reloader.stakater.com/reload") { + t.Fatal("llm template must not carry a configmap Reloader annotation — model_list changes are hot-applied; a CM-triggered rollout gaps inference (issue #321)") } if strings.Contains(out, "custom_provider_map") { diff --git a/internal/x402/forwardauth.go b/internal/x402/forwardauth.go index 4b445786..212b1a4b 100644 --- a/internal/x402/forwardauth.go +++ b/internal/x402/forwardauth.go @@ -376,6 +376,7 @@ func NewForwardAuthMiddleware(cfg ForwardAuthConfig, requirements []x402types.Pa // understand; it remains the default when ForwardAuthConfig.SendPaymentRequired // is unset and the fallback when the renderer has nothing else to do. func sendPaymentRequiredJSON(w http.ResponseWriter, r *http.Request, requirements []x402types.PaymentRequirements, extensions map[string]any) { + setCatalogLinkHeader(w) resp := buildPaymentRequired(r, requirements, extensions) body, err := json.Marshal(resp) @@ -390,6 +391,16 @@ func sendPaymentRequiredJSON(w http.ResponseWriter, r *http.Request, requirement _, _ = w.Write(body) } +// setCatalogLinkHeader advertises the seller's machine-readable service +// catalog on every 402 response (RFC 8288 web linking). An agent that lands +// on a paid endpoint directly — with no prior knowledge of the seller's +// layout — can follow the link to /api/services.json and self-serve +// discovery of every other offer. Header-only addition: the 402 body schema +// and the verification/settlement flow are unchanged. +func setCatalogLinkHeader(w http.ResponseWriter) { + w.Header().Set("Link", `; rel="catalog"`) +} + // buildPaymentRequired assembles the v2 PaymentRequired object for the // incoming request, including the bazaar service metadata on the resource // block (serviceName/iconUrl — see specs/extensions/bazaar.md, soft-drop diff --git a/internal/x402/forwardauth_test.go b/internal/x402/forwardauth_test.go index 83603f8d..0fc7c3f8 100644 --- a/internal/x402/forwardauth_test.go +++ b/internal/x402/forwardauth_test.go @@ -866,3 +866,49 @@ func TestForwardAuth_SettlesInProcess_SuppressesWarning(t *testing.T) { t.Fatalf("SettlesInProcess=true must suppress the verifyOnly=false warning, got:\n%s", gotLog) } } + +// TestForwardAuth_402CarriesCatalogLinkHeader locks the discovery Link +// header on the default (JSON) 402 path through the middleware itself — +// both on the no-payment challenge and on the re-issued challenge after an +// invalid payment. Header-only: verification behaviour is asserted by the +// sibling tests and must not change. +func TestForwardAuth_402CarriesCatalogLinkHeader(t *testing.T) { + const wantLink = `; rel="catalog"` + + var verifyCalled, settleCalled atomic.Int32 + fac := mockFacilitatorV1(false, true, &verifyCalled, &settleCalled) + defer fac.Close() + + mw := NewForwardAuthMiddleware(ForwardAuthConfig{ + FacilitatorURL: fac.URL, + VerifyOnly: true, + }, testRequirements()) + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("inner handler should not be called on a 402") + }) + + t.Run("no payment", func(t *testing.T) { + req := httptest.NewRequest("POST", "/v1/chat/completions", nil) + rec := httptest.NewRecorder() + mw(inner).ServeHTTP(rec, req) + if rec.Code != http.StatusPaymentRequired { + t.Fatalf("status = %d, want 402", rec.Code) + } + if got := rec.Header().Get("Link"); got != wantLink { + t.Errorf("Link = %q, want %q", got, wantLink) + } + }) + + t.Run("invalid payment rechallenge", func(t *testing.T) { + req := httptest.NewRequest("POST", "/v1/chat/completions", nil) + req.Header.Set("X-PAYMENT", validPaymentHeader()) + rec := httptest.NewRecorder() + mw(inner).ServeHTTP(rec, req) + if rec.Code != http.StatusPaymentRequired { + t.Fatalf("status = %d, want 402", rec.Code) + } + if got := rec.Header().Get("Link"); got != wantLink { + t.Errorf("Link = %q, want %q", got, wantLink) + } + }) +} diff --git a/internal/x402/paymentrequired.go b/internal/x402/paymentrequired.go index 979cb083..f9bd445e 100644 --- a/internal/x402/paymentrequired.go +++ b/internal/x402/paymentrequired.go @@ -210,6 +210,9 @@ func isLocalHost(host string) bool { // full OG metadata, a service-info card, three "ways to pay" prompt cards // (Obol Agent, other AI agent, raw JSON), and copy buttons. func sendPaymentRequiredHTML(w http.ResponseWriter, r *http.Request, requirements []x402types.PaymentRequirements, extensions map[string]any, display PaymentDisplay) { + // Catalog discovery link rides on the HTML branch too (and survives the + // JSON fallbacks below — Header().Set is idempotent). + setCatalogLinkHeader(w) jsonBody := buildPaymentRequired(r, requirements, extensions) indented, err := json.MarshalIndent(jsonBody, "", " ") if err != nil { diff --git a/internal/x402/paymentrequired_test.go b/internal/x402/paymentrequired_test.go index 4cf77151..ef969f47 100644 --- a/internal/x402/paymentrequired_test.go +++ b/internal/x402/paymentrequired_test.go @@ -505,3 +505,38 @@ func TestAgentCopy_NoModelProseAndRunnableExample(t *testing.T) { // The "other AI agent" prompt embeds the concrete message too. mustContain(t, c.PromptOther, defaultAgentTaskExample) } + +// Every 402 — JSON and HTML branch alike — must advertise the seller's +// machine-readable catalog via an RFC 8288 Link header, so an agent that +// lands on a paid endpoint with no prior knowledge can self-serve discovery. +// Header-only invariant: the body schema assertions live in the sibling +// tests above and must not change. +func TestPaymentRequired_CarriesCatalogLinkHeader(t *testing.T) { + const wantLink = `; rel="catalog"` + render := NewHTMLAwarePaymentRequired(sampleDisplay()) + + t.Run("json", func(t *testing.T) { + r := httptest.NewRequest("GET", "/services/agent-quant", nil) + w := httptest.NewRecorder() + render(w, r, []x402types.PaymentRequirements{sampleRequirement()}, nil) + if w.Code != http.StatusPaymentRequired { + t.Fatalf("status = %d, want 402", w.Code) + } + if got := w.Header().Get("Link"); got != wantLink { + t.Errorf("Link = %q, want %q", got, wantLink) + } + }) + + t.Run("html", func(t *testing.T) { + r := httptest.NewRequest("GET", "/services/agent-quant", nil) + r.Header.Set("Accept", "text/html") + w := httptest.NewRecorder() + render(w, r, []x402types.PaymentRequirements{sampleRequirement()}, nil) + if w.Code != http.StatusPaymentRequired { + t.Fatalf("status = %d, want 402", w.Code) + } + if got := w.Header().Get("Link"); got != wantLink { + t.Errorf("Link = %q, want %q", got, wantLink) + } + }) +} diff --git a/obolup.sh b/obolup.sh index d6d2aeca..70caec19 100755 --- a/obolup.sh +++ b/obolup.sh @@ -473,6 +473,101 @@ EOF log_success "Installed development wrapper at $OBOL_BIN_DIR/obol" } +# Compute the SHA256 hash of a file. +# Portable: macOS ships shasum, Linux ships sha256sum. +sha256_file() { + local file="$1" + if command_exists sha256sum; then + sha256sum "$file" | awk '{print $1}' + elif command_exists shasum; then + shasum -a 256 "$file" | awk '{print $1}' + else + return 1 + fi +} + +# Download a URL to a file with retries (3 attempts, short sleep between). +# Used for the obol release binary and its SHA256SUMS file only. +download_with_retries() { + local url="$1" + local dest="$2" + local attempts=3 + local attempt + + for attempt in $(seq 1 "$attempts"); do + if command_exists curl; then + if curl -fsSL "$url" -o "$dest"; then + return 0 + fi + elif command_exists wget; then + if wget -q "$url" -O "$dest"; then + return 0 + fi + else + log_error "Neither curl nor wget is available" + return 1 + fi + + if [[ "$attempt" -lt "$attempts" ]]; then + log_warn "Download failed (attempt $attempt/$attempts), retrying in 2s..." + sleep 2 + fi + done + + rm -f "$dest" + return 1 +} + +# Verify a downloaded release binary against the published SHA256SUMS file. +# Fails (returns 1) on checksum mismatch. Warns and continues (returns 0) if +# SHA256SUMS is unavailable (older releases) or no local sha256 tool exists. +verify_release_checksum() { + local release_tag="$1" + local binary_name="$2" + local binary_path="$3" + + local sums_url="https://github.com/ObolNetwork/obol-stack/releases/download/${release_tag}/SHA256SUMS" + local tmp_sums="${binary_path}.sha256sums" + + if ! download_with_retries "$sums_url" "$tmp_sums"; then + log_warn "SHA256SUMS not published for $release_tag, skipping checksum verification" + return 0 + fi + + # Match on the artifact name as published (e.g. obol_darwin_arm64), + # handling both "hash name" and "hash *name" (binary mode) formats. + local expected_hash + expected_hash=$(awk -v name="$binary_name" '$2 == name || $2 == "*"name {print $1; exit}' "$tmp_sums") + rm -f "$tmp_sums" + + if [[ -z "$expected_hash" ]]; then + log_warn "No checksum entry for $binary_name in SHA256SUMS, skipping verification" + return 0 + fi + + local actual_hash + if ! actual_hash=$(sha256_file "$binary_path"); then + log_warn "Neither sha256sum nor shasum is available, skipping checksum verification" + return 0 + fi + + if [[ "$actual_hash" != "$expected_hash" ]]; then + log_error "Checksum mismatch for $binary_name ($release_tag)" + echo "" + echo " Expected: $expected_hash" + echo " Actual: $actual_hash" + echo "" + echo "The downloaded binary may be corrupted or tampered with." + echo "Please re-run the installer. If the problem persists, report an issue at:" + echo " https://github.com/ObolNetwork/obol-stack/issues" + echo "" + return 1 + fi + + log_success "Checksum verified for $binary_name" + return 0 +} + # Download and install binary from GitHub releases download_release() { local release_tag="$1" @@ -506,21 +601,16 @@ download_release() { local tmp_obol="$OBOL_BIN_DIR/obol.tmp" - # Download binary - if command_exists curl; then - if ! curl -fsSL "$download_url" -o "$tmp_obol"; then - log_warn "Failed to download release $release_tag" - rm -f "$tmp_obol" - return 1 - fi - elif command_exists wget; then - if ! wget -q "$download_url" -O "$tmp_obol"; then - log_warn "Failed to download release $release_tag" - rm -f "$tmp_obol" - return 1 - fi - else - log_error "Neither curl nor wget is available" + # Download binary (with retries) + if ! download_with_retries "$download_url" "$tmp_obol"; then + log_warn "Failed to download release $release_tag" + rm -f "$tmp_obol" + return 1 + fi + + # Verify against the published SHA256SUMS before installing + if ! verify_release_checksum "$release_tag" "$binary_name" "$tmp_obol"; then + rm -f "$tmp_obol" return 1 fi @@ -700,7 +790,7 @@ copy_bootstrap_script() { log_warn "Failed to download bootstrap script (non-critical)" log_info "You can manually download it later with:" echo "" - echo " curl -sSL $script_source_url -o $OBOL_BIN_DIR/obolup.sh" + echo " curl -fsSL $script_source_url -o $OBOL_BIN_DIR/obolup.sh" echo " chmod +x $OBOL_BIN_DIR/obolup.sh" echo "" fi @@ -818,7 +908,7 @@ install_kubectl() { # Download kubectl local download_url="https://dl.k8s.io/release/v${target_version}/bin/${platform}/${arch}/kubectl" - if curl -sSL "$download_url" -o "$OBOL_BIN_DIR/kubectl.tmp"; then + if curl -fsSL "$download_url" -o "$OBOL_BIN_DIR/kubectl.tmp"; then chmod +x "$OBOL_BIN_DIR/kubectl.tmp" mv "$OBOL_BIN_DIR/kubectl.tmp" "$OBOL_BIN_DIR/kubectl" log_success "kubectl v$target_version installed" @@ -877,7 +967,7 @@ install_helm() { local tmp_dir=$(mktemp -d) local download_url="https://get.helm.sh/helm-v${target_version}-${platform}-${arch}.tar.gz" - if curl -sSL "$download_url" | tar xz -C "$tmp_dir" 2>/dev/null; then + if curl -fsSL "$download_url" | tar xz -C "$tmp_dir" 2>/dev/null; then mv "$tmp_dir/${platform}-${arch}/helm" "$OBOL_BIN_DIR/helm" chmod +x "$OBOL_BIN_DIR/helm" rm -rf "$tmp_dir" @@ -940,7 +1030,7 @@ install_k3d() { # Download k3d local download_url="https://github.com/k3d-io/k3d/releases/download/v${target_version}/k3d-${k3d_platform}-${k3d_arch}" - if curl -sSL "$download_url" -o "$OBOL_BIN_DIR/k3d.tmp"; then + if curl -fsSL "$download_url" -o "$OBOL_BIN_DIR/k3d.tmp"; then chmod +x "$OBOL_BIN_DIR/k3d.tmp" mv "$OBOL_BIN_DIR/k3d.tmp" "$OBOL_BIN_DIR/k3d" log_success "k3d v$target_version installed" @@ -1010,7 +1100,7 @@ install_helmfile() { local tmp_dir=$(mktemp -d) local download_url="https://github.com/helmfile/helmfile/releases/download/v${target_version}/helmfile_${target_version}_${helmfile_platform}_${helmfile_arch}.tar.gz" - if curl -sSL "$download_url" | tar xz -C "$tmp_dir" 2>/dev/null; then + if curl -fsSL "$download_url" | tar xz -C "$tmp_dir" 2>/dev/null; then mv "$tmp_dir/helmfile" "$OBOL_BIN_DIR/helmfile" chmod +x "$OBOL_BIN_DIR/helmfile" rm -rf "$tmp_dir" @@ -1140,7 +1230,7 @@ install_k9s() { local tmp_dir=$(mktemp -d) local download_url="https://github.com/derailed/k9s/releases/download/v${target_version}/k9s_${k9s_platform}_${k9s_arch}.tar.gz" - if curl -sSL "$download_url" | tar xz -C "$tmp_dir" 2>/dev/null; then + if curl -fsSL "$download_url" | tar xz -C "$tmp_dir" 2>/dev/null; then mv "$tmp_dir/k9s" "$OBOL_BIN_DIR/k9s" chmod +x "$OBOL_BIN_DIR/k9s" rm -rf "$tmp_dir" @@ -1556,10 +1646,10 @@ install_dependencies() { check_hosts_file() { log_info "Checking /etc/hosts for obol.stack entry..." - # Check if /etc/hosts contains obol.stack pointing to localhost - if grep -q "obol.stack" /etc/hosts 2>/dev/null; then + # Check if /etc/hosts contains obol.stack (exact hostname, not e.g. dev.obol.stack) + if grep -E "^[^#]*[[:space:]]obol\.stack([[:space:]]|$)" /etc/hosts >/dev/null 2>&1; then # Check if it points to localhost (127.0.0.1 or ::1) - if grep -E "^(127\.0\.0\.1|::1)[[:space:]].*obol\.stack" /etc/hosts >/dev/null 2>&1; then + if grep -E "^(127\.0\.0\.1|::1)[[:space:]]([^#]*[[:space:]])?obol\.stack([[:space:]]|$)" /etc/hosts >/dev/null 2>&1; then log_success "obol.stack already configured in /etc/hosts" return 0 else @@ -1578,6 +1668,13 @@ update_hosts_file() { local hosts_entry="127.0.0.1 obol.stack" + # Idempotency guard: skip the append if a localhost entry already exists + # (protects against duplicates on re-run, even if the caller's check changes) + if grep -E "^(127\.0\.0\.1|::1)[[:space:]]([^#]*[[:space:]])?obol\.stack([[:space:]]|$)" /etc/hosts >/dev/null 2>&1; then + log_success "obol.stack already configured in /etc/hosts, skipping" + return 0 + fi + # Check if sudo is available if ! command_exists sudo; then log_error "sudo not available, cannot update /etc/hosts automatically"