diff --git a/.gitignore b/.gitignore index 0f7193b87..086ae06c1 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,9 @@ hud/rl/checkpoints_test/ docs/internal +# Fetched by docs/scripts/gen_api_examples.py +docs/openapi.json + environments/ experiments/ diff --git a/docs/custom.css b/docs/custom.css index 7d23ba8be..178686a7f 100644 --- a/docs/custom.css +++ b/docs/custom.css @@ -595,3 +595,184 @@ div:has(> ul.sidebar-group > li[data-group-tag]) > .sidebar-group-header { font-family: "Inter", ui-sans-serif, system-ui, sans-serif; font-size: 0.92em; } + +/* ── Endpoint list (REST API reference) ──────────────────────────────────── + Accordions used as an endpoint index: the summary line is a method and a + path, so it reads as code, not prose. Scoped to `.api-endpoints` because + accordions elsewhere carry ordinary sentence titles. */ +#content .api-endpoints .accordion summary p:first-of-type { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, "Cascadia Code", monospace; + font-size: 0.85rem; + letter-spacing: -0.01em; +} + +/* Signature: a fixed-width method label, the path, then the one-line note, all + on one baseline. The label keeps a minimum width so paths line up in a column + down the list however long the method is (see snippets/endpoint.jsx). */ +#content .api-sig { + display: inline-flex; + align-items: baseline; + flex-wrap: wrap; + gap: 0.15rem 0.5rem; +} +/* The column, not the tint, holds the paths in line, so the tint can hug its + letters instead of being padded out to the width of DELETE. */ +#content .api-method-col { + flex: none; + min-width: 3.1rem; +} +/* Flat tint rather than a bordered pill: the method reads as part of the path, + not as a button sitting next to one. */ +#content .api-method { + display: inline-block; + font-size: 0.62rem; + font-weight: 700; + letter-spacing: 0.05em; + line-height: 1.15; + padding: 0.18rem 0.22rem; + border-radius: 4px; + color: var(--api-fg); + background-color: var(--api-bg); +} +#content .api-path { + color: var(--tw-prose-headings); + font-weight: 500; +} +/* The note drops back to the body font so it reads as prose beside the code. */ +#content .api-note { + font-family: "Inter", ui-sans-serif, system-ui, sans-serif; + font-size: 0.82rem; + font-weight: 400; + letter-spacing: 0; + margin-left: 0.25rem; + color: #8a8a8a; +} +.dark #content .api-note { + color: #9a9a9a; +} +/* Path placeholders read as slots to fill, not as literal path segments. */ +#content .api-param { + color: #a3a3a3; + font-weight: 400; +} +.dark #content .api-param { + color: #7d7d85; +} + +/* One hue per method, at a shared saturation so no single row shouts. Reads + as: green retrieves, gold creates, violet edits, blue replaces, red removes. */ +#content .api-get { + --api-fg: #227a5f; + --api-bg: rgba(34, 122, 95, 0.1); +} +#content .api-post { + --api-fg: #a8780a; + --api-bg: rgba(202, 138, 4, 0.12); +} +#content .api-patch { + --api-fg: #7b52c4; + --api-bg: rgba(123, 82, 196, 0.11); +} +#content .api-put { + --api-fg: #2568a8; + --api-bg: rgba(37, 104, 168, 0.11); +} +#content .api-delete { + --api-fg: #b04231; + --api-bg: rgba(176, 66, 49, 0.11); +} +.dark #content .api-get { + --api-fg: #5ecfa6; + --api-bg: rgba(94, 207, 166, 0.13); +} +.dark #content .api-post { + --api-fg: #e8b93f; + --api-bg: rgba(232, 185, 63, 0.14); +} +.dark #content .api-patch { + --api-fg: #b79df0; + --api-bg: rgba(183, 157, 240, 0.14); +} +.dark #content .api-put { + --api-fg: #7db4e6; + --api-bg: rgba(125, 180, 230, 0.14); +} +.dark #content .api-delete { + --api-fg: #ef8b78; + --api-bg: rgba(239, 139, 120, 0.14); +} + +/* ── On-this-page directory ──────────────────────────────────────────────── + A two-line section index where each line is labeled, so a long page can say + which stretch explains and which stretch lists. The bold lead-in is the + label; the links stay ordinary prose links. */ +#content .page-nav span[data-as="p"] { + display: block; + margin-bottom: 0.35rem; +} +#content .page-nav { + margin: 1.5rem 0; + font-size: 0.93em; +} +#content .page-nav strong { + display: inline-block; + min-width: 9.5rem; + text-transform: uppercase; + letter-spacing: 0.08em; + font-size: 0.68rem; + font-weight: 700; + color: #8a8a8a; +} +.dark #content .page-nav strong { + color: #9a9a9a; +} + +/* Section eyebrow: a gold label over a hairline, marking where a page changes + register (the point the REST page stops explaining and starts listing). */ +#content .section-eyebrow { + text-transform: uppercase; + letter-spacing: 0.09em; + font-size: 0.68rem; + font-weight: 700; + color: #a8780a; + border-top: 1px solid #e5e5e5; + padding-top: 1.4rem; + margin-top: 3rem; + margin-bottom: -1.2rem; +} +.dark #content .section-eyebrow { + color: #e8b93f; + border-top-color: rgba(255, 255, 255, 0.1); +} +/* Tighter stack: an index of 5+ endpoints should read as one list. Each closed + row is a single line, so it needs less height than a default accordion. */ +#content .api-endpoints .accordion { + margin-bottom: 0.4rem !important; +} +#content .api-endpoints .accordion > summary { + padding-top: 0.62rem; + padding-bottom: 0.62rem; +} +/* AccordionGroup applies `divide-y!`, which zeroes the top border of every row + but the last one so a flush stack shows single hairlines between rows. These + rows are spaced cards, so the last one ended up outlined while the rest were + open at the top. Give them all four sides. The override has to sit in the + utilities layer: an important declaration inside a layer outranks an + important declaration outside one, whatever the specificity. */ +/* `divide-inherit!` then paints those same rows in the group's border color + rather than their own, which is why the last row read darker in light mode. + Pin every row to the standard card hairline. */ +@layer utilities { + #content .api-endpoints .accordion { + border-width: 1px !important; + border-color: color-mix(in oklab, rgb(var(--gray-200)) 70%, transparent) !important; + } + .dark #content .api-endpoints .accordion { + border-color: rgba(255, 255, 255, 0.1) !important; + } +} +/* Parameter tables sit inside a narrower column than page tables. */ +#content .api-endpoints table { + font-size: 0.85em; +} + diff --git a/docs/docs.json b/docs/docs.json index db492c615..10d946a82 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -79,6 +79,20 @@ ] } ] + }, + { + "tab": "Platform", + "icon": "cloud", + "groups": [ + { + "group": "Start here", + "pages": ["platform/introduction"] + }, + { + "group": "API", + "pages": ["platform/rest-api", "platform/mcp"] + } + ] } ] }, diff --git a/docs/platform/introduction.mdx b/docs/platform/introduction.mdx new file mode 100644 index 000000000..8d1096775 --- /dev/null +++ b/docs/platform/introduction.mdx @@ -0,0 +1,37 @@ +--- +title: "Introduction" +description: "The HUD platform: the hosted service that runs your evals, stores every graded trace, and exposes all of it as one HTTP API." +icon: "compass" +mode: "wide" +--- + +## Motivation + +Evals have to run somewhere. A real suite means hundreds of sandboxes starting at once, agents +acting inside them for minutes at a time, and a graded record of every attempt that still makes +sense a month later. + +The **platform** is the hosted service that does this. You point the [SDK](/v6/start/index) or the +`hud` CLI at it, it places each attempt on its own infrastructure, and it keeps the result as a +trace you can reopen, share, and compare. + +## One API behind everything + +The SDK, the CLI, and [hud.ai](https://hud.ai) are all clients of the same HTTP API, and nothing is +reserved for the UI. Anything the platform can show you, your own code can fetch: a dashboard of +your team's scores, a bot that posts last night's results, a CI check that fails a build when a +score drops. The platform works as a backend you build on, not just a site you visit. + +Base URL `https://api.beta.hud.ai`, every public route under `/v2`, authenticated with a project +`HUD_API_KEY`. + +## Reading the docs + +- [**REST API**](/platform/rest-api) - the whole HTTP surface, grouped by what each part is for, with a request and response example for every endpoint. +- [**MCP**](/platform/mcp) - a read-only Model Context Protocol server so coding agents can query environments, tasksets, jobs, and traces with the same API key. + +For exact schemas and field-level detail, the [generated reference](https://api.beta.hud.ai/docs) +is always in sync with the server. For an agent-readable version, point your agent at +`https://api.beta.hud.ai/openapi.json`, the OpenAPI document both are built from. To attach the +platform as tools in an agent host, use the [Platform MCP](/platform/mcp) at +`https://api.beta.hud.ai/v2/mcp/`. diff --git a/docs/platform/mcp.mdx b/docs/platform/mcp.mdx new file mode 100644 index 000000000..3c7f71280 --- /dev/null +++ b/docs/platform/mcp.mdx @@ -0,0 +1,296 @@ +--- +title: "MCP" +description: "Connect a coding agent to the HUD platform over MCP - setup, authentication, and every read-only tool for environments, tasksets, jobs, and traces." +icon: "magnifying-glass" +mode: "wide" +--- + +The **Platform MCP** is a read-only [Model Context Protocol](https://modelcontextprotocol.io/) +server on the HUD API. Coding agents call it to find environments, tasksets, jobs, and traces +without you pasting IDs by hand. Same project API key as the [REST API](/platform/rest-api); a +different transport (streamable HTTP JSON-RPC at `/v2/mcp/`). + +This is not the `mcp` [capability](/v6/reference/capabilities) you declare inside an +environment. Platform MCP queries platform data. An env capability exposes tools inside a live +sandbox. + +
+ +**Orientation** [Setup](#setup) · [Authentication](#authentication) · [How tools chain](#how-tools-chain) + +**Tool reference** [Entrypoints](#entrypoints) · [Information](#information) · [Runs](#runs) + +
+ +## Setup + +Create a key in [Settings → API Keys](https://hud.ai/project/api-keys). Point your MCP client at +`https://api.beta.hud.ai/v2/mcp/` with that key on every request. + + + +```json Cursor (~/.cursor/mcp.json) +{ + "mcpServers": { + "hud-platform": { + "url": "https://api.beta.hud.ai/v2/mcp/", + "headers": { + "Authorization": "Bearer YOUR_HUD_API_KEY" + } + } + } +} +``` + +```bash Claude Code +claude mcp add --transport http hud-platform \ + https://api.beta.hud.ai/v2/mcp/ \ + --header "Authorization: Bearer $HUD_API_KEY" +``` + +```json Generic MCP client +{ + "hud-platform": { + "url": "https://api.beta.hud.ai/v2/mcp/", + "headers": { + "Authorization": "Bearer YOUR_HUD_API_KEY" + } + } +} +``` + + + +
+ +IF YOU ARE AN AGENT SETTING THIS UP: add an MCP server named +`hud-platform` with URL `https://api.beta.hud.ai/v2/mcp/` and header +`Authorization: Bearer ` (or `X-API-Key: `). Then call +`list_environments` to verify. Do not invent tool names; use the catalog the server returns. +Before creating or editing HUD environments or tasks, install the docs skill if +missing: `npx skills add https://docs.hud.ai` - then follow +`hud-environment-builder`. This server is for catalog and runs only. + +
+ +## Authentication + +Same project `HUD_API_KEY` as REST. Send it as `Authorization: Bearer sk-hud-…` or as +`X-API-Key`. Missing or invalid credentials fail the tool call as unauthorized. + +The server is **read-only**: it can list and fetch, not create jobs, deploy environments, or +mutate tasks. Visibility matches your team's key (same authz as the REST surface). + +## How tools chain + +Tools fall into three groups. **Entrypoints** discover IDs. **Information** is the task catalog. +**Runs** is execution: jobs and what each attempt did. Responses carry chainable IDs +(`environment_id`, `taskset_id`, `job_id`, `trace_id`), so you can jump between catalog and runs. + +| Concept | What it is | +| --- | --- | +| **Environment** | A hosted sandbox image your team can run agents in. | +| **Taskset** | A named bundle of concrete tasks. | +| **Job** | One batch run of a suite (metadata plus rollup stats). | +| **Trace** | One attempt inside a job, from first action to reward. | + +```mermaid +flowchart TB + subgraph EP["Entrypoints"] + LE["list_environments"] + LT["list_tasksets"] + LJ["list_jobs"] + end + + subgraph INF["Information"] + GE["get_environment"] + GTT["get_taskset_tasks"] + end + + subgraph RUN["Runs"] + GJT["get_job_traces"] + GT["get_trace"] + GTE["get_trace_events"] + end + + LE -->|"environment_id"| GE + GE -->|"taskset_id"| GTT + GE -->|"taskset_id"| LJ + LT -->|"taskset_id"| GTT + LT -->|"taskset_id"| LJ + LJ -->|"job_id"| GJT + GJT -->|"trace_id"| GT + GT -->|"seq"| GTE +``` + +Typical paths from zero IDs: + +- `list_environments` → `get_environment` → `get_taskset_tasks` or `list_jobs` +- `list_tasksets` → `get_taskset_tasks` or `list_jobs` +- `list_jobs` → `get_job_traces` → `get_trace` → `get_trace_events` + +List tools page with `limit` (default `20`, max `100`) and `offset`. When more rows remain, the +response includes `next_offset`. Event paging uses `since_seq` instead (see `get_trace_events`). + +
Tool reference
+ +## Entrypoints + +Discovery when you do not have IDs, or when you want to re-enter from a list. + +
+ + + + +Start here when you have no IDs. Optional `search` is a case-insensitive substring match on +environment name. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `search` | `string \| null` | `null` | Substring match on name; omit to list all visible envs. | +| `limit` | `integer` | `20` | Page size (max `100`). | +| `offset` | `integer` | `0` | Row offset. | + +**Returns** paged `{ items, total, next_offset? }` with `id`, `name`, `build_status`, `public`. +**Next:** `get_environment`. + + + + + +Optional `search` matches taskset name. For tasksets on one environment, prefer +`get_environment` (it embeds them). + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `search` | `string \| null` | `null` | Substring match on name. | +| `limit` | `integer` | `20` | Page size (max `100`). | +| `offset` | `integer` | `0` | Row offset. | + +**Returns** paged rows with `id`, `name`, `task_count`, `environment_id`. +**Next:** `get_taskset_tasks` or `list_jobs`. + + + + + +Scan runs across the team, or filter to one taskset. Each row includes a compact traces rollup +(`total_traces`, `avg_reward`, non-zero status counts). + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `taskset_id` | `uuid \| null` | `null` | Restrict to that taskset's jobs. | +| `limit` | `integer` | `20` | Page size (max `100`). | +| `offset` | `integer` | `0` | Row offset. | + +**Returns** paged rows with `id`, `name`, `status`, `taskset_id`, `taskset_name`, `created_at`, +`traces` rollup. +**Next:** `get_job_traces`; `get_taskset_tasks` when a row has `taskset_id`. + + + + +
+ +## Information + +What can run and what each task is. + +
+ + + + +**Templates** are parameterized definitions on the latest build. **Tasksets** are bundles of +concrete tasks. + +| Parameter | Type | Required | Description | +| --- | --- | --- | --- | +| `environment_id` | `uuid` | yes | From `list_environments`. | + +**Returns** `id`, `name`, `description`, `github_url`, `templates`, `latest_build`, `tasksets`. +**Next:** `get_taskset_tasks` or `list_jobs` with a `taskset_id`. + + + + + +Each row includes which template it uses and the args that fill it. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `taskset_id` | `uuid` | required | From entrypoints or `get_environment`. | +| `limit` | `integer` | `20` | Page size (max `100`). | +| `offset` | `integer` | `0` | Row offset. | + +**Returns** paged tasks with `id`, `name`, `description`, `template`, `args`. +**Next:** `list_jobs` for runs on this bundle. + + + + +
+ +## Runs + +How tasks ran: open a job, then a trace, then zoom into events. + +
+ + + + +`limit` / `offset` page that job's attempts, not the job list. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `job_id` | `uuid` | required | From `list_jobs`. | +| `limit` | `integer` | `20` | Trace page size (max `100`). | +| `offset` | `integer` | `0` | Trace offset. | + +**Returns** job header plus `traces: { stats, items, total, next_offset? }`. Each item has `id`, +`status`, `reward`, clipped `error`, `created_at`. +**Next:** `get_trace` on a row from `traces.items`. + + + + + +Pair with `get_trace_events`. The outline and `points_of_interest` use **seq** (the event index) +as the cursor: zoom into seq `K` with `get_trace_events(trace_id, since_seq=K-1)` (`since_seq` is +exclusive). Outline lines tagged `[screenshot]` point at frames; open `screenshot_url` from the +events tool (never inlined). + +| Parameter | Type | Required | Description | +| --- | --- | --- | --- | +| `trace_id` | `uuid` | yes | From `get_job_traces`. | + +**Returns** status, reward, error, task prompt, `trajectory_outline`, `points_of_interest`, and a +short response `guide`. +**Next:** `get_trace_events`. + + + + + +Pick `seq` values from `get_trace`'s outline or `points_of_interest`. Screenshots are HTTPS links +on `tool_call` events (`screenshot_url`), not inline bytes. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `trace_id` | `uuid` | required | Trace to open. | +| `since_seq` | `integer` | `-1` | Exclusive cursor; pass `K-1` to start at seq `K`. | +| `limit` | `integer` | `50` | Events per page (max `100`). | +| `kinds` | `string[] \| null` | `null` | Optional filter, e.g. `["agent_message"]`. | + +**Returns** `events`, a `showing` summary, and `next_since_seq` / `remaining` when more match. +**Next:** continue paging, or return to `get_trace` for the outline. + + + + +
+ +For the full HTTP surface these tools sit beside, see the [REST API](/platform/rest-api). For how +platform objects fit together, see the [platform introduction](/platform/introduction). diff --git a/docs/platform/rest-api.mdx b/docs/platform/rest-api.mdx new file mode 100644 index 000000000..2591f7d5f --- /dev/null +++ b/docs/platform/rest-api.mdx @@ -0,0 +1,4127 @@ +--- +title: "REST API" +description: "Call the HUD platform over HTTP - authentication, the objects the API is built around, and every endpoint grouped by what it is for." +icon: "cloud" +mode: "wide" +--- + +import { Endpoint } from "/snippets/endpoint.jsx"; + +The **REST API** is the HTTP interface to the HUD platform: the hosted service that stores the +environments agents work in, runs agents inside them, and keeps a graded record of every run. The +CLI, the SDK, and the platform UI are all clients of this one surface, so anything they can do, a +client of yours can do too. + +Base URL `https://api.beta.hud.ai`, every public route under `/v2`, authenticated with a project +API key. A first call, returning what an agent actually did on a single attempt: + +```bash +curl https://api.beta.hud.ai/v2/trace//events \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +
+ +**Orientation** [Authentication](#authentication) · [Conventions](#conventions) · [Platform model](#platform-model) + +**Endpoint reference** [Core](#core) · [Runs and evals](#runs-and-evals) · [Secondary](#secondary) · [Utilities](#utilities) · [Secrets](#secrets) · [Account](#account) · [Errors](#errors) + +
+ +## Authentication + +Create a key in [Settings → API Keys](https://hud.ai/project/api-keys) and send it as a bearer +token on every request (see above). + +Missing or invalid credentials return `401`. A valid key that may not touch the resource returns +`403`. + +Three kinds of access appear across the surface. An API key is the default; the groups below call +out where an endpoint differs. + +| Access | What it means | +| --- | --- | +| **API key** | A `HUD_API_KEY` bearer token. Used by scripts, CI, and the SDK. | +| **Session** | A signed-in browser session. A few endpoints accept only this and return `401` for API keys. | +| **Optional** | No credentials required for published jobs and traces. A key widens what you can see. | + +## Conventions + +| Convention | Detail | +| --- | --- | +| **Prefix** | Every public route lives under `/v2`. | +| **Format** | JSON in, JSON out, except file uploads, which are multipart. | +| **IDs** | Resources are addressed by UUID in the path. | +| **Pagination** | List endpoints take `limit` and `offset` and return `{ items, total, limit, offset }`. | +| **Time** | Timestamps are UTC, ISO 8601. | +| **Mutations** | Creates return `201`, deletes return `204`, everything else returns `200`. | + +Every example on this page is generated from the +[OpenAPI document](https://api.beta.hud.ai/openapi.json), so field names match what the server +sends and receives. The values are placeholders. For a console that sends real requests, use the +[interactive reference](https://api.beta.hud.ai/docs). + +## Platform model + +Nothing runs until three things exist: a place for an agent to act, something to ask it to do, and +a model to do it. The API is organized around those objects. + +| Concept | What it is | +| --- | --- | +| **Environment** | The sandboxed application an agent works in, such as a browser or a spreadsheet, packaged as a container image the platform hosts. | +| **Task** | One graded request inside an environment, together with the check that decides whether the agent succeeded. | +| **Taskset** | A named bundle of tasks: the suite you evaluate against. | +| **Job** | One batch run: this suite, these models, this many attempts. | +| **Trace** | The record of a single attempt, from the first message to the reward. | +| **Rollout** | A request to place and run one attempt on hosted infrastructure. | +| **Model** | An entry in the model catalog: a base model, or your team's own fine-tune. | +| **Instance** | A sandbox that is live right now, running one attempt. | + +```mermaid +flowchart LR + E["Environment"] -->|"declares tasks"| T["Taskset"] + T -->|"run with a model"| J["Job"] + J -->|"one attempt each"| R["Traces"] +``` + +Environments and builds are the stage, tasksets and tasks are the script, jobs and rollouts are the +action, and traces are the recording. The remaining groups cover credentials, models, live +capacity, and cost. + +
Endpoint reference
+ +## Core + +The objects you set up before anything runs: the environment an agent will act in, the suite you +grade it on, and the individual tasks inside that suite. + +### Environments + +An **environment** is the sandboxed application an agent works in, packaged as a container image +that HUD hosts. The API calls the catalog of those images the **registry**, and one entry is one +environment: its name, its owner, and the work it knows how to run. Reading the registry is how a +client discovers what it can run against; creating an entry is what [builds](#builds) do. + +Each environment declares **scenarios**, the parameterized task definitions baked into its image. +Tasks point at a scenario, which is how a taskset ends up tied to a specific environment. + +
+ + +}> + +{/* api:GET /v2/registry */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `limit` | query | `integer` | | Defaults to `50`. | +| `offset` | query | `integer` | | Defaults to `0`. | +| `search` | query | `string?` | | ILIKE-match on name | +| `public_only` | query | `boolean` | | Only public registries (Explore tab) Defaults to `False`. | +| `owner_only` | query | `boolean` | | Only registries owned by the caller's membership; ignored when public_only=True Defaults to `False`. | +| `sort_by` | query | `starred \| date \| name \| stars` | | Sort order: starred (caller's starred first), date (updated_at DESC), name (A-Z), stars (star_count DESC) Defaults to `starred`. | + +```bash Request +curl https://api.beta.hud.ai/v2/registry \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "items": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "registry_type": "string", + "public": false, + "created_at": "2026-07-26T17:04:11Z", + "updated_at": "2026-07-26T17:04:11Z", + "description": "string", + "latest_build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "github_url": "string", + "branch": "string", + "entry_folder": "string", + "build_args": {} + } + ], + "total": 0, + "limit": 0, + "offset": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/registry/{registry_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `registry_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/registry/ \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "registry_type": "string", + "public": false, + "created_at": "2026-07-26T17:04:11Z", + "updated_at": "2026-07-26T17:04:11Z", + "description": "string", + "latest_build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "github_url": "string", + "branch": "string", + "entry_folder": "string", + "build_args": {} +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/registry/{registry_id}/scenarios */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `registry_id` | path | `uuid` | yes | | +| `build_id` | query | `uuid?` | | Filter scenarios to a specific build of this registry. Defaults to the registry's latest build when omitted. | + +```bash Request +curl https://api.beta.hud.ai/v2/registry//scenarios \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +[ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "created_at": "2026-07-26T17:04:11Z", + "slug": "string", + "description": "string", + "args_schema": {}, + "arguments": [], + "metadata": {}, + "trace_name_template": "string" + } +] +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/registry/{registry_id}/source-url */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `registry_id` | path | `uuid` | yes | | +| `build_id` | query | `uuid?` | | Download a specific build's source archive. Defaults to the registry's latest build when omitted. | + +```bash Request +curl https://api.beta.hud.ai/v2/registry//source-url \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "download_url": "string", + "github_url": "string" +} +``` + +{/* /api */} + + + + +
+ +### Tasksets + +A **taskset** is a named bundle of tasks and the unit an evaluation is defined against: a public +benchmark, your team's regression suite, a customer's acceptance set. Two jobs are comparable +because they ran the same taskset. + +Two reads answer different questions. Fetching a taskset returns the card: name, task count, +ownership. Exporting returns the tasks themselves, in the same portable shape that +`POST /v2/tasks/upload` accepts, which makes export and upload a round trip you can use to clone a +suite, back it up, or sync it into CI. + + +Export requires an API key; a browser session alone returns `401`. No public endpoint lists +tasksets today, so resolve them by name or by ID. + + +
+ + +}> + +{/* api:GET /v2/tasksets/{taskset_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `taskset_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasksets/ \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "public": false, + "created_at": "2026-07-26T17:04:11Z", + "updated_at": "2026-07-26T17:04:11Z", + "description": "string", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "purpose": "EVAL", + "star_count": 0, + "task_count": 0, + "sample_unit_count": 0, + "system_prompt": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/tasksets/{taskset_id}/export */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `taskset_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasksets//export \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "tasks": [ + { + "name": "string", + "env": "string", + "scenario": "string", + "args": {}, + "validation": [ + {} + ], + "agent_config": {}, + "runtime_config": {} + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/tasksets/by-name/{name} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `name` | path | `string` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasksets/by-name/ \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "tasks": [ + { + "name": "string", + "env": "string", + "scenario": "string", + "args": {}, + "validation": [ + {} + ], + "agent_config": {}, + "runtime_config": {} + } + ] +} +``` + +{/* /api */} + + + + +
+ +### Tasks + +A **task** is one graded request inside an environment plus the check that decides whether the +agent succeeded. Tasks are what a job actually runs. Most of this group exists because tasks are +written and reviewed by people before anyone trusts them to grade a model. + +A task begins as a **brief**: a prompt and any attachments describing the idea, before it is backed +by a runnable scenario. Once it is real, it carries a **status**, the review state everyone reads. + +| Status | Meaning | +| --- | --- | +| `draft` | An idea captured as a brief. Not runnable. | +| `pending` | Being built. | +| `ready` | The author considers it finished. | +| `verified`, `revision`, `rejected` | The reviewer's decision. | + +While a task is `pending`, its taskset can define **stages**: an optional checklist inside that one +status, such as write, then QA, then polish. A task advances one stage at a time, or jumps back +when someone returns it for more work, and it must reach the last stage before it can become +`ready`. Tasksets with no stages configured ignore this machinery entirely. + +A **ready-check** evaluates the taskset's submission requirements against a task: enough graded +traces, accuracy thresholds, particular models, QA checks. It reports each requirement as passed or +failed along with what is missing. It reports; it does not change the status. + +**Comments** and **activity** are the review trail: human discussion on one side, and an audit +timeline of version and status changes on the other. + +#### Authoring and review state + +
+ + +}> + +{/* api:POST /v2/tasks/upload */} + +```bash Request +curl https://api.beta.hud.ai/v2/tasks/upload \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "taskset_name": "string", + "tasks": [ + { + "name": "string", + "description": "string", + "external_id": "string", + "env": {}, + "task_id": "string", + "scenario": "string", + "args": {}, + "validation": [ + {} + ], + "system_prompt": "string", + "agent_config": {}, + "runtime_config": { + "image": "string", + "resources": {}, + "limits": {} + }, + "scenario_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + ], + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "description": "string", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + }' +``` + +```json Response 201 +{ + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "task_count": 0, + "task_version_ids": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ], + "tasks_created": 0, + "tasks_updated": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/tasks/status */} + +```bash Request +curl https://api.beta.hud.ai/v2/tasks/status \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "ids": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ], + "slugs": [ + "string" + ], + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "pending", + "clear": false, + "reviewed": false + }' +``` + +```json Response 200 +{ + "succeeded": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ], + "unchanged": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ], + "failures": [ + { + "input": "string", + "reason": "string", + "code": "not_found", + "resolved_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/tasks/stage */} + +```bash Request +curl https://api.beta.hud.ai/v2/tasks/stage \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "ids": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ], + "stage": "string" + }' +``` + +```json Response 200 +{ + "succeeded": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ], + "unchanged": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ], + "failures": [ + { + "input": "string", + "reason": "string", + "code": "not_found", + "resolved_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/tasks/{task_id}/ready-check */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//ready-check \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "passed": false, + "k": 0, + "candidate_count": 0, + "evaluated_at": "2026-07-26T17:04:11Z", + "evaluated_trace_ids": [ + "string" + ], + "requirements_version": 1, + "checks": [ + { + "key": "string", + "label": "string", + "passed": false, + "required": "string", + "current": "string", + "how_to_fix": "string", + "severity": "error", + "offending_trace_ids": [ + "string" + ] + } + ], + "can_override": true, + "has_requirements": true +} +``` + +{/* /api */} + + + + +
+ +#### Briefs + +Briefs belong to draft tasks only. These three routes are written for the platform UI and accept a +browser session, so an API key returns `401`. + +
+ + +}> + +{/* api:GET /v2/tasks/{task_id}/brief */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//brief \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "prompt": "string", + "attachments": [ + { + "name": "string", + "url": "string", + "storage_url": "string", + "size": 0, + "content_type": "string", + "status": "staged", + "error": "string", + "archived_at": "2026-07-26T17:04:11Z" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/tasks/{task_id}/brief */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//brief \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "string", + "attachments": [ + { + "name": "string", + "url": "string", + "size": 0, + "content_type": "string" + } + ] + }' +``` + +```json Response 200 +{ + "prompt": "string", + "attachments": [ + { + "name": "string", + "url": "string", + "storage_url": "string", + "size": 0, + "content_type": "string", + "status": "staged", + "error": "string", + "archived_at": "2026-07-26T17:04:11Z" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/tasks/{task_id}/brief/download */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//brief/download \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +Returns `200` with a file body. + +{/* /api */} + + + + +
+ +#### Comments and activity + +
+ + +}> + +{/* api:GET /v2/tasks/{task_id}/comments */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//comments \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "comments": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "user_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "message": "string", + "created_at": "2026-07-26T17:04:11Z", + "updated_at": "2026-07-26T17:04:11Z", + "user_name": "string", + "avatar_url": "string" + } + ], + "can_comment": false, + "reason": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/tasks/{task_id}/comments */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//comments \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "body": "string" + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "user_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "message": "string", + "created_at": "2026-07-26T17:04:11Z", + "updated_at": "2026-07-26T17:04:11Z", + "user_name": "string", + "avatar_url": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/tasks/{task_id}/comments/{comment_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | +| `comment_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//comments/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "body": "string" + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "user_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "message": "string", + "created_at": "2026-07-26T17:04:11Z", + "updated_at": "2026-07-26T17:04:11Z", + "user_name": "string", + "avatar_url": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:DELETE /v2/tasks/{task_id}/comments/{comment_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | +| `comment_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//comments/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X DELETE +``` + +Returns `204` with an empty body. + +{/* /api */} + + + +}> + +{/* api:GET /v2/tasks/{task_id}/activity */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `task_id` | path | `uuid` | yes | | +| `limit` | query | `integer` | | Defaults to `50`. | + +```bash Request +curl https://api.beta.hud.ai/v2/tasks//activity \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "events": [ + { + "event": "string", + "at": "string", + "by": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "by_name": "string", + "by_role": "string", + "key": "string", + "done": false, + "summary": "string", + "version": 0 + } + ] +} +``` + +{/* /api */} + + + + +
+ +## Runs and evals + +What happens once you actually run something. Rollouts start the work, a job groups it, a trace +records each attempt, and an instance is the sandbox an attempt is running in right now. + +### Jobs + +A **job** is one batch of work: this taskset, these models, this many attempts, grouped so the +results can be read as a single score. Every graded attempt belongs to a job. + +There is no `POST /v2/jobs`. Jobs come into existence when work is launched, either through +[rollouts](#rollouts) or when the SDK enters a job as it starts running, and they are cancelled +through rollouts as well. What lives here is everything you do with a job once it exists: read it, +summarize it, page through its attempts, and decide whether its results count. + +Single-job reads use optional authentication, so a published job is readable without a key. + +#### Core resource + +
+ + +}> + +{/* api:GET /v2/jobs */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `limit` | query | `integer` | | Defaults to `50`. | +| `offset` | query | `integer` | | Defaults to `0`. | +| `taskset_id` | query | `uuid?` | | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "items": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "max_steps": 0, + "group_size": 0, + "is_remote": false, + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "description": "string", + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "max_concurrent": 0, + "metadata": { + "resolved_agents": [], + "sdk_agent_params": {}, + "parent_job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + } + ], + "total": 0, + "limit": 0, + "offset": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/jobs/{job_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs/ \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "max_steps": 0, + "group_size": 0, + "is_remote": false, + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "description": "string", + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "max_concurrent": 0, + "metadata": { + "resolved_agents": [], + "sdk_agent_params": {}, + "parent_job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/jobs/{job_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "name": "string", + "description": "string" + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "max_steps": 0, + "group_size": 0, + "is_remote": false, + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "description": "string", + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "max_concurrent": 0, + "metadata": { + "resolved_agents": [], + "sdk_agent_params": {}, + "parent_job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } +} +``` + +{/* /api */} + + + + +
+ +#### Aggregates + +Rollups over the job's attempts, so a client does not have to fetch every trace and add the numbers +up itself. + +
+ + +}> + +{/* api:GET /v2/jobs/{job_id}/summary */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//summary \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "total_traces": 0, + "total_tasks": 0, + "completed": 0, + "running": 0, + "pending": 0, + "initializing": 0, + "cancelling": 0, + "error": 0, + "cancelled": 0, + "avg_reward": 0.0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/jobs/{job_id}/usage */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//usage \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "calculated_at": "2026-07-26T17:04:11Z", + "job_name": "string", + "trace_count": 0, + "environment_cost": { + "cost": 0.0, + "count": 0, + "details": {} + }, + "inference_cost": { + "cost": 0.0, + "count": 0, + "details": {} + }, + "inference_cost_incomplete": false, + "total_cost": 0.0, + "training_cost": { + "total_cost": 0.0, + "hourly_rate": 0.0, + "runtime_seconds": 0.0 + }, + "traces": [ + { + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2026-07-26T17:04:11Z", + "end_time": "2026-07-26T17:04:11Z", + "status": "string", + "environment_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "environment_cost": 0.0, + "environment_transactions": 0, + "environment_hourly_rate": 0.0, + "environment_still_running": false, + "environment_baseline_cost": 0.0, + "environment_additional_cost": 0.0, + "environment_baseline_minutes": 5 + } + ], + "environments_still_running": 0, + "note": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/jobs/{job_id}/taskset-coverage */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//taskset-coverage \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "tasks": [ + { + "task_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_name": "string", + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "trace_count": 0, + "avg_reward": 0.0, + "statuses": [ + "string" + ], + "invalidated_count": 0 + } + ], + "total_tasks": 0, + "tasks_with_traces": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/jobs/{job_id}/chart-data */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | +| `limit` | query | `integer` | | Defaults to `5000`. | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//chart-data \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "data_points": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "reward": 0.0, + "is_valid": true, + "created_at": "2026-07-26T17:04:11Z", + "end_time": "2026-07-26T17:04:11Z", + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + ], + "total": 0 +} +``` + +{/* /api */} + + + + +
+ +#### Attempts in the job + +Two views of the same attempts. Traces are the full records, suited to scripts. Thumbnails are the +compact tiles the platform's grid renders, with counts per outcome attached. + +
+ + +}> + +{/* api:GET /v2/jobs/{job_id}/traces */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | +| `limit` | query | `integer` | | Defaults to `50`. | +| `offset` | query | `integer` | | Defaults to `0`. | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//traces \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "items": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_slug": "string", + "instance_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "inference_llm_model_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "reward": 0.0, + "stop_reason": "done", + "views": 0, + "error": "string" + } + ], + "total": 0, + "limit": 0, + "offset": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/jobs/{job_id}/thumbnails */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | +| `limit` | query | `integer` | | Defaults to `100`. | +| `offset` | query | `integer` | | Defaults to `0`. | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//thumbnails \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "thumbnails": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "job_name": "string", + "reward": 0.0, + "is_valid": true, + "error": "string", + "start_time": "2026-07-26T17:04:11Z", + "end_time": "2026-07-26T17:04:11Z", + "views": 0, + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + ], + "next_offset": 0, + "has_more": false, + "category_counts": { + "all": 0, + "live": 0, + "successful": 0, + "failed": 0, + "no_score": 0 + } +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/jobs/{job_id}/entity-links */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//entity-links \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "links": [ + { + "registry": { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string" + }, + "taskset": { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string" + }, + "model": { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string" + } + } + ] +} +``` + +{/* /api */} + + + + +
+ +#### List projections + +Shapes built for dashboards: the jobs still running, and job cards with their scores already rolled +up so a list of jobs renders in one round trip. + +
+ + +}> + +{/* api:GET /v2/jobs/active */} + +```bash Request +curl https://api.beta.hud.ai/v2/jobs/active \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +[ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "max_steps": 0, + "group_size": 0, + "is_remote": false, + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "description": "string", + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "max_concurrent": 0, + "metadata": { + "resolved_agents": [], + "sdk_agent_params": {}, + "parent_job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + } +] +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/jobs/enriched */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `taskset_id` | query | `uuid` | yes | | +| `limit` | query | `integer` | | Defaults to `50`. | +| `offset` | query | `integer` | | Defaults to `0`. | + +```bash Request +curl "https://api.beta.hud.ai/v2/jobs/enriched?taskset_id=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "jobs": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "average_accuracy": 0.0, + "total_tasks": 0, + "completed_tasks": 0, + "failed_count": 0, + "invalidated_count": 0, + "user_name": "string", + "user_avatar_url": "string", + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + ], + "total_count": 0, + "taskset_tasks": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/jobs/enrich */} + +```bash Request +curl https://api.beta.hud.ai/v2/jobs/enrich \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '[ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ]' +``` + +```json Response 200 +{ + "enrichments": {} +} +``` + +{/* /api */} + + + + +
+ +#### State + +Publishing makes a job readable without a key. Invalidating marks every attempt in it as not +counting toward scores, which is how a run spoiled by a broken environment or a bad prompt is taken +out of the record without deleting it. + +
+ + +}> + +{/* api:POST /v2/jobs/{job_id}/publish */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//publish \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 200 +{ + "message": "string", + "public_url": "string", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "success": true +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/jobs/{job_id}/invalidate */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//invalidate \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 200 +{ + "updated_count": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/jobs/{job_id}/revalidate */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `job_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/jobs//revalidate \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 200 +{ + "updated_count": 0 +} +``` + +{/* /api */} + + + + +
+ +### Rollouts + +A **rollout** is one attempt placed on hosted infrastructure: pick up a task, start a sandbox, run +the agent, record the result. This group is the go and stop switch for that work, and nothing else. +Reading results happens under [jobs](#jobs) and [traces](#traces). + +Launching returns immediately with a `job_id` and a `trace_id`. The work itself is queued, so a +client polls the trace, or the job, until it finishes. + +The three launch endpoints differ in what they are given. Running one task needs a task version and +a model. A batch takes lists of both and groups the result into a single job. Submitting is the +path the SDK uses when it has already minted its own identifiers and describes the work by +environment and task name instead. + +#### Launch + +
+ + +}> + +{/* api:POST /v2/rollouts/run */} + +```bash Request +curl https://api.beta.hud.ai/v2/rollouts/run \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "model_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "max_steps": 100, + "trace_name": "string", + "runtime_config": { + "image": "string", + "resources": { + "cpu": 0.0, + "memory_mb": 0, + "gpu": { + "type": "string", + "count": 1 + } + }, + "limits": { + "startup_timeout_s": 0, + "run_timeout_s": 0 + } + } + }' +``` + +```json Response 201 +{ + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "queued" +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/rollouts/run_list */} + +```bash Request +curl https://api.beta.hud.ai/v2/rollouts/run_list \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 201 +{ + "total": 0, + "accepted": 0, + "rejected": 0, + "results": [ + {} + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/rollouts/submit */} + +```bash Request +curl https://api.beta.hud.ai/v2/rollouts/submit \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "env": "string", + "task": "string", + "agent": { + "type": "claude", + "config": {} + }, + "group_id": "string", + "slug": "string", + "args": {}, + "runtime_config": { + "image": "string", + "resources": { + "cpu": 0.0, + "memory_mb": 0, + "gpu": { + "type": "string", + "count": 1 + } + }, + "limits": { + "startup_timeout_s": 0, + "run_timeout_s": 0 + } + } + }' +``` + +```json Response 201 +{ + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "queued" +} +``` + +{/* /api */} + + + + +
+ +#### Cancel + +Cancelling is scoped: one attempt, one job, or everything you have running. + +
+ + +}> + +{/* api:POST /v2/rollouts/cancel */} + +```bash Request +curl https://api.beta.hud.ai/v2/rollouts/cancel \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + }' +``` + +```json Response 200 +{ + "status": "accepted" +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/rollouts/cancel_job */} + +```bash Request +curl https://api.beta.hud.ai/v2/rollouts/cancel_job \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + }' +``` + +```json Response 200 +{ + "cancelled": 0, + "status": "accepted" +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/rollouts/cancel_user_jobs */} + +```bash Request +curl https://api.beta.hud.ai/v2/rollouts/cancel_user_jobs \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 200 +{ + "jobs_cancelled": 0, + "total_tasks_cancelled": 0, + "job_details": [ + { + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "cancelled": 0 + } + ], + "status": "accepted" +} +``` + +{/* /api */} + + + + +
+ +### Traces + +A **trace** is one attempt at one task: what the agent saw, every tool call it made, what the +environment said back, how long it took, what it cost, and the reward it ended with. Traces are the +evidence behind every score on the platform. + +The record itself is small. The interesting read is **events**, a flat and typed projection of the +attempt's telemetry into messages, tool calls, and results, ordered by a sequence number. Passing +the last `latest_seq` you saw as `since_seq` returns only what is new, which is how a viewer +follows a run that is still going. + +Most reads use optional authentication, so a trace inside a published job is readable without a +key. Writes and the analysis bundle require a key or a session. + +#### Core resource + +
+ + +}> + +{/* api:GET /v2/trace/{trace_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/trace/ \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_slug": "string", + "instance_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "inference_llm_model_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "reward": 0.0, + "stop_reason": "done", + "views": 0, + "error": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/trace/{trace_id}/info */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/trace//info \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "execution_mode": "external", + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_slug": "string", + "task_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "instance_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "inference_llm_model_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "reward": 0.0, + "stop_reason": "done" +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/trace/{trace_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/trace/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "name": "string", + "description": "string" + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "execution_mode": "external", + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_slug": "string", + "task_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "instance_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "inference_llm_model_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "reward": 0.0, + "stop_reason": "done" +} +``` + +{/* /api */} + + + + +
+ +#### Telemetry and logs + +Three levels of detail: the projected trajectory, the runner's own logs, and the raw output of the +environment container. The analysis bundle packages all of it for an agent that has been asked why +a run failed. + +
+ + +}> + +{/* api:GET /v2/trace/{trace_id}/events */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | +| `since_seq` | query | `integer` | | Return only events derived from spans with sequence > since_seq. Use -1 (default) for the full trajectory; set to the previously received `latest_seq` for incremental polling. | + +```bash Request +curl https://api.beta.hud.ai/v2/trace//events \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "events": [ + { + "id": "string", + "text": "string", + "parent_id": "string", + "started_at": "2026-07-26T17:04:11Z", + "ended_at": "2026-07-26T17:04:11Z", + "seq": 0, + "kind": "user_message" + } + ], + "latest_seq": 0, + "status": "string", + "reward": 0.0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/trace/{trace_id}/logs */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/trace//logs \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "logs": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/trace/{trace_id}/env-logs */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/trace//env-logs \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "logs": [ + { + "stream": "string", + "log": "string", + "time": "string" + } + ], + "count": 0, + "has_more": false, + "last_timestamp": 0, + "still_running": false +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/trace/{trace_id}/analysis-context */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | +| `include_spans` | query | `boolean` | | Defaults to `True`. | +| `include_env_logs` | query | `boolean` | | Defaults to `True`. | +| `include_worker_logs` | query | `boolean` | | Defaults to `True`. | + +```bash Request +curl https://api.beta.hud.ai/v2/trace//analysis-context \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "task_external_id": "string", + "task_name": "string", + "reward": 0.0, + "error": "string", + "prompt": "string", + "scenario_name": "string", + "scenario_args": {} +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/trace/{trace_id}/usage */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/trace//usage \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2026-07-26T17:04:11Z", + "end_time": "2026-07-26T17:04:11Z", + "status": "string", + "environment_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "environment_cost": 0.0, + "environment_transactions": 0, + "environment_hourly_rate": 0.0, + "environment_still_running": false, + "environment_baseline_cost": 0.0, + "environment_additional_cost": 0.0, + "environment_baseline_minutes": 5 +} +``` + +{/* /api */} + + + + +
+ +#### Related traces and validity + +
+ + +}> + +{/* api:GET /v2/trace/{trace_id}/related */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `trace_id` | path | `uuid` | yes | | +| `kind` | query | `job \| task \| recent \| discover` | yes | `job` (same job_id), `task` (every version of the same task), `recent` (caller's own traces, requires auth), or `discover` (most-recent visible traces overall). | +| `limit` | query | `integer` | | Defaults to `50`. | +| `scope_to_job` | query | `boolean` | | Only meaningful with `kind=task`: further restrict to the source trace's job. Used by the info-card sibling-run squares. Defaults to `False`. | +| `status` | query | `string?` | | Comma-separated status whitelist applied to `Trace.status` - e.g. `completed` to exclude live/cancelled rows. | +| `order` | query | `asc \| desc` | | Sort by created_at. | + +```bash Request +curl "https://api.beta.hud.ai/v2/trace//related?kind=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +[ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "job_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "job_name": "string", + "reward": 0.0, + "is_valid": true, + "error": "string", + "start_time": "2026-07-26T17:04:11Z", + "end_time": "2026-07-26T17:04:11Z", + "views": 0, + "task_version_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } +] +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/trace/batch-validate */} + +```bash Request +curl https://api.beta.hud.ai/v2/trace/batch-validate \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "trace_ids": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ], + "is_valid": false + }' +``` + +```json Response 200 +{ + "updated_count": 0 +} +``` + +{/* /api */} + + + + +
+ +### Instances + +An **instance** is a sandbox that is live right now, running one attempt of one environment. Each +one holds real compute, so instances are the surface you reach for when a run has hung, when +something is stuck holding capacity, or when you want to see what is currently occupied. + +
+ + +}> + +{/* api:GET /v2/instance */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `registry_id` | query | `uuid` | yes | Filter instances to this registry | +| `include_terminated` | query | `boolean` | | Include recently terminated instances Defaults to `False`. | +| `limit` | query | `integer` | | Defaults to `50`. | +| `offset` | query | `integer` | | Defaults to `0`. | + +```bash Request +curl "https://api.beta.hud.ai/v2/instance?registry_id=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "instances": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "terminated_at": "2026-07-26T17:04:11Z", + "public_ip": "string", + "ec2_instance_type": "string", + "max_runtime_seconds": 0, + "user_name": "string", + "user_avatar_url": "string", + "actual_cost": 0.0 + } + ], + "total": 0, + "running_count": 0, + "terminated_count": 0, + "has_more": false +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/instance/stop */} + +```bash Request +curl https://api.beta.hud.ai/v2/instance/stop \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "instance_ids": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ] + }' +``` + +```json Response 200 +{ + "stopped": 0, + "errors": [ + "string" + ] +} +``` + +{/* /api */} + + + + +
+ +## Secondary + +Two supporting catalogs behind the core loop: how an environment image gets made, and which models +you can point at it. + +### Builds + +A **build** turns environment source into the container image that runs attempts. The shape of it +is the same as shipping any image: ask where to put the source, upload it, start the build, watch +it, then decide whether the result becomes the version new runs use. + +Uploading is a two-step handshake. Requesting an upload URL returns a `build_id` and a one-time +link; your client sends the source archive to that link directly, then triggers the build by ID. +Marking a build as latest is a separate call, which leaves earlier builds in place to roll back to. + +#### Lifecycle + +
+ + +}> + +{/* api:POST /v2/builds/upload-url */} + +```bash Request +curl https://api.beta.hud.ai/v2/builds/upload-url \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 200 +{ + "upload_url": "string", + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/builds/trigger */} + +```bash Request +curl https://api.beta.hud.ai/v2/builds/trigger \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "source": "string", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "environment_variables": {}, + "build_secrets": {}, + "runtime_provider": "ec2", + "runtime_config": { + "image": "string", + "resources": { + "cpu": 0.0, + "memory_mb": 0, + "gpu": { + "type": "string", + "count": 1 + } + }, + "limits": { + "startup_timeout_s": 0, + "run_timeout_s": 0 + } + }, + "entry_folder": "string", + "build_args": {}, + "no_cache": false + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "source_index_status": "pending", + "created_at": "2026-07-26T17:04:11Z", + "version": 0, + "uri": "string", + "image_name": "string", + "digest": "string", + "lock": { + "tools": [], + "prompts": [], + "resources": [], + "transport": "http" + }, + "manifest": { + "env": { + "name": "string", + "version": "0.0.0" + }, + "capabilities": [], + "tasks": [] + }, + "duration_seconds": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/builds/trigger-direct */} + +```bash Request +curl https://api.beta.hud.ai/v2/builds/trigger-direct \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "source": "string", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "environment_variables": {}, + "build_secrets": {}, + "runtime_provider": "ec2", + "runtime_config": { + "image": "string", + "resources": { + "cpu": 0.0, + "memory_mb": 0, + "gpu": { + "type": "string", + "count": 1 + } + }, + "limits": { + "startup_timeout_s": 0, + "run_timeout_s": 0 + } + }, + "entry_folder": "string", + "build_args": {}, + "no_cache": false + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "source_index_status": "pending", + "created_at": "2026-07-26T17:04:11Z", + "version": 0, + "uri": "string", + "image_name": "string", + "digest": "string", + "lock": { + "tools": [], + "prompts": [], + "resources": [], + "transport": "http" + }, + "manifest": { + "env": { + "name": "string", + "version": "0.0.0" + }, + "capabilities": [], + "tasks": [] + }, + "duration_seconds": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/builds */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `registry_id` | query | `uuid` | yes | Filter builds to this registry | +| `limit` | query | `integer` | | Defaults to `50`. | + +```bash Request +curl "https://api.beta.hud.ai/v2/builds?registry_id=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "builds": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "source_index_status": "pending", + "created_at": "2026-07-26T17:04:11Z", + "version": 0, + "uri": "string", + "image_name": "string", + "digest": "string", + "lock": { + "tools": [], + "prompts": [], + "resources": [], + "transport": "http" + }, + "manifest": { + "env": {}, + "capabilities": [], + "tasks": [] + }, + "duration_seconds": 0 + } + ], + "total": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/builds/{build_id}/status */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/builds//status \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "source_index_status": "pending", + "created_at": "2026-07-26T17:04:11Z", + "version": 0, + "uri": "string", + "image_name": "string", + "digest": "string", + "lock": { + "tools": [], + "prompts": [], + "resources": [], + "transport": "http" + }, + "manifest": { + "env": { + "name": "string", + "version": "0.0.0" + }, + "capabilities": [], + "tasks": [] + }, + "duration_seconds": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/builds/{build_id}/cancel */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/builds//cancel \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "source_index_status": "pending", + "created_at": "2026-07-26T17:04:11Z", + "version": 0, + "uri": "string", + "image_name": "string", + "digest": "string", + "lock": { + "tools": [], + "prompts": [], + "resources": [], + "transport": "http" + }, + "manifest": { + "env": { + "name": "string", + "version": "0.0.0" + }, + "capabilities": [], + "tasks": [] + }, + "duration_seconds": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/builds/{build_id}/set-latest */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/builds//set-latest \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "source_index_status": "pending", + "created_at": "2026-07-26T17:04:11Z", + "version": 0, + "uri": "string", + "image_name": "string", + "digest": "string", + "lock": { + "tools": [], + "prompts": [], + "resources": [], + "transport": "http" + }, + "manifest": { + "env": { + "name": "string", + "version": "0.0.0" + }, + "capabilities": [], + "tasks": [] + }, + "duration_seconds": 0 +} +``` + +{/* /api */} + + + + +
+ +#### Source and workspace + +After a build indexes its context, its files can be browsed: the source that was uploaded, and the +workspace the build produced. The platform uses this to answer "what did we actually ship in this +image?" without a rebuild. + +
+ + +}> + +{/* api:GET /v2/builds/{build_id}/source */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | +| `path` | query | `string` | | Defaults to ``. | +| `cursor` | query | `string?` | | | +| `limit` | query | `integer` | | Defaults to `100`. | + +```bash Request +curl https://api.beta.hud.ai/v2/builds//source \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "pending", + "path": "string", + "error": "string", + "entries": [], + "next_cursor": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/builds/{build_id}/source/preview */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | +| `path` | query | `string` | yes | | + +```bash Request +curl "https://api.beta.hud.ai/v2/builds//source/preview?path=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "path": "string", + "content": "string", + "size": 0, + "content_hash": "string", + "content_type": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/builds/{build_id}/source/download */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | +| `path` | query | `string` | yes | | + +```bash Request +curl "https://api.beta.hud.ai/v2/builds//source/download?path=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "path": "string", + "url": "string", + "expires_in": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/builds/{build_id}/workspace/preview */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | +| `path` | query | `string` | yes | | + +```bash Request +curl "https://api.beta.hud.ai/v2/builds//workspace/preview?path=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "path": "string", + "content": "string", + "size": 0, + "content_hash": "string", + "content_type": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/builds/{build_id}/workspace/download */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/builds//workspace/download \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +Returns `200` with a file body. + +{/* /api */} + + + + +
+ +#### Deprecated + +
+ + +}> + +Production discovers an image's scenarios automatically when the build finishes. This route runs +the image on the calling host instead, and exists only for local debugging. + +{/* api:POST /v2/builds/{build_id}/discover-scenarios */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `build_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/builds//discover-scenarios \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST +``` + +```json Response 200 +[ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "build_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "created_at": "2026-07-26T17:04:11Z", + "slug": "string", + "description": "string", + "args_schema": {}, + "arguments": [], + "metadata": {}, + "trace_name_template": "string" + } +] +``` + +{/* /api */} + + + + +
+ +### Models + +A **model** is an entry in the catalog you point evaluations at. Two kinds share the resource: base +models the platform provides, and trainable models your team owns. A team model is created by +**forking** a trainable base, which gives it its own line of **checkpoints**, one of which is the +**head** that inference uses. + +Names are resolved rather than guessed: pass whatever model string you have, such as a slug from a +config file, and resolve turns it into a catalog record with an ID. Results are the other end of +the loop, reporting how the model scored on the tasksets it has been run against. + +#### Catalog + +
+ + +}> + +{/* api:GET /v2/models */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `limit` | query | `integer` | | Defaults to `100`. | +| `offset` | query | `integer` | | Defaults to `0`. | + +```bash Request +curl https://api.beta.hud.ai/v2/models \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "items": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "model_name": "string", + "sdk_agent_type": "string", + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "provider": { + "id": 0, + "name": "string", + "api_base": "string", + "supported_routes": [], + "icon_url": "string", + "metadata": {} + }, + "routes": [], + "is_beta": false, + "hidden": false, + "is_trainable": false, + "public": true, + "checkpoint_count": 0 + } + ], + "total": 0, + "limit": 0, + "offset": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/models/resolve */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model` | query | `string` | yes | Model slug (or id) to resolve to its catalog record | + +```bash Request +curl "https://api.beta.hud.ai/v2/models/resolve?model=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "model_name": "string", + "sdk_agent_type": "string", + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "provider": { + "id": 0, + "name": "string", + "api_base": "string", + "supported_routes": [], + "icon_url": "string", + "metadata": {} + }, + "routes": [], + "is_beta": false, + "hidden": false, + "is_trainable": false, + "public": true, + "checkpoint_count": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/models/{model_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/models/ \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "model_name": "string", + "sdk_agent_type": "string", + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "provider": { + "id": 0, + "name": "string", + "api_base": "string", + "supported_routes": [], + "icon_url": "string", + "metadata": {} + }, + "routes": [], + "is_beta": false, + "hidden": false, + "is_trainable": false, + "public": true, + "checkpoint_count": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/models/{model_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/models/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "name": "string", + "hidden": false, + "request_defaults": {} + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "model_name": "string", + "sdk_agent_type": "string", + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "provider": { + "id": 0, + "name": "string", + "api_base": "string", + "supported_routes": [], + "icon_url": "string", + "metadata": {} + }, + "routes": [], + "is_beta": false, + "hidden": false, + "is_trainable": false, + "public": true, + "checkpoint_count": 0 +} +``` + +{/* /api */} + + + + +
+ +#### Training lineage + +
+ + +}> + +{/* api:POST /v2/models/fork */} + +```bash Request +curl https://api.beta.hud.ai/v2/models/fork \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "source_model_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string" + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "model_name": "string", + "sdk_agent_type": "string", + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "provider": { + "id": 0, + "name": "string", + "api_base": "string", + "supported_routes": [], + "icon_url": "string", + "metadata": {} + }, + "routes": [], + "is_beta": false, + "hidden": false, + "is_trainable": false, + "public": true, + "checkpoint_count": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/models/{model_id}/checkpoints */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/models//checkpoints \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +[ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "checkpoint_name": "string", + "prev_model_checkpoint_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "is_active": false, + "num_traces": 0, + "num_datums": 0, + "num_tokens": 0, + "mean_reward": 0.0, + "learning_rate": 0.0, + "loss_fn": "string" + } +] +``` + +{/* /api */} + + + +}> + +{/* api:PUT /v2/models/{model_id}/head */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/models//head \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PUT \ + -H "Content-Type: application/json" \ + -d '{ + "checkpoint_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "model_name": "string", + "sdk_agent_type": "string", + "created_at": "2026-07-26T17:04:11Z", + "name": "string", + "provider": { + "id": 0, + "name": "string", + "api_base": "string", + "supported_routes": [], + "icon_url": "string", + "metadata": {} + }, + "routes": [], + "is_beta": false, + "hidden": false, + "is_trainable": false, + "public": true, + "checkpoint_count": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/models/{model_id}/checkpoints/{checkpoint_id}/traces */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model_id` | path | `uuid` | yes | | +| `checkpoint_id` | path | `uuid` | yes | | +| `limit` | query | `integer` | | Defaults to `200`. | + +```bash Request +curl https://api.beta.hud.ai/v2/models//checkpoints//traces \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +[ + { + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "status": "string", + "created_at": "2026-07-26T17:04:11Z", + "reward": 0.0 + } +] +``` + +{/* /api */} + + + + +
+ +#### Results and usage + +
+ + +}> + +{/* api:GET /v2/models/{model_id}/results */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model_id` | path | `uuid` | yes | | +| `checkpoint_ids` | query | `string?` | | | +| `purpose` | query | `EVAL \| TRAINING?` | | | + +```bash Request +curl https://api.beta.hud.ai/v2/models//results \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "model_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "results": [ + { + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "is_public": false, + "total_tasks": 0, + "tasks_with_traces": 0, + "avg_score": 0.0, + "tasks_in_range_0_99": 0, + "tasks_in_range_5_75": 0, + "tasks_in_range_15_40": 0, + "min_traces_per_task": 0, + "max_traces_per_task": 0, + "per_task_scores": [ + {} + ] + } + ], + "suggested": [ + { + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "is_public": false, + "task_count": 0, + "purpose": "string", + "updated_at": "2026-07-26T17:04:11Z" + } + ], + "checkpoint_ids": [] +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/models/{model_id}/suggested-tasksets */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model_id` | path | `uuid` | yes | | +| `exclude` | query | `string?` | | | +| `search` | query | `string?` | | | +| `limit` | query | `integer` | | Defaults to `50`. | +| `purpose` | query | `EVAL \| TRAINING?` | | | + +```bash Request +curl https://api.beta.hud.ai/v2/models//suggested-tasksets \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "suggested": [ + { + "taskset_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "taskset_name": "string", + "is_public": false, + "task_count": 0, + "purpose": "string", + "updated_at": "2026-07-26T17:04:11Z" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/models/{model_id}/logs */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `model_id` | path | `uuid` | yes | | +| `limit` | query | `integer` | | Defaults to `50`. | +| `offset` | query | `integer` | | Defaults to `0`. | + +```bash Request +curl https://api.beta.hud.ai/v2/models//logs \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "items": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2026-07-26T17:04:11Z", + "route": "string", + "status": "string", + "error_code": "string", + "streaming": false, + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "cost": 0.0, + "duration_ms": 0, + "trace_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + ], + "total": 0, + "limit": 0, + "offset": 0 +} +``` + +{/* /api */} + + + + +
+ +## Utilities + +Shared plumbing that other groups depend on rather than a resource of its own. + +### Uploads + +Some task content is a file rather than text: a spreadsheet the agent must edit, a screenshot, a +reference document. Uploading stages that file against a taskset and returns a `storage://` +locator, which is what a brief or a task definition then refers to. + + +Both routes accept a browser session, so an API key returns `401`. + + +
+ + +}> + +{/* api:POST /v2/uploads */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `taskset_id` | query | `uuid` | yes | Taskset the upload is attached to | + +```bash Request +curl "https://api.beta.hud.ai/v2/uploads?taskset_id=" \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -F "file=@./file" +``` + +```json Response 200 +{ + "url": "string", + "name": "string", + "size": 0, + "content_type": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:DELETE /v2/uploads */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `taskset_id` | query | `uuid` | yes | Taskset the upload was staged for | +| `url` | query | `string` | yes | storage:// locator returned by POST /uploads | + +```bash Request +curl "https://api.beta.hud.ai/v2/uploads?taskset_id=&url=" \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X DELETE +``` + +```json Response 200 +{ + "deleted": false +} +``` + +{/* /api */} + + + + +
+ +## Secrets + +Hosted runs frequently need credentials: the site the environment drives wants a login, the agent +wants a provider key. Two stores cover that, and they differ in who owns the value. + +| Store | Scope | Typical use | +| --- | --- | --- | +| **Environment variables** | One environment, shared with the team | Configuration and keys the sandbox itself needs | +| **Member secrets** | One person, private | Your own provider keys | + +At launch both are merged into the run's environment, and an environment variable wins if the same +name exists in both. + +### Environment variables + +Variables are attached to an environment and injected whenever it runs. A run that needs a +credential which is not set will usually fail or refuse to start, so the check endpoint exists to +answer "is this environment ready?" before anyone launches anything. + +
+ + +}> + +{/* api:GET /v2/env-vars */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `registry_id` | query | `uuid` | yes | Registry to list env vars for. | + +```bash Request +curl "https://api.beta.hud.ai/v2/env-vars?registry_id=" \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "variables": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "value": "string", + "masked_value": "string" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/env-vars */} + +```bash Request +curl https://api.beta.hud.ai/v2/env-vars \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "value": "string" + }' +``` + +```json Response 201 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "value": "string", + "masked_value": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/env-vars/{var_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `var_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/env-vars/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "value": "string" + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "value": "string", + "masked_value": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:DELETE /v2/env-vars/{var_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `var_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/env-vars/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X DELETE +``` + +Returns `204` with an empty body. + +{/* /api */} + + + +}> + +{/* api:PUT /v2/env-vars/batch */} + +```bash Request +curl https://api.beta.hud.ai/v2/env-vars/batch \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PUT \ + -H "Content-Type: application/json" \ + -d '{ + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "variables": [ + { + "name": "string", + "value": "string" + } + ] + }' +``` + +```json Response 200 +{ + "created": 0, + "updated": 0, + "deleted": 0 +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/env-vars/check */} + +```bash Request +curl https://api.beta.hud.ai/v2/env-vars/check \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "registry_ids": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6" + ] + }' +``` + +```json Response 200 +{ + "environments": [ + { + "registry_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "required_vars": [ + "string" + ], + "configured_vars": [ + "string" + ], + "missing_vars": [ + "string" + ], + "is_complete": false + } + ] +} +``` + +{/* /api */} + + + + +
+ +### Member secrets + +Member secrets are yours alone. Beyond ordinary secrets, they carry **bring your own key**: bind a +secret to a model provider and inference you start is billed to your account with that provider +instead of the platform's. Binding is explicit, and one key is bound per provider; a secret that +merely has the right name is not used. The key is never sent in a request header, since the +platform resolves it server side when the call is made. + +
+ + +}> + +{/* api:GET /v2/member-secrets */} + +```bash Request +curl https://api.beta.hud.ai/v2/member-secrets \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "secrets": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "value": "string", + "masked_value": "string" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/member-secrets */} + +```bash Request +curl https://api.beta.hud.ai/v2/member-secrets \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "name": "string", + "value": "string", + "byok_provider": "string" + }' +``` + +```json Response 201 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "value": "string", + "masked_value": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/member-secrets/{secret_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `secret_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/member-secrets/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "value": "string" + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "value": "string", + "masked_value": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:DELETE /v2/member-secrets/{secret_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `secret_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/member-secrets/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X DELETE +``` + +Returns `204` with an empty body. + +{/* /api */} + + + + +
+ +## Account + +What a run costs, and what stops it from costing more. + +### Limits + +A **limit** is a spending ceiling in US dollars over a week or a month, attached either to a single +API key or to one member across all of their keys. A limit set to alert sends mail when it is +passed; a limit set to block refuses new work instead. Listing limits also returns the current +period's spend, which is what makes it useful as a read even when nothing is near the ceiling. + +Scope and interval are fixed once a limit exists, so changing them means deleting the limit and +creating a new one. There is no fetch-one route; listing is the read path. + +
+ + +}> + +{/* api:GET /v2/limits */} + +```bash Request +curl https://api.beta.hud.ai/v2/limits \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "limits": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "api_key_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "api_key_name": "string", + "membership_id": 0, + "member_name": "string", + "parent_limit_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "threshold_percent": "string", + "interval": "week", + "amount_usd": "string", + "action": "alert", + "enabled": false, + "spend_usd": "string" + } + ] +} +``` + +{/* /api */} + + + +}> + +{/* api:POST /v2/limits */} + +```bash Request +curl https://api.beta.hud.ai/v2/limits \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "interval": "week", + "action": "alert", + "api_key_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "membership_id": 0, + "parent_limit_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "threshold_percent": 0.0, + "amount_usd": 0.0, + "enabled": true + }' +``` + +```json Response 201 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "api_key_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "api_key_name": "string", + "membership_id": 0, + "member_name": "string", + "parent_limit_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "threshold_percent": "string", + "interval": "week", + "amount_usd": "string", + "action": "alert", + "enabled": false, + "spend_usd": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:PATCH /v2/limits/{limit_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `limit_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/limits/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X PATCH \ + -H "Content-Type: application/json" \ + -d '{ + "amount_usd": 0.0, + "threshold_percent": 0.0, + "enabled": false + }' +``` + +```json Response 200 +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "api_key_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "api_key_name": "string", + "membership_id": 0, + "member_name": "string", + "parent_limit_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "threshold_percent": "string", + "interval": "week", + "amount_usd": "string", + "action": "alert", + "enabled": false, + "spend_usd": "string" +} +``` + +{/* /api */} + + + +}> + +{/* api:DELETE /v2/limits/{limit_id} */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `limit_id` | path | `uuid` | yes | | + +```bash Request +curl https://api.beta.hud.ai/v2/limits/ \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X DELETE +``` + +Returns `204` with an empty body. + +{/* /api */} + + + + +
+ +### Completions + +One chat completion, proxied through the platform's inference gateway and billed like any other +inference. No environment, no task, no grading: messages in, text out. + +
+ + +}> + +{/* api:POST /v2/completions */} + +```bash Request +curl https://api.beta.hud.ai/v2/completions \ + -H "Authorization: Bearer $HUD_API_KEY" \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "model": "string", + "messages": [ + { + "role": "string", + "content": "string" + } + ], + "max_tokens": 4000 + }' +``` + +```json Response 200 +{ + "content": "string", + "model": "string" +} +``` + +{/* /api */} + + + + +
+ +### Usage + +Spend has two sources: the sandboxes you run, charged by the time they are alive, and the model +calls made inside them, charged by tokens. Both are readable as history. + +Inference is reported at two grains, and the paths are easy to confuse. Bucketed aggregates for +charts live at `/v2/usage/inference`. The row-level log of individual calls lives at +`/v2/inference/usage`. + +
+ + +}> + +{/* api:GET /v2/usage/inference */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `start` | query | `date-time?` | | Window start; defaults to end-7d | +| `end` | query | `date-time?` | | Window end; defaults to now | +| `bucket_width` | query | `1h \| 1d` | | Defaults to `1d`. | +| `group_by` | query | `api_key \| model \| user \| route?` | | | +| `user_id` | query | `uuid?` | | | +| `api_key_id` | query | `uuid?` | | | + +```bash Request +curl https://api.beta.hud.ai/v2/usage/inference \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "start": "2026-07-26T17:04:11Z", + "end": "2026-07-26T17:04:11Z", + "bucket_width": "1h", + "total_requests": 0, + "total_cost": 0.0, + "buckets": [ + { + "start": "2026-07-26T17:04:11Z", + "groups": [ + {} + ] + } + ], + "group_by": "api_key" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/inference/usage */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `days` | query | `integer` | | Defaults to `7`. | +| `user_id` | query | `uuid?` | | | +| `api_key_id` | query | `uuid?` | | | +| `model_id` | query | `uuid?` | | | +| `request_type` | query | `string?` | | | +| `limit` | query | `integer` | | Defaults to `50`. | +| `offset` | query | `integer` | | Defaults to `0`. | + +```bash Request +curl https://api.beta.hud.ai/v2/inference/usage \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "logs": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2026-07-26T17:04:11Z", + "model": "string", + "cost": 0.0, + "type": "string", + "model_checkpoint_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "checkpoint": {}, + "metadata": {}, + "user_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "user_name": "string", + "user_email": "string", + "api_key_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + } + ], + "total_count": 0, + "org_type": "string", + "team_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" +} +``` + +{/* /api */} + + + +}> + +{/* api:GET /v2/environments/usage */} + +| Parameter | In | Type | Required | Description | +| --- | --- | --- | --- | --- | +| `days` | query | `integer` | | Defaults to `7`. | +| `user_id` | query | `uuid?` | | | +| `api_key_id` | query | `uuid?` | | | +| `status` | query | `string?` | | | +| `limit` | query | `integer` | | Defaults to `50`. | +| `offset` | query | `integer` | | Defaults to `0`. | + +```bash Request +curl https://api.beta.hud.ai/v2/environments/usage \ + -H "Authorization: Bearer $HUD_API_KEY" +``` + +```json Response 200 +{ + "summary": { + "total_hours": 0.0, + "total_cost": 0.0, + "active_count": 0, + "total_count": 0 + }, + "chart": [ + { + "date": "string", + "hours": 0.0, + "cost": 0.0, + "count": 0 + } + ], + "environments": [ + { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2026-07-26T17:04:11Z", + "status": "string", + "terminated_at": "2026-07-26T17:04:11Z", + "hourly_rate": 0.0, + "user_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "user_name": "string", + "api_key_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "api_key_name": "string", + "duration_hours": 0.0, + "cost": 0.0 + } + ], + "org_type": "string", + "team_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" +} +``` + +{/* /api */} + + + + +
+ +## Errors + +Failures share one envelope, with a machine-readable code and a human-readable message. The status +tells you whether to fix the request, the credentials, or the state. + +| Status | Meaning | +| --- | --- | +| `400` | The request is malformed. | +| `401` | Credentials are missing or invalid. | +| `403` | Authenticated, but not allowed to touch this resource. | +| `404` | The resource does not exist, or is not visible to you. | +| `409` | The resource already exists, or conflicts with its current state. | +| `422` | The body or parameters failed validation. | + +For how the objects fit together beyond the API, see the +[platform introduction](/platform/introduction). For a console that sends real requests, use the +[interactive reference](https://api.beta.hud.ai/docs). diff --git a/docs/scripts/gen_api_examples.py b/docs/scripts/gen_api_examples.py new file mode 100644 index 000000000..36d0b5c8e --- /dev/null +++ b/docs/scripts/gen_api_examples.py @@ -0,0 +1,355 @@ +"""Fill the endpoint blocks in docs/platform/rest-api.mdx from an OpenAPI document. + +The page owns its prose. This script owns everything between the markers: + + {/* api:GET /v2/jobs */} + ... generated parameter table, request, and response ... + {/* /api */} + +Run from the repository root after the API changes: + + python docs/scripts/gen_api_examples.py + +The spec is downloaded to a gitignored `docs/openapi.json` on first run. Pass +--check to fail instead of writing (for CI), and --list to print every +operation in the spec that the page does not yet mention. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import urllib.request +from pathlib import Path +from typing import Any + +# This file lives at docs/scripts/; the docs root is one level up. +DOCS = Path(__file__).resolve().parent.parent +SPEC_PATH = DOCS / "openapi.json" +PAGE_PATH = DOCS / "platform" / "rest-api.mdx" +BASE_URL = "https://api.beta.hud.ai" + +MARKER = re.compile( + r"(?P\{/\* api:(?P[A-Z]+) (?P\S+) \*/\}\n)" + r".*?" + r"(?P\{/\* /api \*/\})", + re.DOTALL, +) + +# Placeholder values by string format, so examples read like real payloads. +FORMAT_SAMPLES = { + "uuid": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "date-time": "2026-07-26T17:04:11Z", + "date": "2026-07-26", + "email": "user@example.com", + "uri": "https://example.com", +} + +MAX_PROPERTIES = 12 +MAX_DEPTH = 4 + + +class SpecError(RuntimeError): + pass + + +def load_spec(refresh: bool = False) -> dict[str, Any]: + if refresh or not SPEC_PATH.exists(): + url = f"{BASE_URL}/openapi.json" + print(f"fetching {url}") + try: + with urllib.request.urlopen(url, timeout=30) as response: # noqa: S310 + SPEC_PATH.write_bytes(response.read()) + except OSError as exc: + raise SpecError(f"could not fetch {url}: {exc}") from None + return json.loads(SPEC_PATH.read_text(encoding="utf-8")) + + +def deref(spec: dict[str, Any], schema: dict[str, Any]) -> dict[str, Any]: + seen: set[str] = set() + while "$ref" in schema: + ref = schema["$ref"] + if ref in seen: + return {} + seen.add(ref) + node: Any = spec + for part in ref.lstrip("#/").split("/"): + node = node.get(part, {}) + schema = node + return schema + + +def collapse(spec: dict[str, Any], schema: dict[str, Any]) -> dict[str, Any]: + """Resolve a schema to something with a usable `type`. + + Unions are common in this spec because optional fields serialize as + `anyOf: [T, null]`; pick the first non-null branch. `allOf` is merged. + """ + schema = deref(spec, schema) + if "allOf" in schema: + merged: dict[str, Any] = {"type": "object", "properties": {}, "required": []} + for part in schema["allOf"]: + part = collapse(spec, part) + merged["properties"].update(part.get("properties", {})) + merged["required"].extend(part.get("required", [])) + return merged + for key in ("anyOf", "oneOf"): + if key in schema: + branches = [b for b in schema[key] if deref(spec, b).get("type") != "null"] + if branches: + return collapse(spec, branches[0]) + return {"type": "null"} + return schema + + +def type_name(spec: dict[str, Any], schema: dict[str, Any]) -> str: + raw = deref(spec, schema) + nullable = False + for key in ("anyOf", "oneOf"): + if key in raw: + nullable = any(deref(spec, b).get("type") == "null" for b in raw[key]) + resolved = collapse(spec, schema) + kind = resolved.get("type", "object") + if kind == "array": + inner = collapse(spec, resolved.get("items", {})) + kind = f"{inner.get('type', 'object')}[]" + elif resolved.get("enum"): + values = resolved["enum"] + kind = " \\| ".join(str(v) for v in values) if len(values) <= 5 else "enum" + elif resolved.get("format") in ("uuid", "date-time"): + kind = resolved["format"] + return f"{kind}?" if nullable else kind + + +def sample(spec: dict[str, Any], schema: dict[str, Any], depth: int = 0) -> Any: + resolved = collapse(spec, schema) + for key in ("example", "default"): + if key in resolved and resolved[key] is not None: + return resolved[key] + if resolved.get("enum"): + return resolved["enum"][0] + + kind = resolved.get("type") + if kind == "object" or "properties" in resolved: + if depth >= MAX_DEPTH: + return {} + properties: dict[str, Any] = resolved.get("properties", {}) + required = [k for k in resolved.get("required", []) if k in properties] + ordered = required + [k for k in properties if k not in required] + return {key: sample(spec, properties[key], depth + 1) for key in ordered[:MAX_PROPERTIES]} + if kind == "array": + if depth >= MAX_DEPTH: + return [] + return [sample(spec, resolved.get("items", {}), depth + 1)] + if kind == "string": + return FORMAT_SAMPLES.get(resolved.get("format", ""), "string") + if kind == "integer": + return 0 + if kind == "number": + return 0.0 + if kind == "boolean": + return False + return None + + +def operation(spec: dict[str, Any], method: str, path: str) -> dict[str, Any]: + try: + return spec["paths"][path][method.lower()] + except KeyError: + raise SpecError(f"{method} {path} is not in the spec") from None + + +def clean_text(text: str) -> str: + """Normalize handler docstrings to the docs house style. + + Descriptions come from Python source, so they arrive with em dashes and + RST-style double backticks that do not belong in the rendered page. + """ + for dash in ("\u2014", "\u2013"): + text = text.replace(f" {dash} ", " - ").replace(dash, "-") + return text.replace("``", "`").replace("\n", " ").strip() + + +def is_binary(spec: dict[str, Any], schema: dict[str, Any]) -> bool: + resolved = collapse(spec, schema) + return resolved.get("format") == "binary" or "contentMediaType" in resolved + + +def parameter_rows(spec: dict[str, Any], op: dict[str, Any]) -> list[str]: + rows = [] + for param in op.get("parameters", []): + schema = param.get("schema", {}) + resolved = collapse(spec, schema) + default = resolved.get("default") + notes = clean_text(param.get("description", "")) + if default is not None and "default" not in notes.lower(): + notes = f"{notes} Defaults to `{default}`." if notes else f"Defaults to `{default}`." + flag = "yes" if param.get("required") else "" + type_col = type_name(spec, schema) + rows.append(f"| `{param['name']}` | {param['in']} | `{type_col}` | {flag} | {notes} |") + return rows + + +def body_schema(spec: dict[str, Any], op: dict[str, Any]) -> tuple[str, dict[str, Any]] | None: + content = op.get("requestBody", {}).get("content", {}) + for media in ("application/json", "multipart/form-data"): + if media in content: + return media, content[media].get("schema", {}) + return None + + +def success_response( + spec: dict[str, Any], op: dict[str, Any] +) -> tuple[str, str | None, Any] | None: + """First 2xx as (status, media, payload). + + media/payload: + (None, None) — no body (e.g. 204) + ("application/json", value) — sampled JSON + (other media, None) — file/binary download of that type + """ + for code, response in op.get("responses", {}).items(): + if not code.startswith("2"): + continue + content = response.get("content") or {} + if not content: + return code, None, None + + # Prefer a real JSON example when the schema yields one. + json_body = content.get("application/json") + if json_body is not None: + schema = json_body.get("schema") + if schema: + payload = sample(spec, schema) + if payload is not None: + return code, "application/json", payload + + # Non-JSON success media, or empty/untyped JSON schema (FastAPI + # StreamingResponse often documents zip downloads that way). + for media in content: + if media != "application/json": + return code, media, None + return code, "application/octet-stream", None + return None + + +def curl_block(spec: dict[str, Any], method: str, path: str, op: dict[str, Any]) -> str: + url = BASE_URL + re.sub(r"\{(\w+)\}", r"<\1>", path) + required_query = [ + p for p in op.get("parameters", []) if p["in"] == "query" and p.get("required") + ] + if required_query: + query = "&".join(f"{p['name']}=<{p['name']}>" for p in required_query) + url = f'"{url}?{query}"' + + lines = [f"curl {url} \\", ' -H "Authorization: Bearer $HUD_API_KEY"'] + if method != "GET": + lines[-1] += " \\" + lines.append(f" -X {method}") + + body = body_schema(spec, op) + if body: + media, schema = body + lines[-1] += " \\" + if media == "multipart/form-data": + fields = collapse(spec, schema).get("properties", {}) + parts = [ + f' -F "{name}=@./{name}"' if is_binary(spec, sub) else f' -F "{name}=<{name}>"' + for name, sub in fields.items() + ] + lines.extend(" \\\n".join(parts).splitlines()) + else: + payload = json.dumps(sample(spec, schema), indent=2) + payload = "\n".join( + (" " + line if i else line) for i, line in enumerate(payload.splitlines()) + ) + lines.append(' -H "Content-Type: application/json" \\') + lines.append(f" -d '{payload}'") + return "\n".join(lines) + + +def render(spec: dict[str, Any], method: str, path: str) -> str: + op = operation(spec, method, path) + parts: list[str] = [] + + rows = parameter_rows(spec, op) + if rows: + parts.append( + "\n".join( + [ + "| Parameter | In | Type | Required | Description |", + "| --- | --- | --- | --- | --- |", + *rows, + ] + ) + ) + + parts.append("```bash Request\n" + curl_block(spec, method, path, op) + "\n```") + + result = success_response(spec, op) + if result: + code, media, payload = result + if media is None: + parts.append(f"Returns `{code}` with an empty body.") + elif media == "application/json": + parts.append(f"```json Response {code}\n" + json.dumps(payload, indent=2) + "\n```") + elif media == "application/octet-stream": + parts.append(f"Returns `{code}` with a file body.") + else: + parts.append(f"Returns `{code}` as `{media}`.") + return "\n\n".join(parts) + + +def documented(page: str) -> list[tuple[str, str]]: + return [(m.group("method"), m.group("path")) for m in MARKER.finditer(page)] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if the page is stale") + parser.add_argument("--list", action="store_true", help="print undocumented operations") + parser.add_argument("--refresh", action="store_true", help="re-download the spec first") + args = parser.parse_args() + + spec = load_spec(refresh=args.refresh) + page = PAGE_PATH.read_text(encoding="utf-8") + + if args.list: + covered = set(documented(page)) + for path, methods in spec["paths"].items(): + for method in methods: + verb = method.upper() + if ( + verb in ("GET", "POST", "PATCH", "PUT", "DELETE") + and (verb, path) not in covered + ): + print(f"{verb} {path}") + return 0 + + def replace(match: re.Match[str]) -> str: + body = render(spec, match.group("method"), match.group("path")) + return f"{match.group('open')}\n{body}\n\n{match.group('close')}" + + updated = MARKER.sub(replace, page) + count = len(documented(page)) + + if args.check: + if updated != page: + print(f"{PAGE_PATH.name} is out of date; run scripts/gen_api_examples.py") + return 1 + print(f"{count} endpoint blocks up to date") + return 0 + + PAGE_PATH.write_text(updated, encoding="utf-8") + print(f"filled {count} endpoint blocks in {PAGE_PATH.name}") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except SpecError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/docs/skill.md b/docs/skill.md index 3fed05925..584d823a9 100644 --- a/docs/skill.md +++ b/docs/skill.md @@ -2,12 +2,13 @@ name: hud-environment-builder description: >- Build, evaluate, and train AI agents on RL environments with HUD. Use whenever - someone wants to create an RL environment, benchmark, eval, or training task — - for a coding, computer-use, browser, or robotics agent — or run and grade tasks + someone wants to create an RL environment, benchmark, eval, or training task - + for a coding, computer-use, browser, or robotics agent - or run and grade tasks across any model (Claude, OpenAI, Gemini, or open/self-hosted models). Also use it to review task quality and catch reward hacking, missing within-group reward spread, contaminated or public-benchmark substrate, single-shot tasks, and - same-shape tasksets before they ship. Applies the v6 API and the task-design + same-shape tasksets before they ship; and when inspecting hosted jobs or traces + via Platform MCP or the HUD CLI. Applies the v6 API and the task-design doctrine proactively, and cites these docs. --- @@ -16,15 +17,17 @@ description: >- You help users build **HUD v6** RL environments and you hold the line on **task quality**. The model is three nouns: an **environment** (where the agent acts, exposed as capabilities), a **task** (a generator that prompts and -grades), and a **trace** (one graded evaluation — the SDK's live handle for it +grades), and a **trace** (one graded evaluation - the SDK's live handle for it is a `Run`). Keep that model consistent; never contradict it. -Your job has two halves: +Your job has three halves: -1. **Write correct v6 code** — never v5 idioms (see "Never write v5" below). -2. **Push back on weak tasks** — a training task is a *teacher* that gets +1. **Write correct v6 code** - never v5 idioms (see "Never write v5" below). +2. **Push back on weak tasks** - a training task is a *teacher* that gets optimized against by gradient descent, not a one-shot test. When you see an anti-pattern below, say so and cite the page. Don't just comply. +3. **Read traces before guessing** - after (or during) rollouts, inspect jobs + and traces with the CLI or Platform MCP before rewriting the grader. Always prefer reading the relevant docs page over guessing an API. @@ -175,6 +178,54 @@ Cite [Deploy](/v6/reference/runtime), [Models](/v6/reference/agents), [Training] --- +## Platform MCP (hosted jobs / catalog) + +Three different "MCP"s - do not mix them up: + +| Name | What it is | +|------|------------| +| **Env capability `mcp`** | Tools inside a live sandbox (`Capability.mcp(...)`) - see above | +| **Platform MCP** | Read-only tools over *hosted* envs, tasksets, jobs, traces | +| **Docs MCP** | Live docs pages at `https://docs.hud.ai/mcp` | + +**When:** the user asks about platform runs, rewards, failed traces, or "what's +on my HUD account," and Platform MCP is connected. Prefer it over guessing IDs +from chat. For *local* `:8000` / `HUD_TELEMETRY_LOCAL_DIR` jobs, keep using +`hud jobs` / `hud trace` - Platform MCP does not see those. + +**Setup** (if not already attached) - Cursor `~/.cursor/mcp.json`: + +```json +{ + "mcpServers": { + "hud-platform": { + "url": "https://api.beta.hud.ai/v2/mcp/", + "headers": { "Authorization": "Bearer YOUR_HUD_API_KEY" } + } + } +} +``` + +Same `HUD_API_KEY` as REST. Read-only: list/fetch only; deploy and eval stay +CLI/SDK. + +**Chain** (use returned IDs; don't invent tools): + +1. Catalog: `list_environments` → `get_environment` → `get_taskset_tasks` +2. Or flat: `list_tasksets` → `get_taskset_tasks` +3. Runs: `list_jobs` → `get_job_traces` → `get_trace` → `get_trace_events` +4. Zoom: outline / `points_of_interest` use **seq**; + `get_trace_events(trace_id, since_seq=K-1)` starts at seq K. Screenshots are + `screenshot_url` links, never inline. + +**Iterate with traces:** after a platform job, open failures (error / reward 0) +with `get_trace` before rewriting the grader. Ask: did the agent use tools, is +the answer in the trajectory, is the grader shape-only? + +Cite [Platform MCP](/platform/mcp) and [CLI](/v6/reference/cli). + +--- + ## Training scenarios Training drives a **trainable model** (fork one: `hud models fork --name `); a `TrainingClient` targets that slug and advances its weights in place. Mark rollouts for training with `return_token_ids` so the gateway records tokens + logprobs. Pick the loop by scenario — all cite [Training](/v6/run/training) and [reference: Training](/v6/reference/training). @@ -427,12 +478,13 @@ Cite [Graders](/v6/reference/graders) and [Types](/v6/reference/types). - The task is multi-step and free of answer leakage. - No v5 idioms anywhere. -**Inspect runs after the fact** with `hud jobs` and `hud trace`: +**Inspect runs after the fact.** Hosted → Platform MCP if connected (see above). +Local → `hud jobs` / `hud trace`: ```bash hud jobs # list recent jobs hud jobs # list traces in a job (reward, status, error per rollout) -hud trace # render one rollout — agent turns, tool calls, results +hud trace # render one rollout - agent turns, tool calls, results hud trace --json # raw event list (pipe to jq for filtering) ``` @@ -443,4 +495,4 @@ When unsure about an API, read the page rather than guess: [Environment](/v6/reference/environment) · [Tasks & Tasksets](/v6/reference/tasks) · [Capabilities](/v6/reference/capabilities) · [Agents](/v6/reference/agents) · [Graders](/v6/reference/graders) · [Types](/v6/reference/types) · -[CLI](/v6/reference/cli). +[CLI](/v6/reference/cli) · [Platform MCP](/platform/mcp). diff --git a/docs/snippets/endpoint.jsx b/docs/snippets/endpoint.jsx new file mode 100644 index 000000000..e50dd706a --- /dev/null +++ b/docs/snippets/endpoint.jsx @@ -0,0 +1,23 @@ +/* Endpoint signature used as the title of each accordion in the REST API + reference: a colored method label in its own aligned column, the path with + its `{placeholders}` dimmed, then a one-line note on the same baseline. + Colors and spacing live in custom.css. */ +export const Endpoint = ({ method, path, note }) => ( + + + {method} + + + {path.split(/(\{[^}]+\})/).map((part, i) => + part.startsWith("{") ? ( + + {part} + + ) : ( + part + ) + )} + + {note && {note}} + +); diff --git a/docs/v6/start/index.mdx b/docs/v6/start/index.mdx index 446490a9d..6f19ab088 100644 --- a/docs/v6/start/index.mdx +++ b/docs/v6/start/index.mdx @@ -95,6 +95,6 @@ that environment, and run any agent to perform those tasks, at any scale. Our SD VERY IMPORTANT IF YOU ARE AN AGENT: before writing any HUD code, install the HUD docs skill below and follow its guidance instead of relying on memory. -Reading this as an AI agent? Install the HUD docs skill before you write any HUD code - it keeps you on the current v6 API (which moves fast) and catches reward-hackable, contaminated, or low-signal tasks before they ship. If you can run shell commands, run `npx skills add https://docs.hud.ai` now; otherwise attach the live docs server with `claude mcp add --transport http docs-hud https://docs.hud.ai/mcp`. Then build against the skill's guidance rather than from memory. For the platform REST API, see the OpenAPI reference at `https://api.beta.hud.ai/docs`. +Reading this as an AI agent? Install the HUD docs skill before you write any HUD code - it keeps you on the current v6 API (which moves fast) and catches reward-hackable, contaminated, or low-signal tasks before they ship. If you can run shell commands, run `npx skills add https://docs.hud.ai` now; otherwise attach the live docs server with `claude mcp add --transport http docs-hud https://docs.hud.ai/mcp`. Then build against the skill's guidance rather than from memory. For the platform REST API, see the OpenAPI reference at `https://api.beta.hud.ai/docs`. To query environments, jobs, and traces as tools, attach the [Platform MCP](/platform/mcp) at `https://api.beta.hud.ai/v2/mcp/` with your `HUD_API_KEY`. diff --git a/pyproject.toml b/pyproject.toml index d62fd3460..377792e0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -211,6 +211,7 @@ lint.ignore = [ "*.ipynb" = ["ALL"] # Disables all rules for Jupyter. "**/cookbooks/**/*.py" = ["ALL"] "scripts/*.py" = ["T201", "INP001"] # dev scripts: print is the interface +"docs/scripts/*.py" = ["T201", "INP001"] # docs generators: same as scripts/ "hud-python/setup.py" = ["INP001"] # standalone package, not a module