diff --git a/docs/batch-client-reference.md b/docs/batch-client-reference.md
index c2a4c92..875b3cb 100644
--- a/docs/batch-client-reference.md
+++ b/docs/batch-client-reference.md
@@ -108,7 +108,7 @@ for anyway.
def submit_regular(urls: list[str | TaskInputDict] | None = None,
*,
file_input_id: str | None = None,
- zenrows_params: dict[str, str] | None = None,
+ zenrows_params: ParamMap | None = None,
external_id: str | None = None,
name: str | None = None,
metadata: dict[str, str] | None = None,
@@ -140,7 +140,7 @@ open-and-extend pattern see `submit_open`.
```python
def submit_open(urls: list[str | TaskInputDict] | None = None,
*,
- zenrows_params: dict[str, str] | None = None,
+ zenrows_params: ParamMap | None = None,
external_id: str | None = None,
name: str | None = None,
metadata: dict[str, str] | None = None,
@@ -169,7 +169,7 @@ def submit_scheduled(schedule: Schedule | JobScheduleDict,
urls: list[str | TaskInputDict] | None = None,
*,
file_input_id: str | None = None,
- zenrows_params: dict[str, str] | None = None,
+ zenrows_params: ParamMap | None = None,
external_id: str | None = None,
name: str | None = None,
metadata: dict[str, str] | None = None,
@@ -1597,6 +1597,12 @@ exactly one tier. If a malformed param map carries both, `auto`
wins here (it's what the engine would honor) — but the server would
reject that body at submit anyway.
+
+
+#### ParamValue
+
+dict form: `custom_headers` map
+
## Tier Objects
@@ -2132,12 +2138,13 @@ rows). Flip via `POST /jobs/{id}/pause` and
class IngestStatus(Enum)
```
-Present only on runs created by a large (202) submission.
-`pending` — task rows are still streaming into storage;
-reads may return partial pages and `addTasks` returns
-`409`. `done` — every accepted task row is visible.
-Omitted on runs whose tasks were written on the request
-path (201 submissions, `addTasks` batches, reruns).
+Present only on runs created by a large (202) submission
+or a large (202) rerun. `pending` — task rows are still
+streaming into storage; reads may return partial pages
+and `addTasks` returns `409`. `done` — every accepted
+task row is visible. Omitted on runs whose tasks were
+written on the request path (201 submissions and
+reruns, `addTasks` batches).
@@ -2182,12 +2189,13 @@ rows). Flip via `POST /jobs/{id}/pause` and
#### ingest\_status
-Present only on runs created by a large (202) submission.
-`pending` — task rows are still streaming into storage;
-reads may return partial pages and `addTasks` returns
-`409`. `done` — every accepted task row is visible.
-Omitted on runs whose tasks were written on the request
-path (201 submissions, `addTasks` batches, reruns).
+Present only on runs created by a large (202) submission
+or a large (202) rerun. `pending` — task rows are still
+streaming into storage; reads may return partial pages
+and `addTasks` returns `409`. `done` — every accepted
+task row is visible. Omitted on runs whose tasks were
+written on the request path (201 submissions and
+reruns, `addTasks` batches).
@@ -2737,7 +2745,7 @@ Omitted when the caller did not supply one.
#### result\_url
-2-hour presigned download URL for the result body, or a
+24-hour presigned download URL for the result body, or a
`/v1/jobs//runs//tasks//content` URL you can
fetch directly. Empty for non-successful tasks.
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index 8acb757..e467732 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -422,6 +422,12 @@ paths:
- Auto-closes an `open` job (existing behavior).
- Honors `Idempotency-Key`: a repeat call with the same
key returns the original new-run id.
+
+ Large reruns are accepted asynchronously: past a server-side
+ row-count threshold (the same one that upgrades large
+ submissions) the call returns `202` and the new run's task
+ rows stream into storage off the request path. Treat `201`
+ and `202` both as success.
parameters:
- in: query
name: status
@@ -437,7 +443,20 @@ paths:
- $ref: '#/components/parameters/IdempotencyKey'
responses:
'201':
- description: New run created.
+ description: New run created (sync path). All task rows are written.
+ content:
+ application/json:
+ schema: { $ref: '#/components/schemas/RerunJobResponse' }
+ '202':
+ description: |
+ New run accepted (large rerun). `retried_tasks` /
+ `inherited_tasks` are the final counts, but individual
+ task rows stream into storage off the request path.
+ `latest_run.ingest_status` is `pending` in this response
+ and flips to `done` on `GET /v1/jobs/{id}` once every
+ task row is visible; the run's `stats.total` is correct
+ from this response forward, so completion polling is
+ safe. Body shape matches the 201 response.
content:
application/json:
schema: { $ref: '#/components/schemas/RerunJobResponse' }
@@ -1246,11 +1265,24 @@ components:
`session_id`, `original_status`, `allowed_status_codes`,
`wait_for`, `wait`, `block_resources`, `json_response`,
`css_extractor`, `autoparse`, `response_type`, `outputs`.
+
+ `custom_headers` additionally accepts an object of header
+ name → value pairs (or that object serialised as a JSON
+ string), e.g. `{"custom_headers": {"Referer":
+ "https://www.google.com/"}}`. The headers are forwarded to
+ the scraping target, subject to the scraper API's usual
+ header sanitisation. At most 32 headers / 8 KB serialised;
+ names must be valid HTTP tokens. Header filtering and
+ browser-header sanitisation are owned by the scraper API,
+ not Conveyor. A task-level
+ `custom_headers` replaces the job-level object wholesale —
+ no per-name merging.
additionalProperties:
oneOf:
- { type: string }
- { type: boolean }
- { type: integer }
+ - { type: object, additionalProperties: { type: string } }
TaskInput:
type: object
@@ -1509,12 +1541,13 @@ components:
type: string
enum: [pending, done]
description: |
- Present only on runs created by a large (202) submission.
- `pending` — task rows are still streaming into storage;
- reads may return partial pages and `addTasks` returns
- `409`. `done` — every accepted task row is visible.
- Omitted on runs whose tasks were written on the request
- path (201 submissions, `addTasks` batches, reruns).
+ Present only on runs created by a large (202) submission
+ or a large (202) rerun. `pending` — task rows are still
+ streaming into storage; reads may return partial pages
+ and `addTasks` returns `409`. `done` — every accepted
+ task row is visible. Omitted on runs whose tasks were
+ written on the request path (201 submissions and
+ reruns, `addTasks` batches).
created_at: { type: string, format: date-time }
updated_at: { type: string, format: date-time }
failure_reason:
@@ -1731,7 +1764,7 @@ components:
type: string
nullable: true
description: |
- 2-hour presigned download URL for the result body, or a
+ 24-hour presigned download URL for the result body, or a
`/v1/jobs//runs//tasks//content` URL you can
fetch directly. Empty for non-successful tasks.
error:
diff --git a/src/zenrows/batch/_estimate.py b/src/zenrows/batch/_estimate.py
index 01e5be2..f730ab6 100644
--- a/src/zenrows/batch/_estimate.py
+++ b/src/zenrows/batch/_estimate.py
@@ -50,7 +50,7 @@
_TRUTHY = frozenset({"true", "1", "yes", "on"})
# What `Tier` keys can appear, in the order a breakdown should render.
-ParamValue = str | bool | int
+ParamValue = str | bool | int | dict[str, str] # dict form: `custom_headers` map
ParamMap = dict[str, ParamValue]
TaskLike = str | TaskInput | dict
diff --git a/src/zenrows/batch/client.py b/src/zenrows/batch/client.py
index 1855af1..dc5ea3a 100644
--- a/src/zenrows/batch/client.py
+++ b/src/zenrows/batch/client.py
@@ -46,7 +46,7 @@
download_to_dir,
download_to_memory,
)
-from zenrows.batch._estimate import CostEstimate, _estimate_cost
+from zenrows.batch._estimate import CostEstimate, ParamMap, _estimate_cost
from zenrows.batch._resources import (
ExportHandle,
ExportRef,
@@ -236,7 +236,7 @@ def submit_regular(
urls: list[str | TaskInputDict] | None = None,
*,
file_input_id: str | None = None,
- zenrows_params: dict[str, str] | None = None,
+ zenrows_params: ParamMap | None = None,
external_id: str | None = None,
name: str | None = None,
metadata: dict[str, str] | None = None,
@@ -279,7 +279,7 @@ def submit_open(
self,
urls: list[str | TaskInputDict] | None = None,
*,
- zenrows_params: dict[str, str] | None = None,
+ zenrows_params: ParamMap | None = None,
external_id: str | None = None,
name: str | None = None,
metadata: dict[str, str] | None = None,
@@ -319,7 +319,7 @@ def submit_scheduled(
urls: list[str | TaskInputDict] | None = None,
*,
file_input_id: str | None = None,
- zenrows_params: dict[str, str] | None = None,
+ zenrows_params: ParamMap | None = None,
external_id: str | None = None,
name: str | None = None,
metadata: dict[str, str] | None = None,
@@ -1156,7 +1156,7 @@ def _build_submit_body(
urls: list[str | TaskInputDict] | None,
file_input_id: str | None,
status: Literal["open", "closed"],
- zenrows_params: dict[str, str] | None,
+ zenrows_params: ParamMap | None,
schedule: JobScheduleDict | None = None,
external_id: str | None = None,
name: str | None = None,
diff --git a/src/zenrows/batch/models.py b/src/zenrows/batch/models.py
index 75e924b..90361c0 100644
--- a/src/zenrows/batch/models.py
+++ b/src/zenrows/batch/models.py
@@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: openapi.yaml
-# timestamp: 2026-07-08T16:45:00+00:00
+# timestamp: 2026-07-21T11:46:07+00:00
from __future__ import annotations
@@ -164,7 +164,7 @@ class TaskInput(BaseModel):
Must be http(s). Other schemes rejected at submit.
"""
metadata: Annotated[dict[str, str] | None, Field(max_length=20)] = None
- zenrows_params: dict[str, str | bool | int] | None = None
+ zenrows_params: dict[str, str | bool | int | dict[str, str]] | None = None
"""
Per-task scraper params. Override the job-level
`zenrows_params` on key collision (task wins).
@@ -276,12 +276,13 @@ class PauseState(Enum):
class IngestStatus(Enum):
"""
- Present only on runs created by a large (202) submission.
- `pending` — task rows are still streaming into storage;
- reads may return partial pages and `addTasks` returns
- `409`. `done` — every accepted task row is visible.
- Omitted on runs whose tasks were written on the request
- path (201 submissions, `addTasks` batches, reruns).
+ Present only on runs created by a large (202) submission
+ or a large (202) rerun. `pending` — task rows are still
+ streaming into storage; reads may return partial pages
+ and `addTasks` returns `409`. `done` — every accepted
+ task row is visible. Omitted on runs whose tasks were
+ written on the request path (201 submissions and
+ reruns, `addTasks` batches).
"""
@@ -326,12 +327,13 @@ class Run(BaseModel):
"""
ingest_status: IngestStatus | None = None
"""
- Present only on runs created by a large (202) submission.
- `pending` — task rows are still streaming into storage;
- reads may return partial pages and `addTasks` returns
- `409`. `done` — every accepted task row is visible.
- Omitted on runs whose tasks were written on the request
- path (201 submissions, `addTasks` batches, reruns).
+ Present only on runs created by a large (202) submission
+ or a large (202) rerun. `pending` — task rows are still
+ streaming into storage; reads may return partial pages
+ and `addTasks` returns `409`. `done` — every accepted
+ task row is visible. Omitted on runs whose tasks were
+ written on the request path (201 submissions and
+ reruns, `addTasks` batches).
"""
created_at: AwareDatetime
@@ -859,7 +861,7 @@ class TaskResult(BaseModel):
type: ResultType | None = None
result_url: str | None = None
"""
- 2-hour presigned download URL for the result body, or a
+ 24-hour presigned download URL for the result body, or a
`/v1/jobs//runs//tasks//content` URL you can
fetch directly. Empty for non-successful tasks.
@@ -1023,7 +1025,7 @@ class SubmitJobRequest(BaseModel):
is auto-closed.
"""
- zenrows_params: dict[str, str | bool | int] | None = None
+ zenrows_params: dict[str, str | bool | int | dict[str, str]] | None = None
"""
Job-level scraper params, applied to every task of every
run of the job. Each task can override individual keys