diff --git a/docs/batch-client-reference.md b/docs/batch-client-reference.md index c2a4c92..875b3cb 100644 --- a/docs/batch-client-reference.md +++ b/docs/batch-client-reference.md @@ -108,7 +108,7 @@ for anyway. def submit_regular(urls: list[str | TaskInputDict] | None = None, *, file_input_id: str | None = None, - zenrows_params: dict[str, str] | None = None, + zenrows_params: ParamMap | None = None, external_id: str | None = None, name: str | None = None, metadata: dict[str, str] | None = None, @@ -140,7 +140,7 @@ open-and-extend pattern see `submit_open`. ```python def submit_open(urls: list[str | TaskInputDict] | None = None, *, - zenrows_params: dict[str, str] | None = None, + zenrows_params: ParamMap | None = None, external_id: str | None = None, name: str | None = None, metadata: dict[str, str] | None = None, @@ -169,7 +169,7 @@ def submit_scheduled(schedule: Schedule | JobScheduleDict, urls: list[str | TaskInputDict] | None = None, *, file_input_id: str | None = None, - zenrows_params: dict[str, str] | None = None, + zenrows_params: ParamMap | None = None, external_id: str | None = None, name: str | None = None, metadata: dict[str, str] | None = None, @@ -1597,6 +1597,12 @@ exactly one tier. If a malformed param map carries both, `auto` wins here (it's what the engine would honor) — but the server would reject that body at submit anyway. + + +#### ParamValue + +dict form: `custom_headers` map + ## Tier Objects @@ -2132,12 +2138,13 @@ rows). Flip via `POST /jobs/{id}/pause` and class IngestStatus(Enum) ``` -Present only on runs created by a large (202) submission. -`pending` — task rows are still streaming into storage; -reads may return partial pages and `addTasks` returns -`409`. `done` — every accepted task row is visible. -Omitted on runs whose tasks were written on the request -path (201 submissions, `addTasks` batches, reruns). +Present only on runs created by a large (202) submission +or a large (202) rerun. `pending` — task rows are still +streaming into storage; reads may return partial pages +and `addTasks` returns `409`. `done` — every accepted +task row is visible. Omitted on runs whose tasks were +written on the request path (201 submissions and +reruns, `addTasks` batches). @@ -2182,12 +2189,13 @@ rows). Flip via `POST /jobs/{id}/pause` and #### ingest\_status -Present only on runs created by a large (202) submission. -`pending` — task rows are still streaming into storage; -reads may return partial pages and `addTasks` returns -`409`. `done` — every accepted task row is visible. -Omitted on runs whose tasks were written on the request -path (201 submissions, `addTasks` batches, reruns). +Present only on runs created by a large (202) submission +or a large (202) rerun. `pending` — task rows are still +streaming into storage; reads may return partial pages +and `addTasks` returns `409`. `done` — every accepted +task row is visible. Omitted on runs whose tasks were +written on the request path (201 submissions and +reruns, `addTasks` batches). @@ -2737,7 +2745,7 @@ Omitted when the caller did not supply one. #### result\_url -2-hour presigned download URL for the result body, or a +24-hour presigned download URL for the result body, or a `/v1/jobs//runs//tasks//content` URL you can fetch directly. Empty for non-successful tasks. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 8acb757..e467732 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -422,6 +422,12 @@ paths: - Auto-closes an `open` job (existing behavior). - Honors `Idempotency-Key`: a repeat call with the same key returns the original new-run id. + + Large reruns are accepted asynchronously: past a server-side + row-count threshold (the same one that upgrades large + submissions) the call returns `202` and the new run's task + rows stream into storage off the request path. Treat `201` + and `202` both as success. parameters: - in: query name: status @@ -437,7 +443,20 @@ paths: - $ref: '#/components/parameters/IdempotencyKey' responses: '201': - description: New run created. + description: New run created (sync path). All task rows are written. + content: + application/json: + schema: { $ref: '#/components/schemas/RerunJobResponse' } + '202': + description: | + New run accepted (large rerun). `retried_tasks` / + `inherited_tasks` are the final counts, but individual + task rows stream into storage off the request path. + `latest_run.ingest_status` is `pending` in this response + and flips to `done` on `GET /v1/jobs/{id}` once every + task row is visible; the run's `stats.total` is correct + from this response forward, so completion polling is + safe. Body shape matches the 201 response. content: application/json: schema: { $ref: '#/components/schemas/RerunJobResponse' } @@ -1246,11 +1265,24 @@ components: `session_id`, `original_status`, `allowed_status_codes`, `wait_for`, `wait`, `block_resources`, `json_response`, `css_extractor`, `autoparse`, `response_type`, `outputs`. + + `custom_headers` additionally accepts an object of header + name → value pairs (or that object serialised as a JSON + string), e.g. `{"custom_headers": {"Referer": + "https://www.google.com/"}}`. The headers are forwarded to + the scraping target, subject to the scraper API's usual + header sanitisation. At most 32 headers / 8 KB serialised; + names must be valid HTTP tokens. Header filtering and + browser-header sanitisation are owned by the scraper API, + not Conveyor. A task-level + `custom_headers` replaces the job-level object wholesale — + no per-name merging. additionalProperties: oneOf: - { type: string } - { type: boolean } - { type: integer } + - { type: object, additionalProperties: { type: string } } TaskInput: type: object @@ -1509,12 +1541,13 @@ components: type: string enum: [pending, done] description: | - Present only on runs created by a large (202) submission. - `pending` — task rows are still streaming into storage; - reads may return partial pages and `addTasks` returns - `409`. `done` — every accepted task row is visible. - Omitted on runs whose tasks were written on the request - path (201 submissions, `addTasks` batches, reruns). + Present only on runs created by a large (202) submission + or a large (202) rerun. `pending` — task rows are still + streaming into storage; reads may return partial pages + and `addTasks` returns `409`. `done` — every accepted + task row is visible. Omitted on runs whose tasks were + written on the request path (201 submissions and + reruns, `addTasks` batches). created_at: { type: string, format: date-time } updated_at: { type: string, format: date-time } failure_reason: @@ -1731,7 +1764,7 @@ components: type: string nullable: true description: | - 2-hour presigned download URL for the result body, or a + 24-hour presigned download URL for the result body, or a `/v1/jobs//runs//tasks//content` URL you can fetch directly. Empty for non-successful tasks. error: diff --git a/src/zenrows/batch/_estimate.py b/src/zenrows/batch/_estimate.py index 01e5be2..f730ab6 100644 --- a/src/zenrows/batch/_estimate.py +++ b/src/zenrows/batch/_estimate.py @@ -50,7 +50,7 @@ _TRUTHY = frozenset({"true", "1", "yes", "on"}) # What `Tier` keys can appear, in the order a breakdown should render. -ParamValue = str | bool | int +ParamValue = str | bool | int | dict[str, str] # dict form: `custom_headers` map ParamMap = dict[str, ParamValue] TaskLike = str | TaskInput | dict diff --git a/src/zenrows/batch/client.py b/src/zenrows/batch/client.py index 1855af1..dc5ea3a 100644 --- a/src/zenrows/batch/client.py +++ b/src/zenrows/batch/client.py @@ -46,7 +46,7 @@ download_to_dir, download_to_memory, ) -from zenrows.batch._estimate import CostEstimate, _estimate_cost +from zenrows.batch._estimate import CostEstimate, ParamMap, _estimate_cost from zenrows.batch._resources import ( ExportHandle, ExportRef, @@ -236,7 +236,7 @@ def submit_regular( urls: list[str | TaskInputDict] | None = None, *, file_input_id: str | None = None, - zenrows_params: dict[str, str] | None = None, + zenrows_params: ParamMap | None = None, external_id: str | None = None, name: str | None = None, metadata: dict[str, str] | None = None, @@ -279,7 +279,7 @@ def submit_open( self, urls: list[str | TaskInputDict] | None = None, *, - zenrows_params: dict[str, str] | None = None, + zenrows_params: ParamMap | None = None, external_id: str | None = None, name: str | None = None, metadata: dict[str, str] | None = None, @@ -319,7 +319,7 @@ def submit_scheduled( urls: list[str | TaskInputDict] | None = None, *, file_input_id: str | None = None, - zenrows_params: dict[str, str] | None = None, + zenrows_params: ParamMap | None = None, external_id: str | None = None, name: str | None = None, metadata: dict[str, str] | None = None, @@ -1156,7 +1156,7 @@ def _build_submit_body( urls: list[str | TaskInputDict] | None, file_input_id: str | None, status: Literal["open", "closed"], - zenrows_params: dict[str, str] | None, + zenrows_params: ParamMap | None, schedule: JobScheduleDict | None = None, external_id: str | None = None, name: str | None = None, diff --git a/src/zenrows/batch/models.py b/src/zenrows/batch/models.py index 75e924b..90361c0 100644 --- a/src/zenrows/batch/models.py +++ b/src/zenrows/batch/models.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: openapi.yaml -# timestamp: 2026-07-08T16:45:00+00:00 +# timestamp: 2026-07-21T11:46:07+00:00 from __future__ import annotations @@ -164,7 +164,7 @@ class TaskInput(BaseModel): Must be http(s). Other schemes rejected at submit. """ metadata: Annotated[dict[str, str] | None, Field(max_length=20)] = None - zenrows_params: dict[str, str | bool | int] | None = None + zenrows_params: dict[str, str | bool | int | dict[str, str]] | None = None """ Per-task scraper params. Override the job-level `zenrows_params` on key collision (task wins). @@ -276,12 +276,13 @@ class PauseState(Enum): class IngestStatus(Enum): """ - Present only on runs created by a large (202) submission. - `pending` — task rows are still streaming into storage; - reads may return partial pages and `addTasks` returns - `409`. `done` — every accepted task row is visible. - Omitted on runs whose tasks were written on the request - path (201 submissions, `addTasks` batches, reruns). + Present only on runs created by a large (202) submission + or a large (202) rerun. `pending` — task rows are still + streaming into storage; reads may return partial pages + and `addTasks` returns `409`. `done` — every accepted + task row is visible. Omitted on runs whose tasks were + written on the request path (201 submissions and + reruns, `addTasks` batches). """ @@ -326,12 +327,13 @@ class Run(BaseModel): """ ingest_status: IngestStatus | None = None """ - Present only on runs created by a large (202) submission. - `pending` — task rows are still streaming into storage; - reads may return partial pages and `addTasks` returns - `409`. `done` — every accepted task row is visible. - Omitted on runs whose tasks were written on the request - path (201 submissions, `addTasks` batches, reruns). + Present only on runs created by a large (202) submission + or a large (202) rerun. `pending` — task rows are still + streaming into storage; reads may return partial pages + and `addTasks` returns `409`. `done` — every accepted + task row is visible. Omitted on runs whose tasks were + written on the request path (201 submissions and + reruns, `addTasks` batches). """ created_at: AwareDatetime @@ -859,7 +861,7 @@ class TaskResult(BaseModel): type: ResultType | None = None result_url: str | None = None """ - 2-hour presigned download URL for the result body, or a + 24-hour presigned download URL for the result body, or a `/v1/jobs//runs//tasks//content` URL you can fetch directly. Empty for non-successful tasks. @@ -1023,7 +1025,7 @@ class SubmitJobRequest(BaseModel): is auto-closed. """ - zenrows_params: dict[str, str | bool | int] | None = None + zenrows_params: dict[str, str | bool | int | dict[str, str]] | None = None """ Job-level scraper params, applied to every task of every run of the job. Each task can override individual keys