From e0359e1d90d1576fcf46df3a0916e14dd0779951 Mon Sep 17 00:00:00 2001 From: Emirlan Rasulov Date: Mon, 27 Jul 2026 13:39:57 +0600 Subject: [PATCH] sync openapi spec with conveyor; add per-task method/body (ENG-117) --- docs/batch-client-reference.md | 70 +++++++++++++++++++++++++++++++ docs/openapi.yaml | 30 +++++++++++++ src/zenrows/batch/_estimate.py | 6 +++ src/zenrows/batch/_typed_dicts.py | 7 ++++ src/zenrows/batch/models.py | 63 +++++++++++++++++++++++++++- tests/test_batch_client.py | 38 +++++++++++++++++ 6 files changed, 213 insertions(+), 1 deletion(-) diff --git a/docs/batch-client-reference.md b/docs/batch-client-reference.md index 875b3cb..a988a37 100644 --- a/docs/batch-client-reference.md +++ b/docs/batch-client-reference.md @@ -1597,6 +1597,12 @@ exactly one tier. If a malformed param map carries both, `auto` wins here (it's what the engine would honor) — but the server would reject that body at submit anyway. +The per-task `method` / `body` fields (POST tasks) do NOT affect the +rate card — pricing is driven by the flags above regardless of HTTP +method, and the render-tier combinations the platform can't execute +for POST (`js_render`, `js_instructions`, `json_response`) are +rejected at submit, so a priced job is a billable job. + #### ParamValue @@ -1982,6 +1988,24 @@ Precedence: Stamped on every successful task result and used to set the right `Content-Type` when you fetch the content. + + +## Method Objects + +```python +class Method(Enum) +``` + +HTTP method used against `url`. Case-insensitive. POST is +for **safe/idempotent** requests only (GraphQL queries, +search endpoints): tasks are retried on transient failures +and reruns, so the target may see the same POST more than +once. Callers that cannot tolerate a duplicate should +disable reruns. POST rides the standard (non-headless) +scraping path — combining it with `js_render`, +`js_instructions`, or `json_response` is rejected with +400 `method_param_conflict`. + ## TaskInput Objects @@ -2009,6 +2033,32 @@ assigned `task_id`. Must be http(s). Other schemes rejected at submit. + + +#### method + +HTTP method used against `url`. Case-insensitive. POST is +for **safe/idempotent** requests only (GraphQL queries, +search endpoints): tasks are retried on transient failures +and reruns, so the target may see the same POST more than +once. Callers that cannot tolerate a duplicate should +disable reruns. POST rides the standard (non-headless) +scraping path — combining it with `js_render`, +`js_instructions`, or `json_response` is rejected with +400 `method_param_conflict`. + + + +#### body + +Request body, only with `method: POST`. Any JSON value, +16 KiB max. An object/array/number/boolean is sent as its +JSON encoding with `Content-Type: application/json`; a +string is sent verbatim with +`Content-Type: application/x-www-form-urlencoded`. Set a +different target Content-Type via the `custom_headers` +zenrows param. Never echoed in results listings. + #### zenrows\_params @@ -2234,6 +2284,18 @@ Picks which days the schedule fires on. Exactly one of Fire every day. No knobs. + + +## Method1 Objects + +```python +class Method1(Enum) +``` + +The task's HTTP method. Omitted for GET (the default). +The request `body` is intentionally not part of listing +responses. + ## WebhookConfig Objects @@ -2741,6 +2803,14 @@ class TaskResult(BaseModel) Caller-supplied correlation id from submit/AddTasks. Omitted when the caller did not supply one. + + +#### method + +The task's HTTP method. Omitted for GET (the default). +The request `body` is intentionally not part of listing +responses. + #### result\_url diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e467732..afab490 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1307,6 +1307,29 @@ components: maxLength: 2048 description: Must be http(s). Other schemes rejected at submit. metadata: { $ref: '#/components/schemas/Metadata' } + method: + type: string + enum: [GET, POST] + default: GET + description: | + HTTP method used against `url`. Case-insensitive. POST is + for **safe/idempotent** requests only (GraphQL queries, + search endpoints): tasks are retried on transient failures + and reruns, so the target may see the same POST more than + once. Callers that cannot tolerate a duplicate should + disable reruns. POST rides the standard (non-headless) + scraping path — combining it with `js_render`, + `js_instructions`, or `json_response` is rejected with + 400 `method_param_conflict`. + body: + description: | + Request body, only with `method: POST`. Any JSON value, + 16 KiB max. An object/array/number/boolean is sent as its + JSON encoding with `Content-Type: application/json`; a + string is sent verbatim with + `Content-Type: application/x-www-form-urlencoded`. Set a + different target Content-Type via the `custom_headers` + zenrows param. Never echoed in results listings. zenrows_params: $ref: '#/components/schemas/ScraperParams' description: | @@ -1758,6 +1781,13 @@ components: run_id: { type: string } url: { type: string, format: uri } metadata: { $ref: '#/components/schemas/Metadata' } + method: + type: string + enum: [GET, POST] + description: | + The task's HTTP method. Omitted for GET (the default). + The request `body` is intentionally not part of listing + responses. status: { $ref: '#/components/schemas/TaskStatus' } type: { $ref: '#/components/schemas/ResultType' } result_url: diff --git a/src/zenrows/batch/_estimate.py b/src/zenrows/batch/_estimate.py index f730ab6..9deb559 100644 --- a/src/zenrows/batch/_estimate.py +++ b/src/zenrows/batch/_estimate.py @@ -28,6 +28,12 @@ exactly one tier. If a malformed param map carries both, `auto` wins here (it's what the engine would honor) — but the server would reject that body at submit anyway. + +The per-task `method` / `body` fields (POST tasks) do NOT affect the +rate card — pricing is driven by the flags above regardless of HTTP +method, and the render-tier combinations the platform can't execute +for POST (`js_render`, `js_instructions`, `json_response`) are +rejected at submit, so a priced job is a billable job. """ from collections.abc import Iterable diff --git a/src/zenrows/batch/_typed_dicts.py b/src/zenrows/batch/_typed_dicts.py index 07c5384..98b0fbb 100644 --- a/src/zenrows/batch/_typed_dicts.py +++ b/src/zenrows/batch/_typed_dicts.py @@ -32,6 +32,13 @@ class TaskInputDict(TypedDict, total=False): url: str external_id: NotRequired[str] metadata: NotRequired[dict[str, str]] + # HTTP method against `url`: "GET" (default) | "POST". POST is for + # safe/idempotent requests only — tasks are retried, so the target + # may see the same POST more than once. + method: NotRequired[str] + # Request body, POST only. Any JSON value: object/array/number/bool + # → application/json; a str is sent verbatim as form-urlencoded. + body: NotRequired[Any] zenrows_params: NotRequired[dict[str, Any]] diff --git a/src/zenrows/batch/models.py b/src/zenrows/batch/models.py index 90361c0..185e42a 100644 --- a/src/zenrows/batch/models.py +++ b/src/zenrows/batch/models.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: openapi.yaml -# timestamp: 2026-07-21T11:46:07+00:00 +# timestamp: 2026-07-27T07:37:28+00:00 from __future__ import annotations @@ -144,6 +144,24 @@ class Format(Enum): PDF = "pdf" +class Method(Enum): + """ + HTTP method used against `url`. Case-insensitive. POST is + for **safe/idempotent** requests only (GraphQL queries, + search endpoints): tasks are retried on transient failures + and reruns, so the target may see the same POST more than + once. Callers that cannot tolerate a duplicate should + disable reruns. POST rides the standard (non-headless) + scraping path — combining it with `js_render`, + `js_instructions`, or `json_response` is rejected with + 400 `method_param_conflict`. + + """ + + GET = "GET" + POST = "POST" + + class TaskInput(BaseModel): external_id: Annotated[ str | None, Field(max_length=128, pattern="^[A-Za-z0-9._-]+$") @@ -164,6 +182,30 @@ class TaskInput(BaseModel): Must be http(s). Other schemes rejected at submit. """ metadata: Annotated[dict[str, str] | None, Field(max_length=20)] = None + method: Method | None = "GET" + """ + HTTP method used against `url`. Case-insensitive. POST is + for **safe/idempotent** requests only (GraphQL queries, + search endpoints): tasks are retried on transient failures + and reruns, so the target may see the same POST more than + once. Callers that cannot tolerate a duplicate should + disable reruns. POST rides the standard (non-headless) + scraping path — combining it with `js_render`, + `js_instructions`, or `json_response` is rejected with + 400 `method_param_conflict`. + + """ + body: Any | None = None + """ + Request body, only with `method: POST`. Any JSON value, + 16 KiB max. An object/array/number/boolean is sent as its + JSON encoding with `Content-Type: application/json`; a + string is sent verbatim with + `Content-Type: application/x-www-form-urlencoded`. Set a + different target Content-Type via the `custom_headers` + zenrows param. Never echoed in results listings. + + """ zenrows_params: dict[str, str | bool | int | dict[str, str]] | None = None """ Per-task scraper params. Override the job-level @@ -410,6 +452,18 @@ class ListJobRunsResponse(BaseModel): next_cursor: str | None = None +class Method1(Enum): + """ + The task's HTTP method. Omitted for GET (the default). + The request `body` is intentionally not part of listing + responses. + + """ + + GET = "GET" + POST = "POST" + + class WebhookConfig(BaseModel): """ Webhook delivery config for `run.completed` / `run.failed`. Returned on @@ -857,6 +911,13 @@ class TaskResult(BaseModel): run_id: str url: AnyUrl metadata: Annotated[dict[str, str] | None, Field(max_length=20)] = None + method: Method1 | None = None + """ + The task's HTTP method. Omitted for GET (the default). + The request `body` is intentionally not part of listing + responses. + + """ status: TaskStatus type: ResultType | None = None result_url: str | None = None diff --git a/tests/test_batch_client.py b/tests/test_batch_client.py index 09f893f..04c3fb3 100644 --- a/tests/test_batch_client.py +++ b/tests/test_batch_client.py @@ -1053,3 +1053,41 @@ def test_external_id_filename_coerces_to_safe_name(): missing = _task_result("T1", result_type="html") assert _external_id_filename(missing) == "T1.html" # falls back to task_id + + +@respx.mock +def test_submit_job_post_task_method_body_on_wire(client: ZenRowsBatchClient): + """A POST task's method/body ride the wire verbatim; tasks that + don't set them send neither key (exclude_unset — the server treats + absent method as GET).""" + route = respx.post(f"{BASE_URL}/jobs").mock( + return_value=Response( + 201, + json={ + "job_id": "01J0000000000000000000000", + "status": "closed", + "accepted_tasks": 2, + }, + ) + ) + + client.submit_job( + { + "type": "regular", + "status": "closed", + "tasks": [ + { + "url": "https://api.example.com/graphql", + "method": "POST", + "body": {"query": "{ products { id } }"}, + }, + {"url": "https://example.com/plain"}, + ], + } + ) + + sent = json.loads(route.calls.last.request.content) + assert sent["tasks"][0]["method"] == "POST" + assert sent["tasks"][0]["body"] == {"query": "{ products { id } }"} + assert "method" not in sent["tasks"][1] + assert "body" not in sent["tasks"][1]