From 2760925525af51001e57cd51c63af4f8905eefeb Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 19 Aug 2026 08:44:05 +0200 Subject: [PATCH] feat: Add HTTPX-based HTTP client --- README.md | 11 +- pyproject.toml | 1 + src/apify_client/http_clients/__init__.py | 39 +++- src/apify_client/http_clients/_httpx.py | 244 ++++++++++++++++++++++ tests/integration/conftest.py | 76 +++++-- tests/integration/test_apify_client.py | 4 + tests/integration/test_dataset.py | 2 + tests/integration/test_key_value_store.py | 2 + tests/integration/test_log.py | 4 + tests/unit/conftest.py | 23 +- tests/unit/test_client_headers.py | 73 ++++++- tests/unit/test_client_streaming.py | 4 +- tests/unit/test_client_timeouts.py | 77 +++++-- tests/unit/test_http_clients.py | 110 ++++++++++ tests/unit/test_logging.py | 4 +- tests/unit/test_pluggable_http_client.py | 46 ++++ uv.lock | 6 +- 17 files changed, 668 insertions(+), 58 deletions(-) create mode 100644 src/apify_client/http_clients/_httpx.py diff --git a/README.md b/README.md index ba5b9084..7627e5fe 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,15 @@ uv add "apify-client[brotli]" ``` + [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the + built-in [HTTPX](https://www.python-httpx.org/) client instead, install its optional extra: + + ```bash + pip install "apify-client[httpx]" + # or + uv add "apify-client[httpx]" + ``` + - From [conda-forge](https://anaconda.org/conda-forge/apify-client), it can be installed with [conda](https://docs.conda.io/en/latest/): ```bash @@ -124,7 +133,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). -- **Pluggable HTTP layer** — swap the default [Impit](https://github.com/apify/impit)-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). +- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://www.python-httpx.org/) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)). diff --git a/pyproject.toml b/pyproject.toml index 0019741f..dc23c5df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ [project.optional-dependencies] brotli = ["brotli>=1.0.9"] +httpx = ["httpx>=0.27.0,<1.0.0"] [project.urls] "Apify Homepage" = "https://apify.com" diff --git a/src/apify_client/http_clients/__init__.py b/src/apify_client/http_clients/__init__.py index 417a0705..d1e06c90 100644 --- a/src/apify_client/http_clients/__init__.py +++ b/src/apify_client/http_clients/__init__.py @@ -1,10 +1,35 @@ +from apify_client._utils.try_import import install_import_hook as _install_import_hook +from apify_client._utils.try_import import try_import as _try_import from apify_client.http_clients._base import HttpClient, HttpClientAsync, HttpResponse from apify_client.http_clients._impit import ImpitHttpClient, ImpitHttpClientAsync -__all__ = [ - 'HttpClient', - 'HttpClientAsync', - 'HttpResponse', - 'ImpitHttpClient', - 'ImpitHttpClientAsync', -] +_install_import_hook(__name__) + +# `httpx` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX clients +# without the extra installed raises a clear ImportError instead of failing at package import time. +with _try_import( + __name__, + 'HttpxHttpClient', + 'HttpxHttpClientAsync', + dependency_name='httpx', +) as _httpx_import: + from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync + +if _httpx_import.available: + __all__ = [ + 'HttpClient', + 'HttpClientAsync', + 'HttpResponse', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', + 'ImpitHttpClient', + 'ImpitHttpClientAsync', + ] +else: + __all__ = [ + 'HttpClient', + 'HttpClientAsync', + 'HttpResponse', + 'ImpitHttpClient', + 'ImpitHttpClientAsync', + ] diff --git a/src/apify_client/http_clients/_httpx.py b/src/apify_client/http_clients/_httpx.py new file mode 100644 index 00000000..8e256492 --- /dev/null +++ b/src/apify_client/http_clients/_httpx.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import httpx +from typing_extensions import override + +from apify_client._consts import ( + DEFAULT_MAX_RETRIES, + DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + DEFAULT_TIMEOUT_LONG, + DEFAULT_TIMEOUT_MAX, + DEFAULT_TIMEOUT_MEDIUM, + DEFAULT_TIMEOUT_SHORT, +) +from apify_client._docs import docs_group +from apify_client.http_clients._base import HttpClient, HttpClientAsync + +if TYPE_CHECKING: + from datetime import timedelta + + from apify_client._statistics import ClientStatistics + from apify_client.http_compressors._base import HttpCompressor + + +_PERMANENT_ERRORS = ( + # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value. + httpx.LocalProtocolError, + # A URL scheme HTTPX refuses to speak, which repeating the request cannot change. + httpx.UnsupportedProtocol, + # An over-long redirect chain is a routing loop, which repeating the request cannot break. + httpx.TooManyRedirects, + # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on + # status codes from the response itself. + httpx.HTTPStatusError, +) +"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient.""" + + +@docs_group('HTTP clients') +class HttpxHttpClient(HttpClient): + """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/). + + This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited + (HTTP 429) and server error (HTTP 5xx) responses. + + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. + """ + + def __init__( + self, + *, + token: str | None = None, + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, + max_retries: int = DEFAULT_MAX_RETRIES, + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + statistics: ClientStatistics | None = None, + headers: dict[str, str] | None = None, + http_compressor: HttpCompressor | None = None, + ) -> None: + """Initialize the HTTPX-based synchronous HTTP client. + + Args: + token: Apify API token for authentication. + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. + max_retries: Maximum number of retry attempts for failed requests. + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). + statistics: Statistics tracker for API calls. Created automatically if not provided. + headers: Additional HTTP headers to include in all requests. + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + """ + super().__init__( + token=token, + timeout_short=timeout_short, + timeout_medium=timeout_medium, + timeout_long=timeout_long, + timeout_max=timeout_max, + max_retries=max_retries, + min_delay_between_retries=min_delay_between_retries, + statistics=statistics, + headers=headers, + http_compressor=http_compressor, + ) + + self._httpx_client = httpx.Client( + follow_redirects=True, + event_hooks={'response': [self._clear_response_cookies]}, + ) + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures + # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the + # response status code, not here. + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + + @override + def close(self) -> None: + """Close the underlying HTTPX connection pool.""" + self._httpx_client.close() + + def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_client.cookies.clear() + + @override + def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> httpx.Response: + request = self._httpx_client.build_request( + method=method, + url=url, + headers=headers, + content=content, + timeout=timeout, + ) + _restore_explicit_cookie_header(request, headers) + return self._httpx_client.send(request, stream=stream) + + +@docs_group('HTTP clients') +class HttpxHttpClientAsync(HttpClientAsync): + """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/). + + This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited + (HTTP 429) and server error (HTTP 5xx) responses. + + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. + """ + + def __init__( + self, + *, + token: str | None = None, + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, + max_retries: int = DEFAULT_MAX_RETRIES, + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + statistics: ClientStatistics | None = None, + headers: dict[str, str] | None = None, + http_compressor: HttpCompressor | None = None, + ) -> None: + """Initialize the HTTPX-based asynchronous HTTP client. + + Args: + token: Apify API token for authentication. + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. + max_retries: Maximum number of retry attempts for failed requests. + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). + statistics: Statistics tracker for API calls. Created automatically if not provided. + headers: Additional HTTP headers to include in all requests. + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + """ + super().__init__( + token=token, + timeout_short=timeout_short, + timeout_medium=timeout_medium, + timeout_long=timeout_long, + timeout_max=timeout_max, + max_retries=max_retries, + min_delay_between_retries=min_delay_between_retries, + statistics=statistics, + headers=headers, + http_compressor=http_compressor, + ) + + self._httpx_async_client = httpx.AsyncClient( + follow_redirects=True, + event_hooks={'response': [self._clear_response_cookies]}, + ) + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures + # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the + # response status code, not here. + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + + @override + async def aclose(self) -> None: + """Close the underlying asynchronous HTTPX connection pool.""" + await self._httpx_async_client.aclose() + + async def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_async_client.cookies.clear() + + @override + async def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> httpx.Response: + request = self._httpx_async_client.build_request( + method=method, + url=url, + headers=headers, + content=content, + timeout=timeout, + ) + _restore_explicit_cookie_header(request, headers) + return await self._httpx_async_client.send(request, stream=stream) + + +def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None: + """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar.""" + explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None) + if explicit_cookie is None: + request.headers.pop('cookie', None) + else: + request.headers['cookie'] = explicit_cookie diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1db53b71..e0c9bed6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,6 +2,7 @@ import json import os +from dataclasses import dataclass from typing import TYPE_CHECKING import pytest @@ -17,9 +18,35 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client._consts import DEFAULT_API_URL from apify_client._utils.crypto import create_hmac_signature, create_storage_content_signature +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import AsyncGenerator, Generator + + +@dataclass(frozen=True) +class HttpClientClasses: + """Synchronous and asynchronous variants of a built-in HTTP client.""" + + sync: type[HttpClient] + async_: type[HttpClientAsync] + + +DEFAULT_HTTP_CLIENT_CLASSES = HttpClientClasses(sync=ImpitHttpClient, async_=ImpitHttpClientAsync) +"""HTTP clients the live-API suite runs with unless a test asks for another transport.""" + +ALL_HTTP_CLIENT_CLASSES = [ + pytest.param(DEFAULT_HTTP_CLIENT_CLASSES, id='impit'), + pytest.param(HttpClientClasses(sync=HttpxHttpClient, async_=HttpxHttpClientAsync), id='httpx'), +] +"""Every built-in HTTP client, for tests that exercise transport behavior rather than an API resource.""" # ============================================================================ @@ -110,17 +137,17 @@ def test_kvs_of_another_user(api_token_2: str) -> Generator[KvsFixture]: @pytest.fixture -def apify_client(api_token: str) -> ApifyClient: - """Sync Apify client instance.""" - api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL - return ApifyClient(api_token, api_url=api_url) +def http_client_classes(request: pytest.FixtureRequest) -> HttpClientClasses: + """Return the sync and async classes of the HTTP client the test runs with. + Defaults to Impit so the live-API suite isn't multiplied by every transport. A transport-level test opts into + the full matrix with `@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True)`. + """ + if not hasattr(request, 'param'): + return DEFAULT_HTTP_CLIENT_CLASSES -@pytest.fixture -def apify_client_async(api_token: str) -> ApifyClientAsync: - """Async Apify client instance.""" - api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL - return ApifyClientAsync(api_token, api_url=api_url) + assert isinstance(request.param, HttpClientClasses) + return request.param @pytest.fixture(params=['sync', 'async']) @@ -130,13 +157,30 @@ def client_type(request: pytest.FixtureRequest) -> str: @pytest.fixture -def client( +async def client( client_type: str, - apify_client: ApifyClient, - apify_client_async: ApifyClientAsync, -) -> ApifyClient | ApifyClientAsync: - """Return sync or async client based on parametrization.""" - return apify_client if client_type == 'sync' else apify_client_async + api_token: str, + http_client_classes: HttpClientClasses, +) -> AsyncGenerator[ApifyClient | ApifyClientAsync]: + """Return each sync/async and HTTP client implementation combination.""" + api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL + if client_type == 'sync': + http_client = http_client_classes.sync() + yield ApifyClient.with_custom_http_client( + api_token, + api_url=api_url, + http_client=http_client, + ) + http_client.close() + return + + http_client_async = http_client_classes.async_() + yield ApifyClientAsync.with_custom_http_client( + api_token, + api_url=api_url, + http_client=http_client_async, + ) + await http_client_async.aclose() @pytest.fixture diff --git a/tests/integration/test_apify_client.py b/tests/integration/test_apify_client.py index 126f40b3..4c15eab8 100644 --- a/tests/integration/test_apify_client.py +++ b/tests/integration/test_apify_client.py @@ -4,13 +4,17 @@ from typing import TYPE_CHECKING +import pytest + from .._utils import maybe_await +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import UserPrivateInfo, UserPublicInfo if TYPE_CHECKING: from apify_client import ApifyClient, ApifyClientAsync +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_apify_client(client: ApifyClient | ApifyClientAsync) -> None: """Test basic apify client functionality.""" user_client = client.user('me') diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 333c7229..b7acab4c 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -18,6 +18,7 @@ maybe_await, poll_until_condition, ) +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import Dataset, DatasetListItem, DatasetStatistics, ListOfDatasets from apify_client._resource_clients.dataset import DatasetItemsPage from apify_client.errors import ApifyApiError @@ -698,6 +699,7 @@ async def get_items() -> DatasetItemsPage: await maybe_await(dataset_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_dataset_stream_items(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming dataset items.""" dataset_name = get_random_resource_name('dataset') diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 5d1d9238..fee82954 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -19,6 +19,7 @@ maybe_sleep, poll_until_condition, ) +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import KeyValueStore, KeyValueStoreKey, ListOfKeys, ListOfKeyValueStores from apify_client.errors import ApifyApiError from apify_client.http_clients import HttpResponse @@ -706,6 +707,7 @@ async def get_keys() -> ListOfKeys: await maybe_await(store_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_key_value_store_stream_record_own(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming a record from one's own key-value store (no signature).""" store_name = get_random_resource_name('kvs') diff --git a/tests/integration/test_log.py b/tests/integration/test_log.py index df687e2f..13db91f8 100644 --- a/tests/integration/test_log.py +++ b/tests/integration/test_log.py @@ -5,7 +5,10 @@ from contextlib import AbstractAsyncContextManager, AbstractContextManager from typing import TYPE_CHECKING +import pytest + from .._utils import maybe_await +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import ListOfBuilds, Run from apify_client.http_clients import HttpResponse @@ -72,6 +75,7 @@ async def test_log_get_as_bytes(client: ApifyClient | ApifyClientAsync) -> None: await maybe_await(run_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_log_stream_from_run(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming a run's log via the stream() context manager.""" actor = client.actor(HELLO_WORLD_ACTOR) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index b732cc1c..d1391a3a 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -6,7 +6,14 @@ import pytest from pytest_httpserver import HTTPServer -from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) if TYPE_CHECKING: from collections.abc import Iterable @@ -32,13 +39,23 @@ def httpserver(make_httpserver: HTTPServer) -> Iterable[HTTPServer]: server.clear() -@pytest.fixture(params=[pytest.param(ImpitHttpClient, id='impit')]) +@pytest.fixture( + params=[ + pytest.param(ImpitHttpClient, id='impit'), + pytest.param(HttpxHttpClient, id='httpx'), + ] +) def http_client_class(request: pytest.FixtureRequest) -> type[HttpClient]: """Return each built-in synchronous HTTP client class.""" return request.param -@pytest.fixture(params=[pytest.param(ImpitHttpClientAsync, id='impit')]) +@pytest.fixture( + params=[ + pytest.param(ImpitHttpClientAsync, id='impit'), + pytest.param(HttpxHttpClientAsync, id='httpx'), + ] +) def http_client_async_class(request: pytest.FixtureRequest) -> type[HttpClientAsync]: """Return each built-in asynchronous HTTP client class.""" return request.param diff --git a/tests/unit/test_client_headers.py b/tests/unit/test_client_headers.py index 981d6857..b8e0b259 100644 --- a/tests/unit/test_client_headers.py +++ b/tests/unit/test_client_headers.py @@ -6,8 +6,11 @@ from importlib import metadata from typing import TYPE_CHECKING +import httpx from werkzeug import Request, Response +from apify_client.http_clients import HttpxHttpClient, HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync + if TYPE_CHECKING: from pytest_httpserver import HTTPServer @@ -19,6 +22,18 @@ def _parse_accept_encoding(header: str) -> set[str]: return {enc.strip() for enc in header.split(',')} +def _transport_wire_headers( + client_class: type[HttpClient | HttpClientAsync], +) -> tuple[dict[str, str], set[str]]: + """Return the headers the transport adds on its own and the content encodings it advertises.""" + if issubclass(client_class, (ImpitHttpClient, ImpitHttpClientAsync)): + return {}, {'zstd', 'gzip', 'deflate', 'br'} + # HTTPX advertises whichever decoders happen to be installed alongside it, so read the set off the client + # itself rather than hard-coding it and breaking whenever the environment gains or loses a codec. + with httpx.Client() as probe: + return {'Connection': 'keep-alive'}, _parse_accept_encoding(probe.headers['accept-encoding']) + + def _header_handler(request: Request) -> Response: return Response( status=200, @@ -43,15 +58,17 @@ async def test_default_headers_async(httpserver: HTTPServer, http_client_async_c response = await client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_async_class) expected_headers = { 'User-Agent': _get_user_agent(), 'Accept': 'application/json, */*', 'Authorization': 'Bearer placeholder_token', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings def test_default_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient]) -> None: @@ -63,15 +80,17 @@ def test_default_headers_sync(httpserver: HTTPServer, http_client_class: type[Ht response = client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_class) expected_headers = { 'User-Agent': _get_user_agent(), 'Accept': 'application/json, */*', 'Authorization': 'Bearer placeholder_token', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings async def test_headers_async(httpserver: HTTPServer, http_client_async_class: type[HttpClientAsync]) -> None: @@ -86,6 +105,7 @@ async def test_headers_async(httpserver: HTTPServer, http_client_async_class: ty response = await client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_async_class) expected_headers = { 'Test-Header': 'blah', @@ -93,9 +113,10 @@ async def test_headers_async(httpserver: HTTPServer, http_client_async_class: ty 'Accept': 'application/json, */*', 'Authorization': 'strange_value', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient]) -> None: @@ -114,6 +135,7 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient response = client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_class) expected_headers = { 'Test-Header': 'blah', @@ -121,9 +143,10 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient 'Accept': 'application/json, */*', 'Authorization': 'strange_value', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings async def test_per_request_headers_override_defaults_async( @@ -158,3 +181,45 @@ def test_per_request_headers_override_defaults_sync( # WSGI joins duplicate headers into one comma-separated value, so exact equality # also proves the authorization header was sent only once. assert request_headers['Authorization'] == 'Bearer per-request' + + +def _echo_cookie_handler(request: Request) -> Response: + return Response(json.dumps({'cookie': request.headers.get('Cookie')}), content_type='application/json') + + +def test_httpx_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """A Set-Cookie response must not silently leak into a later API request through HTTPX's shared cookie jar.""" + httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + client.call(method='GET', url=httpserver.url_for('/set-cookie')) + response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +async def test_httpx_async_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """The asynchronous HTTPX pool also remains stateless between API calls.""" + httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + await client.call(method='GET', url=httpserver.url_for('/set-cookie')) + response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: + """Disabling the shared cookie jar must not remove a Cookie header explicitly supplied by the caller.""" + httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + response = client.call( + method='GET', + url=httpserver.url_for('/echo-explicit-cookie'), + headers={'Cookie': 'explicit=value'}, + ) + + assert response.json() == {'cookie': 'explicit=value'} diff --git a/tests/unit/test_client_streaming.py b/tests/unit/test_client_streaming.py index d369455e..9e5782e7 100644 --- a/tests/unit/test_client_streaming.py +++ b/tests/unit/test_client_streaming.py @@ -105,7 +105,7 @@ def test_protocol_check_leaves_stream_unread_sync( with client.dataset(DATASET_ID).stream_items(item_format='json') as response: assert isinstance(response, HttpResponse) - # `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it. + # `is_stream_consumed` is transport state, not part of the protocol, but both built-in clients expose it. raw: Any = response assert raw.is_stream_consumed is False @@ -124,6 +124,6 @@ async def test_protocol_check_leaves_stream_unread_async( async with client.dataset(DATASET_ID).stream_items(item_format='json') as response: assert isinstance(response, HttpResponse) - # `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it. + # `is_stream_consumed` is transport state, not part of the protocol, but both built-in clients expose it. raw: Any = response assert raw.is_stream_consumed is False diff --git a/tests/unit/test_client_timeouts.py b/tests/unit/test_client_timeouts.py index b4410acd..58fd03b2 100644 --- a/tests/unit/test_client_timeouts.py +++ b/tests/unit/test_client_timeouts.py @@ -5,16 +5,27 @@ from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock +import httpx import impit import pytest from apify_client._logging import LoggerOnce, logger_name -from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) from apify_client.http_clients import _base as http_client_base if TYPE_CHECKING: from _pytest.logging import LogCaptureFixture +UNSET_HTTPX_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} +"""What HTTPX stores on a request built with `timeout=None`: every sub-timeout unset, not the client default.""" + @pytest.fixture def fresh_logger_once(monkeypatch: pytest.MonkeyPatch) -> None: @@ -26,6 +37,12 @@ def successful_response() -> Mock: return Mock(status_code=200) +def retryable_error(client: HttpClient | HttpClientAsync) -> Exception: + if isinstance(client, (ImpitHttpClient, ImpitHttpClientAsync)): + return impit.TimeoutException('timeout') + return httpx.ReadTimeout('timeout', request=httpx.Request('GET', 'https://example.com')) + + @pytest.mark.parametrize( ('timeout', 'expected'), [ @@ -93,7 +110,7 @@ async def test_timeout_resolves_for_async_clients( def test_compute_timeout_with_timedelta(http_client_class: type[HttpClient]) -> None: - """Concrete timedeltas double per attempt, are capped at the maximum, and `no_timeout` stays unbounded.""" + """Concrete timedeltas double per attempt and are capped at the configured maximum.""" client = http_client_class(timeout_max=timedelta(seconds=20)) assert client._compute_timeout(timedelta(seconds=5), attempt=1) == 5.0 @@ -160,7 +177,7 @@ def test_dynamic_timeout_sync_client(http_client_class: type[HttpClient], monkey def send_request(*_args: Any, **kwargs: Any) -> Mock: timeouts.append(kwargs['timeout']) if len(timeouts) < 4: - raise impit.TimeoutException('timeout') + raise retryable_error(client) return successful_response() monkeypatch.setattr(client, 'send_request', send_request) @@ -185,7 +202,7 @@ async def test_dynamic_timeout_async_client( async def send_request(*_args: Any, **kwargs: Any) -> Mock: timeouts.append(kwargs['timeout']) if len(timeouts) < 4: - raise impit.TimeoutException('timeout') + raise retryable_error(client) return successful_response() monkeypatch.setattr(client, 'send_request', send_request) @@ -196,23 +213,39 @@ async def send_request(*_args: Any, **kwargs: Any) -> Mock: assert response.status_code == 200 -def test_no_timeout_mapping_for_sync_adapter() -> None: - """The synchronous adapter maps no-timeout to Impit's effectively unbounded value.""" - client = ImpitHttpClient() - client._impit_client = Mock(request=Mock(return_value=successful_response())) - - client.send_request(method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False) - - assert client._impit_client.request.call_args.kwargs['timeout'] == 86_400 - - -async def test_no_timeout_mapping_for_async_adapter() -> None: - """The asynchronous adapter maps no-timeout to Impit's effectively unbounded value.""" - client = ImpitHttpClientAsync() - client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) - - await client.send_request( +def test_no_timeout_mapping_for_sync_adapters(monkeypatch: pytest.MonkeyPatch) -> None: + """Each synchronous adapter maps no-timeout to its underlying library semantics.""" + impit_client = ImpitHttpClient() + impit_client._impit_client = Mock(request=Mock(return_value=successful_response())) + impit_client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) - - assert client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 + assert impit_client._impit_client.request.call_args.kwargs['timeout'] == 86_400 + + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + with HttpxHttpClient() as httpx_client: + send = Mock(return_value=successful_response()) + monkeypatch.setattr(httpx_client._httpx_client, 'send', send) + httpx_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT + + +async def test_no_timeout_mapping_for_async_adapters(monkeypatch: pytest.MonkeyPatch) -> None: + """Each asynchronous adapter maps no-timeout to its underlying library semantics.""" + impit_client = ImpitHttpClientAsync() + impit_client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) + await impit_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert impit_client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 + + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + async with HttpxHttpClientAsync() as httpx_client: + send = AsyncMock(return_value=successful_response()) + monkeypatch.setattr(httpx_client._httpx_async_client, 'send', send) + await httpx_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index 4b252d09..8a3aedd8 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, Mock import brotli +import httpx import impit import pytest @@ -21,6 +22,8 @@ HttpClient, HttpClientAsync, HttpResponse, + HttpxHttpClient, + HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -264,6 +267,24 @@ async def test_http_client_async_creates_async_impit_client() -> None: await client.aclose() +def test_http_client_creates_sync_httpx_client() -> None: + """The synchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClient(token='test_token_123') + + assert isinstance(client._httpx_client, httpx.Client) + client.close() + assert client._httpx_client.is_closed + + +async def test_http_client_async_creates_async_httpx_client() -> None: + """The asynchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClientAsync(token='test_token_123') + + assert isinstance(client._httpx_async_client, httpx.AsyncClient) + await client.aclose() + assert client._httpx_async_client.is_closed + + def test_parse_params_none() -> None: """Test _parse_params with None input.""" assert HttpClient._parse_params(None) is None @@ -392,6 +413,70 @@ async def test_async_http_client_classifies_timeout_errors() -> None: assert not client.is_timeout_error(ValueError('test')) +@pytest.mark.parametrize( + 'exc', + [ + # Even the generic base class is transient: HTTPX subclasses it for every failure mode, so an + # unclassified failure is safer to retry. + pytest.param(httpx.HTTPError('unclassified failure'), id='bare HTTPError'), + pytest.param(httpx.TimeoutException('timeout'), id='TimeoutException'), + pytest.param(httpx.NetworkError('network error'), id='NetworkError'), + pytest.param(httpx.RemoteProtocolError('remote protocol error'), id='RemoteProtocolError'), + pytest.param(httpx.DecodingError('decoding error'), id='DecodingError'), + # One `ProxyError` covers both a proxy rejecting the CONNECT tunnel and a 407, so a transient case cannot + # be told from a permanent one - retrying is the safer default. + pytest.param(httpx.ProxyError('proxy error'), id='ProxyError'), + ], +) +def test_httpx_is_retryable_transport_error(exc: Exception) -> None: + """A transient HTTPX transport failure is classified as retryable.""" + with HttpxHttpClient() as client: + assert client.is_retryable_transport_error(exc) + + +@pytest.mark.parametrize( + 'exc', + [ + pytest.param(httpx.LocalProtocolError('invalid header value'), id='LocalProtocolError'), + pytest.param(httpx.UnsupportedProtocol('unsupported scheme'), id='UnsupportedProtocol'), + pytest.param(httpx.TooManyRedirects('too many redirects'), id='TooManyRedirects'), + pytest.param( + httpx.HTTPStatusError( + 'status error', + request=httpx.Request('GET', 'https://example.com'), + response=httpx.Response(500), + ), + id='HTTPStatusError', + ), + # HTTPX reports a bad URL outside the `httpx.HTTPError` tree entirely. + pytest.param(httpx.InvalidURL('unsupported scheme'), id='InvalidURL'), + pytest.param(ValueError('value error'), id='ValueError'), + pytest.param(RuntimeError('runtime error'), id='RuntimeError'), + pytest.param(Exception('generic exception'), id='Exception'), + ], +) +def test_httpx_is_not_retryable_transport_error(exc: Exception) -> None: + """A transport failure a retry cannot fix, and anything outside HTTPX's hierarchy, is not retried.""" + with HttpxHttpClient() as client: + assert not client.is_retryable_transport_error(exc) + + +def test_sync_httpx_client_classifies_timeout_errors() -> None: + """The built-in synchronous HTTPX client exposes transport-neutral timeout classification.""" + with HttpxHttpClient() as client: + assert client.is_timeout_error(TimeoutError('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) + assert not client.is_timeout_error(ValueError('test')) + + +async def test_async_httpx_client_classifies_timeout_errors() -> None: + """The built-in asynchronous HTTPX client exposes transport-neutral timeout classification.""" + async with HttpxHttpClientAsync() as client: + assert client.is_timeout_error(TimeoutError('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) + assert not client.is_timeout_error(ValueError('test')) + + def test_permanent_transport_error_is_not_retried() -> None: """A transport error a retry cannot fix fails on the first attempt instead of burning the whole backoff.""" client = ImpitHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) @@ -429,6 +514,31 @@ def test_transient_transport_error_is_retried() -> None: assert request.call_count == 3 +def test_httpx_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter feeds the same fail-fast classification into the shared pipeline.""" + with HttpxHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send_request = Mock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client, 'send_request', send_request) + + with pytest.raises(httpx.UnsupportedProtocol): + client.call(method='GET', url='https://api.test.com/endpoint') + + send_request.assert_called_once() + + +def test_httpx_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter keeps a transient transport failure inside the shared retry loop.""" + with HttpxHttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: + send_request = Mock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client, 'send_request', send_request) + + with pytest.raises(httpx.TimeoutException): + client.call(method='GET', url='https://api.test.com/endpoint') + + # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. + assert send_request.call_count == 3 + + def test_error_response_read_failure_is_retried_and_closed() -> None: """A failure while buffering a streamed error body is retried like a failed send, and the response is closed.""" client = ImpitHttpClient(token='test_token', max_retries=1, min_delay_between_retries=timedelta(0)) diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index 35b10e2e..4c2f2271 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -877,7 +877,7 @@ def test_streamed_log_sync_does_not_leak_exception_on_stream_timeout( http_client_class: type[HttpClient], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The streaming thread ends quietly when the transport times out while reading the log stream.""" + """The streaming thread ends quietly when either transport times out while reading the log stream.""" monkeypatch.setattr(StreamedLog, '_stream_timeout', timedelta(seconds=1)) release_server = threading.Event() @@ -970,7 +970,7 @@ async def test_streamed_log_async_does_not_error_on_stream_timeout( http_client_async_class: type[HttpClientAsync], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The async streaming task treats a transport stream timeout as an expected terminal condition.""" + """The async streaming task treats either transport's stream timeout as an expected terminal condition.""" monkeypatch.setattr(StreamedLogAsync, '_stream_timeout', timedelta(seconds=1)) release_server = threading.Event() diff --git a/tests/unit/test_pluggable_http_client.py b/tests/unit/test_pluggable_http_client.py index c499095b..8ec5a164 100644 --- a/tests/unit/test_pluggable_http_client.py +++ b/tests/unit/test_pluggable_http_client.py @@ -2,9 +2,12 @@ import asyncio import json as jsonlib +import subprocess +import sys from dataclasses import dataclass, field from datetime import timedelta from http.client import HTTPConnection +from textwrap import dedent from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock from urllib.parse import urlsplit @@ -368,6 +371,8 @@ def test_public_exports() -> None: 'HttpClient', 'HttpClientAsync', 'HttpResponse', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', 'ImpitHttpClient', 'ImpitHttpClientAsync', ): @@ -377,6 +382,47 @@ def test_public_exports() -> None: assert not hasattr(http_clients_module, 'HttpClientBase') +def test_httpx_clients_raise_clear_error_when_extra_missing() -> None: + """Missing HTTPX keeps normal and star imports usable while explicit HTTPX access raises a clear error.""" + script = dedent( + """ + import sys + + class BlockHttpx: + def find_spec(self, name, *_args): + if name == 'httpx' or name.startswith('httpx.'): + raise ModuleNotFoundError(f"No module named '{name}'", name='httpx') + return None + + sys.meta_path.insert(0, BlockHttpx()) + + import apify_client.http_clients as module + assert module.HttpClient is not None + assert module.ImpitHttpClient is not None + + namespace = {} + exec('from apify_client.http_clients import *', namespace) + assert namespace['HttpClient'] is module.HttpClient + assert 'HttpxHttpClient' not in namespace + + for name in ('HttpxHttpClient', 'HttpxHttpClientAsync'): + try: + getattr(module, name) + except ImportError as exc: + assert "No module named 'httpx'" in str(exc) + else: + raise AssertionError(f'{name} did not raise ImportError') + """ + ) + result = subprocess.run( # noqa: S603 + [sys.executable, '-c', script], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + # -- http_client property -- diff --git a/uv.lock b/uv.lock index 51cff51f..03dfaa88 100644 --- a/uv.lock +++ b/uv.lock @@ -54,6 +54,9 @@ dependencies = [ brotli = [ { name = "brotli" }, ] +httpx = [ + { name = "httpx" }, +] [package.dev-dependencies] dev = [ @@ -80,12 +83,13 @@ dev = [ requires-dist = [ { name = "brotli", marker = "extra == 'brotli'", specifier = ">=1.0.9" }, { name = "colorama", specifier = ">=0.4.0" }, + { name = "httpx", marker = "extra == 'httpx'", specifier = ">=0.27.0,<1.0.0" }, { name = "impit", specifier = "~=0.13.0" }, { name = "more-itertools", specifier = ">=10.0.0" }, { name = "pydantic", extras = ["email"], specifier = ">=2.11.0" }, { name = "typing-extensions", specifier = ">=4.6.0" }, ] -provides-extras = ["brotli"] +provides-extras = ["brotli", "httpx"] [package.metadata.requires-dev] dev = [