From 0ddff779cfe2a8c5445d90dcc3ae75f17e90d0d6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 20:55:33 +0000 Subject: [PATCH 1/3] fix: add default HTTP request timeout to prevent indefinite hangs When an HTTP server accepts a connection but never responds (or stalls mid-transfer), the requests library will block indefinitely if no timeout is set. This causes connector syncs to hang permanently with no log output, no error, and no retry. This adds a default timeout of (30s connect, 300s read) to all HTTP requests made via HttpClient.send_request(). The timeout is only applied when no explicit timeout is provided in request_kwargs, so callers can still override it. ConnectTimeout and ReadTimeout exceptions are already handled as TRANSIENT_EXCEPTIONS in rate_limiting.py and will be retried automatically with exponential backoff. Root cause: observed in source-intercom where a sync hung permanently after an Intercom API 500 error. The retry succeeded but a subsequent request hung indefinitely because no timeout was configured. Co-Authored-By: alfredo.garcia@airbyte.io --- airbyte_cdk/sources/streams/http/http_client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index 3a0a62739..7530f29e9 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -85,6 +85,8 @@ def monkey_patched_get_item(self, key): # type: ignore # this interface is a co class HttpClient: _DEFAULT_MAX_RETRY: int = 5 _DEFAULT_MAX_TIME: int = 60 * 10 + _DEFAULT_CONNECT_TIMEOUT: int = 30 + _DEFAULT_READ_TIMEOUT: int = 300 _ACTIONS_TO_RETRY_ON = { ResponseAction.RETRY, ResponseAction.RATE_LIMITED, @@ -588,6 +590,9 @@ def send_request( ) request_kwargs = {**request_kwargs, **env_settings} + if "timeout" not in request_kwargs: + request_kwargs["timeout"] = (self._DEFAULT_CONNECT_TIMEOUT, self._DEFAULT_READ_TIMEOUT) + response: requests.Response = self._send_with_retry( request=request, request_kwargs=request_kwargs, From 34cdcb1f340560c86900b62bcd16b12b07513e85 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 21:00:30 +0000 Subject: [PATCH 2/3] fix: address review comments - fix type safety and add unit tests - Use Dict[str, Any] (mutable) instead of mutating Mapping[str, Any] for request_kwargs to satisfy mypy type checking - Add test_send_request_applies_default_timeout_when_not_provided - Add test_send_request_respects_explicit_timeout Co-Authored-By: alfredo.garcia@airbyte.io --- .../sources/streams/http/http_client.py | 8 ++-- .../sources/streams/http/test_http_client.py | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index 7530f29e9..2857378c5 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -588,14 +588,14 @@ def send_request( verify=request_kwargs.get("verify"), cert=request_kwargs.get("cert"), ) - request_kwargs = {**request_kwargs, **env_settings} + mutable_request_kwargs: Dict[str, Any] = {**request_kwargs, **env_settings} - if "timeout" not in request_kwargs: - request_kwargs["timeout"] = (self._DEFAULT_CONNECT_TIMEOUT, self._DEFAULT_READ_TIMEOUT) + if "timeout" not in mutable_request_kwargs: + mutable_request_kwargs["timeout"] = (self._DEFAULT_CONNECT_TIMEOUT, self._DEFAULT_READ_TIMEOUT) response: requests.Response = self._send_with_retry( request=request, - request_kwargs=request_kwargs, + request_kwargs=mutable_request_kwargs, log_formatter=log_formatter, exit_on_rate_limit=exit_on_rate_limit, ) diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index ea245c2fb..a12d54376 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -1059,3 +1059,43 @@ def update_response(*args, **kwargs): assert mock_authenticator.access_token == "new_refreshed_token" assert returned_response == valid_response assert call_count == 2 + + +def test_send_request_applies_default_timeout_when_not_provided(mocker): + http_client = test_http_client() + mocked_response = MagicMock(spec=requests.Response) + mocked_response.status_code = 200 + mocked_response.headers = {} + mock_send = mocker.patch.object(requests.Session, "send", return_value=mocked_response) + + http_client.send_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={} + ) + + assert mock_send.call_count == 1 + call_kwargs = mock_send.call_args + # The timeout should be passed as part of the keyword arguments to session.send() + # session.send(request, **request_kwargs) unpacks request_kwargs, so timeout appears as a kwarg + assert call_kwargs.kwargs.get("timeout") == (HttpClient._DEFAULT_CONNECT_TIMEOUT, HttpClient._DEFAULT_READ_TIMEOUT) or \ + call_kwargs[1].get("timeout") == (HttpClient._DEFAULT_CONNECT_TIMEOUT, HttpClient._DEFAULT_READ_TIMEOUT) + + +def test_send_request_respects_explicit_timeout(mocker): + http_client = test_http_client() + mocked_response = MagicMock(spec=requests.Response) + mocked_response.status_code = 200 + mocked_response.headers = {} + mock_send = mocker.patch.object(requests.Session, "send", return_value=mocked_response) + + custom_timeout = (10, 60) + http_client.send_request( + http_method="get", + url="https://test_base_url.com/v1/endpoint", + request_kwargs={"timeout": custom_timeout}, + ) + + assert mock_send.call_count == 1 + call_kwargs = mock_send.call_args + # The explicit timeout should be preserved, not overridden by the default + assert call_kwargs.kwargs.get("timeout") == custom_timeout or \ + call_kwargs[1].get("timeout") == custom_timeout From d2233c5a7608cabab8584b5ffbed0995ba45b9da Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 21:01:33 +0000 Subject: [PATCH 3/3] style: apply ruff format to new code Co-Authored-By: alfredo.garcia@airbyte.io --- airbyte_cdk/sources/streams/http/http_client.py | 5 ++++- .../sources/streams/http/test_http_client.py | 15 +++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index 2857378c5..a2dd67c46 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -591,7 +591,10 @@ def send_request( mutable_request_kwargs: Dict[str, Any] = {**request_kwargs, **env_settings} if "timeout" not in mutable_request_kwargs: - mutable_request_kwargs["timeout"] = (self._DEFAULT_CONNECT_TIMEOUT, self._DEFAULT_READ_TIMEOUT) + mutable_request_kwargs["timeout"] = ( + self._DEFAULT_CONNECT_TIMEOUT, + self._DEFAULT_READ_TIMEOUT, + ) response: requests.Response = self._send_with_retry( request=request, diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index a12d54376..ee00c9a03 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -1076,8 +1076,13 @@ def test_send_request_applies_default_timeout_when_not_provided(mocker): call_kwargs = mock_send.call_args # The timeout should be passed as part of the keyword arguments to session.send() # session.send(request, **request_kwargs) unpacks request_kwargs, so timeout appears as a kwarg - assert call_kwargs.kwargs.get("timeout") == (HttpClient._DEFAULT_CONNECT_TIMEOUT, HttpClient._DEFAULT_READ_TIMEOUT) or \ - call_kwargs[1].get("timeout") == (HttpClient._DEFAULT_CONNECT_TIMEOUT, HttpClient._DEFAULT_READ_TIMEOUT) + assert call_kwargs.kwargs.get("timeout") == ( + HttpClient._DEFAULT_CONNECT_TIMEOUT, + HttpClient._DEFAULT_READ_TIMEOUT, + ) or call_kwargs[1].get("timeout") == ( + HttpClient._DEFAULT_CONNECT_TIMEOUT, + HttpClient._DEFAULT_READ_TIMEOUT, + ) def test_send_request_respects_explicit_timeout(mocker): @@ -1097,5 +1102,7 @@ def test_send_request_respects_explicit_timeout(mocker): assert mock_send.call_count == 1 call_kwargs = mock_send.call_args # The explicit timeout should be preserved, not overridden by the default - assert call_kwargs.kwargs.get("timeout") == custom_timeout or \ - call_kwargs[1].get("timeout") == custom_timeout + assert ( + call_kwargs.kwargs.get("timeout") == custom_timeout + or call_kwargs[1].get("timeout") == custom_timeout + )