From ed6440ca49ef4907ab9d99ba7e329aab702b7173 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 12 Jan 2026 17:41:48 +0000 Subject: [PATCH] Add max_headers parameter (#11955) --- CHANGES/11955.feature.rst | 1 + aiohttp/_http_parser.pyx | 31 ++++++----- aiohttp/client.py | 9 ++++ aiohttp/client_proto.py | 2 + aiohttp/http_exceptions.py | 10 ++-- aiohttp/http_parser.py | 90 +++++++++++++++++-------------- aiohttp/web_protocol.py | 2 + docs/client_reference.rst | 17 ++++-- docs/web_reference.rst | 4 +- tests/test_client_functional.py | 85 +++++++++++++++++++++++------ tests/test_http_exceptions.py | 18 +++---- tests/test_http_parser.py | 96 +++++++++++++++++++++++++-------- 12 files changed, 253 insertions(+), 112 deletions(-) create mode 100644 CHANGES/11955.feature.rst diff --git a/CHANGES/11955.feature.rst b/CHANGES/11955.feature.rst new file mode 100644 index 00000000000..eaea1016e60 --- /dev/null +++ b/CHANGES/11955.feature.rst @@ -0,0 +1 @@ +Added ``max_headers`` parameter to limit the number of headers that should be read from a response -- by :user:`Dreamsorcerer`. diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 0a306a02e2b..f7c393ed42a 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -279,6 +279,7 @@ cdef class HttpParser: object _name bytes _raw_value bint _has_value + int _header_name_size object _protocol object _loop @@ -329,7 +330,7 @@ cdef class HttpParser: self, cparser.llhttp_type mode, object protocol, object loop, int limit, object timer=None, - size_t max_line_size=8190, size_t max_headers=32768, + size_t max_line_size=8190, size_t max_headers=128, size_t max_field_size=8190, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, bint auto_decompress=True, @@ -352,6 +353,7 @@ cdef class HttpParser: self._raw_name = EMPTY_BYTES self._raw_value = EMPTY_BYTES self._has_value = False + self._header_name_size = 0 self._max_line_size = max_line_size self._max_headers = max_headers @@ -383,11 +385,14 @@ cdef class HttpParser: value = self._raw_value.decode('utf-8', 'surrogateescape') self._headers.append((name, value)) + if len(self._headers) > self._max_headers: + raise BadHttpMessage("Too many headers received") if name is CONTENT_ENCODING: self._content_encoding = value self._has_value = False + self._header_name_size = 0 self._raw_headers.append((self._raw_name, self._raw_value)) self._raw_name = EMPTY_BYTES self._raw_value = EMPTY_BYTES @@ -574,7 +579,7 @@ cdef class HttpRequestParser(HttpParser): def __init__( self, protocol, loop, int limit, timer=None, - size_t max_line_size=8190, size_t max_headers=32768, + size_t max_line_size=8190, size_t max_headers=128, size_t max_field_size=8190, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, bint auto_decompress=True, @@ -638,7 +643,7 @@ cdef class HttpResponseParser(HttpParser): def __init__( self, protocol, loop, int limit, timer=None, - size_t max_line_size=8190, size_t max_headers=32768, + size_t max_line_size=8190, size_t max_headers=128, size_t max_field_size=8190, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, bint auto_decompress=True @@ -677,8 +682,8 @@ cdef int cb_on_url(cparser.llhttp_t* parser, cdef HttpParser pyparser = parser.data try: if length > pyparser._max_line_size: - raise LineTooLong( - 'Status line is too long', pyparser._max_line_size, length) + status = pyparser._buf + at[:length] + raise LineTooLong(status[:100] + b"...", pyparser._max_line_size) extend(pyparser._buf, at, length) except BaseException as ex: pyparser._last_error = ex @@ -690,11 +695,10 @@ cdef int cb_on_url(cparser.llhttp_t* parser, cdef int cb_on_status(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data - cdef str reason try: if length > pyparser._max_line_size: - raise LineTooLong( - 'Status line is too long', pyparser._max_line_size, length) + reason = pyparser._buf + at[:length] + raise LineTooLong(reason[:100] + b"...", pyparser._max_line_size) extend(pyparser._buf, at, length) except BaseException as ex: pyparser._last_error = ex @@ -711,8 +715,9 @@ cdef int cb_on_header_field(cparser.llhttp_t* parser, pyparser._on_status_complete() size = len(pyparser._raw_name) + length if size > pyparser._max_field_size: - raise LineTooLong( - 'Header name is too long', pyparser._max_field_size, size) + name = pyparser._raw_name + at[:length] + raise LineTooLong(name[:100] + b"...", pyparser._max_field_size) + pyparser._header_name_size = size pyparser._on_header_field(at, length) except BaseException as ex: pyparser._last_error = ex @@ -727,9 +732,9 @@ cdef int cb_on_header_value(cparser.llhttp_t* parser, cdef Py_ssize_t size try: size = len(pyparser._raw_value) + length - if size > pyparser._max_field_size: - raise LineTooLong( - 'Header value is too long', pyparser._max_field_size, size) + if pyparser._header_name_size + size > pyparser._max_field_size: + value = pyparser._raw_value + at[:length] + raise LineTooLong(value[:100] + b"...", pyparser._max_field_size) pyparser._on_header_value(at, length) except BaseException as ex: pyparser._last_error = ex diff --git a/aiohttp/client.py b/aiohttp/client.py index b7b5c8a7acb..7a5ef206ecd 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -194,6 +194,7 @@ class _RequestOptions(TypedDict, total=False): auto_decompress: bool | None max_line_size: int | None max_field_size: int | None + max_headers: int | None middlewares: Sequence[ClientMiddlewareType] | None @@ -283,6 +284,7 @@ class ClientSession: "_read_bufsize", "_max_line_size", "_max_field_size", + "_max_headers", "_resolve_charset", "_default_proxy", "_default_proxy_auth", @@ -317,6 +319,7 @@ def __init__( read_bufsize: int = 2**16, max_line_size: int = 8190, max_field_size: int = 8190, + max_headers: int = 128, fallback_charset_resolver: _CharsetResolver = lambda r, b: "utf-8", middlewares: Sequence[ClientMiddlewareType] = (), ssl_shutdown_timeout: _SENTINEL | None | float = sentinel, @@ -386,6 +389,7 @@ def __init__( self._read_bufsize = read_bufsize self._max_line_size = max_line_size self._max_field_size = max_field_size + self._max_headers = max_headers # Convert to list of tuples if headers: @@ -485,6 +489,7 @@ async def _request( auto_decompress: bool | None = None, max_line_size: int | None = None, max_field_size: int | None = None, + max_headers: int | None = None, middlewares: Sequence[ClientMiddlewareType] | None = None, ) -> ClientResponse: # NOTE: timeout clamps existing connect and read timeouts. We cannot @@ -571,6 +576,9 @@ async def _request( if max_field_size is None: max_field_size = self._max_field_size + if max_headers is None: + max_headers = self._max_headers + traces = [ Trace( self, @@ -710,6 +718,7 @@ async def _connect_and_send_request( timeout_ceil_threshold=self._connector._timeout_ceil_threshold, max_line_size=max_line_size, max_field_size=max_field_size, + max_headers=max_headers, ) try: resp = await req._send(conn) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 144cb42d52b..601b545c82a 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -230,6 +230,7 @@ def set_response_params( timeout_ceil_threshold: float = 5, max_line_size: int = 8190, max_field_size: int = 8190, + max_headers: int = 128, ) -> None: self._skip_payload = skip_payload @@ -248,6 +249,7 @@ def set_response_params( auto_decompress=auto_decompress, max_line_size=max_line_size, max_field_size=max_field_size, + max_headers=max_headers, ) if self._tail: diff --git a/aiohttp/http_exceptions.py b/aiohttp/http_exceptions.py index 91abed24308..cf3c05434c5 100644 --- a/aiohttp/http_exceptions.py +++ b/aiohttp/http_exceptions.py @@ -78,13 +78,9 @@ class DecompressSizeError(PayloadEncodingError): class LineTooLong(BadHttpMessage): - def __init__( - self, line: str, limit: str = "Unknown", actual_size: str = "Unknown" - ) -> None: - super().__init__( - f"Got more than {limit} bytes ({actual_size}) when reading {line}." - ) - self.args = (line, limit, actual_size) + def __init__(self, line: bytes, limit: int) -> None: + super().__init__(f"Got more than {limit} bytes when reading: {line!r}.") + self.args = (line, limit) class InvalidHeader(BadHttpMessage): diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 79c7f06734f..c5560b7a5ac 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -115,10 +115,7 @@ class ChunkState(IntEnum): class HeadersParser: - def __init__( - self, max_line_size: int = 8190, max_field_size: int = 8190, lax: bool = False - ) -> None: - self.max_line_size = max_line_size + def __init__(self, max_field_size: int = 8190, lax: bool = False) -> None: self.max_field_size = max_field_size self._lax = lax @@ -148,20 +145,10 @@ def parse_headers( raise InvalidHeader(line) bvalue = bvalue.lstrip(b" \t") - if len(bname) > self.max_field_size: - raise LineTooLong( - "request header name {}".format( - bname.decode("utf8", "backslashreplace") - ), - str(self.max_field_size), - str(len(bname)), - ) name = bname.decode("utf-8", "surrogateescape") if not TOKENRE.fullmatch(name): raise InvalidHeader(bname) - header_length = len(bvalue) - # next line lines_idx += 1 line = lines[lines_idx] @@ -171,16 +158,14 @@ def parse_headers( # Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding if continuation: + header_length = len(bvalue) bvalue_lst = [bvalue] while continuation: header_length += len(line) if header_length > self.max_field_size: + header_line = bname + b": " + b"".join(bvalue_lst) raise LineTooLong( - "request header field {}".format( - bname.decode("utf8", "backslashreplace") - ), - str(self.max_field_size), - str(header_length), + header_line[:100] + b"...", self.max_field_size ) bvalue_lst.append(line) @@ -194,15 +179,6 @@ def parse_headers( line = b"" break bvalue = b"".join(bvalue_lst) - else: - if header_length > self.max_field_size: - raise LineTooLong( - "request header field {}".format( - bname.decode("utf8", "backslashreplace") - ), - str(self.max_field_size), - str(header_length), - ) bvalue = bvalue.strip(b" \t") value = bvalue.decode("utf-8", "surrogateescape") @@ -233,6 +209,7 @@ def __init__( loop: asyncio.AbstractEventLoop, limit: int, max_line_size: int = 8190, + max_headers: int = 128, max_field_size: int = 8190, timer: BaseTimerContext | None = None, code: int | None = None, @@ -246,6 +223,7 @@ def __init__( self.loop = loop self.max_line_size = max_line_size self.max_field_size = max_field_size + self.max_headers = max_headers self.timer = timer self.code = code self.method = method @@ -260,7 +238,7 @@ def __init__( self._payload_parser: HttpPayloadParser | None = None self._auto_decompress = auto_decompress self._limit = limit - self._headers_parser = HeadersParser(max_line_size, max_field_size, self.lax) + self._headers_parser = HeadersParser(max_field_size, self.lax) @abc.abstractmethod def parse_message(self, lines: list[bytes]) -> _MsgT: ... @@ -301,6 +279,7 @@ def feed_data( data_len = len(data) start_pos = 0 loop = self.loop + max_line_length = self.max_line_size should_close = False while start_pos < data_len: @@ -321,11 +300,21 @@ def feed_data( line = data[start_pos:pos] if SEP == b"\n": # For lax response parsing line = line.rstrip(b"\r") + if len(line) > max_line_length: + raise LineTooLong(line[:100] + b"...", max_line_length) + self._lines.append(line) + # After processing the status/request line, everything is a header. + max_line_length = self.max_field_size + + if len(self._lines) > self.max_headers: + raise BadHttpMessage("Too many headers received") + start_pos = pos + len(SEP) # \r\n\r\n found if self._lines[-1] == EMPTY: + max_trailers = self.max_headers - len(self._lines) try: msg: _MsgT = self.parse_message(self._lines) finally: @@ -384,6 +373,9 @@ def get_content_length() -> int | None: auto_decompress=self._auto_decompress, lax=self.lax, headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, ) if not payload_parser.done: self._payload_parser = payload_parser @@ -403,6 +395,9 @@ def get_content_length() -> int | None: auto_decompress=self._auto_decompress, lax=self.lax, headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, ) elif not empty_body and length is None and self.read_until_eof: payload = StreamReader( @@ -422,6 +417,9 @@ def get_content_length() -> int | None: auto_decompress=self._auto_decompress, lax=self.lax, headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, ) if not payload_parser.done: self._payload_parser = payload_parser @@ -432,6 +430,8 @@ def get_content_length() -> int | None: should_close = msg.should_close else: self._tail = data[start_pos:] + if len(self._tail) > self.max_line_size: + raise LineTooLong(self._tail[:100] + b"...", self.max_line_size) data = EMPTY break @@ -567,11 +567,6 @@ def parse_message(self, lines: list[bytes]) -> RawRequestMessage: except ValueError: raise BadHttpMethod(line) from None - if len(path) > self.max_line_size: - raise LineTooLong( - "Status line is too long", str(self.max_line_size), str(len(path)) - ) - # method if not TOKENRE.fullmatch(method): raise BadHttpMethod(method) @@ -687,11 +682,6 @@ def parse_message(self, lines: list[bytes]) -> RawResponseMessage: status = status.strip() reason = "" - if len(reason) > self.max_line_size: - raise LineTooLong( - "Status line is too long", str(self.max_line_size), str(len(reason)) - ) - # version match = VERSRE.fullmatch(version) if match is None: @@ -756,6 +746,9 @@ def __init__( lax: bool = False, *, headers_parser: HeadersParser, + max_line_size: int = 8190, + max_field_size: int = 8190, + max_trailers: int = 128, ) -> None: self._length = 0 self._type = ParseState.PARSE_UNTIL_EOF @@ -765,6 +758,9 @@ def __init__( self._auto_decompress = auto_decompress self._lax = lax self._headers_parser = headers_parser + self._max_line_size = max_line_size + self._max_field_size = max_field_size + self._max_trailers = max_trailers self._trailer_lines: list[bytes] = [] self.done = False @@ -820,6 +816,15 @@ def feed_data( # Chunked transfer encoding parser elif self._type == ParseState.PARSE_CHUNKED: if self._chunk_tail: + # We should never have a tail if we're inside the payload body. + assert self._chunk != ChunkState.PARSE_CHUNKED_CHUNK + # We should check the length is sane. + max_line_length = self._max_line_size + if self._chunk == ChunkState.PARSE_TRAILERS: + max_line_length = self._max_field_size + if len(self._chunk_tail) > max_line_length: + raise LineTooLong(self._chunk_tail[:100] + b"...", max_line_length) + chunk = self._chunk_tail + chunk self._chunk_tail = b"" @@ -898,8 +903,15 @@ def feed_data( chunk = chunk[pos + len(SEP) :] if SEP == b"\n": # For lax response parsing line = line.rstrip(b"\r") + + if len(line) > self._max_field_size: + raise LineTooLong(line[:100] + b"...", self._max_field_size) + self._trailer_lines.append(line) + if len(self._trailer_lines) > self._max_trailers: + raise BadHttpMessage("Too many trailers received") + # \r\n\r\n found, end of stream if self._trailer_lines[-1] == b"": # Headers and trailers are defined the same way, diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index 7e576350ac0..bd39c48050d 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -196,6 +196,7 @@ def __init__( access_log: Logger | None = access_logger, access_log_format: str = AccessLogger.LOG_FORMAT, max_line_size: int = 8190, + max_headers: int = 128, max_field_size: int = 8190, lingering_time: float = 10.0, read_bufsize: int = 2**16, @@ -238,6 +239,7 @@ def __init__( read_bufsize, max_line_size=max_line_size, max_field_size=max_field_size, + max_headers=max_headers, payload_exception=RequestPayloadError, auto_decompress=auto_decompress, ) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index ab52fbfa5fb..d085bded527 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -57,6 +57,7 @@ The client session supports the context manager protocol for self closing. read_bufsize=2**16, \ max_line_size=8190, \ max_field_size=8190, \ + max_headers=128, \ fallback_charset_resolver=lambda r, b: "utf-8", \ ssl_shutdown_timeout=0) @@ -229,7 +230,9 @@ The client session supports the context manager protocol for self closing. :param int max_line_size: Maximum allowed size of lines in responses. - :param int max_field_size: Maximum allowed size of header fields in responses. + :param int max_field_size: Maximum allowed size of header name and value combined in responses. + + :param int max_headers: Maximum number of headers and trailers combined in responses. :param Callable[[ClientResponse,bytes],str] fallback_charset_resolver: A :term:`callable` that accepts a :class:`ClientResponse` and the @@ -409,7 +412,8 @@ The client session supports the context manager protocol for self closing. read_bufsize=None, \ auto_decompress=None, \ max_line_size=None, \ - max_field_size=None) + max_field_size=None, \ + max_headers=None) :async: :noindexentry: @@ -573,7 +577,9 @@ The client session supports the context manager protocol for self closing. :param int max_line_size: Maximum allowed size of lines in responses. - :param int max_field_size: Maximum allowed size of header fields in responses. + :param int max_field_size: Maximum allowed size of header name and value combined in responses. + + :param int max_headers: Maximum number of headers and trailers combined in responses. :return ClientResponse: a :class:`client response ` object. @@ -902,6 +908,7 @@ certification chaining. auto_decompress=None, \ max_line_size=None, \ max_field_size=None, \ + max_headers=None, \ version=aiohttp.HttpVersion11, \ connector=None) :async: @@ -1039,7 +1046,9 @@ certification chaining. :param int max_line_size: Maximum allowed size of lines in responses. - :param int max_field_size: Maximum allowed size of header fields in responses. + :param int max_field_size: Maximum allowed size of header name and value combined in responses. + + :param int max_headers: Maximum number of headers and trailers combined in responses. :param aiohttp.protocol.HttpVersion version: Request HTTP version, ``HTTP 1.1`` by default. (optional) diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 51a8dae189a..89615285dd9 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -2723,8 +2723,10 @@ application on specific TCP or Unix socket, e.g.:: :attr:`helpers.AccessLogger.LOG_FORMAT`. :param int max_line_size: Optional maximum header line size. Default: ``8190``. - :param int max_field_size: Optional maximum header field size. Default: + :param int max_field_size: Optional maximum header combined name and value size. Default: ``8190``. + :param int max_headers: Optional maximum number of headers and trailers combined. Default: + ``128``. :param float lingering_time: Maximum time during which the server reads and ignores additional data coming from the client when diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 916261a1d12..fec3d3c6c3e 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -4424,7 +4424,7 @@ async def handler(request: web.Request) -> web.Response: async def test_max_field_size_session_default(aiohttp_client: AiohttpClient) -> None: async def handler(request: web.Request) -> web.Response: - return web.Response(headers={"Custom": "x" * 8190}) + return web.Response(headers={"Custom": "x" * 8182}) app = web.Application() app.add_routes([web.get("/", handler)]) @@ -4432,7 +4432,7 @@ async def handler(request: web.Request) -> web.Response: client = await aiohttp_client(app) async with client.get("/") as resp: - assert resp.headers["Custom"] == "x" * 8190 + assert resp.headers["Custom"] == "x" * 8182 async def test_max_field_size_session_default_fail( @@ -4451,33 +4451,86 @@ async def handler(request: web.Request) -> web.Response: async def test_max_field_size_session_explicit(aiohttp_client: AiohttpClient) -> None: async def handler(request: web.Request) -> web.Response: - return web.Response(headers={"Custom": "x" * 8191}) + return web.Response(headers={"Custom": "x" * 8192}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app, max_field_size=8200) + + async with client.get("/") as resp: + assert resp.headers["Custom"] == "x" * 8192 + + +async def test_max_headers_session_default(aiohttp_client: AiohttpClient) -> None: + async def handler(request: web.Request) -> web.Response: + return web.Response(headers={f"Custom-{i}": "x" for i in range(120)}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with client.get("/") as resp: + assert resp.headers["Custom-119"] == "x" + + +async def test_max_headers_session_default_fail( + aiohttp_client: AiohttpClient, +) -> None: + async def handler(request: web.Request) -> web.Response: + return web.Response(headers={f"Custom-{i}": "x" for i in range(129)}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + with pytest.raises(aiohttp.ClientResponseError): + await client.get("/") + + +async def test_max_headers_session_explicit(aiohttp_client: AiohttpClient) -> None: + async def handler(request: web.Request) -> web.Response: + return web.Response(headers={f"Custom-{i}": "x" for i in range(130)}) app = web.Application() app.add_routes([web.get("/", handler)]) - client = await aiohttp_client(app, max_field_size=8191) + client = await aiohttp_client(app, max_headers=140) async with client.get("/") as resp: - assert resp.headers["Custom"] == "x" * 8191 + assert resp.headers["Custom-129"] == "x" + + +async def test_max_headers_request_explicit(aiohttp_client: AiohttpClient) -> None: + async def handler(request: web.Request) -> web.Response: + return web.Response(headers={f"Custom-{i}": "x" for i in range(130)}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with client.get("/", max_headers=140) as resp: + assert resp.headers["Custom-129"] == "x" async def test_max_field_size_request_explicit(aiohttp_client: AiohttpClient) -> None: async def handler(request: web.Request) -> web.Response: - return web.Response(headers={"Custom": "x" * 8191}) + return web.Response(headers={"Custom": "x" * 8192}) app = web.Application() app.add_routes([web.get("/", handler)]) client = await aiohttp_client(app) - async with client.get("/", max_field_size=8191) as resp: - assert resp.headers["Custom"] == "x" * 8191 + async with client.get("/", max_field_size=8200) as resp: + assert resp.headers["Custom"] == "x" * 8192 async def test_max_line_size_session_default(aiohttp_client: AiohttpClient) -> None: async def handler(request: web.Request) -> web.Response: - return web.Response(status=200, reason="x" * 8190) + return web.Response(status=200, reason="x" * 8177) app = web.Application() app.add_routes([web.get("/", handler)]) @@ -4485,7 +4538,7 @@ async def handler(request: web.Request) -> web.Response: client = await aiohttp_client(app) async with client.get("/") as resp: - assert resp.reason == "x" * 8190 + assert resp.reason == "x" * 8177 async def test_max_line_size_session_default_fail( @@ -4504,28 +4557,28 @@ async def handler(request: web.Request) -> web.Response: async def test_max_line_size_session_explicit(aiohttp_client: AiohttpClient) -> None: async def handler(request: web.Request) -> web.Response: - return web.Response(status=200, reason="x" * 8191) + return web.Response(status=200, reason="x" * 8197) app = web.Application() app.add_routes([web.get("/", handler)]) - client = await aiohttp_client(app, max_line_size=8191) + client = await aiohttp_client(app, max_line_size=8210) async with client.get("/") as resp: - assert resp.reason == "x" * 8191 + assert resp.reason == "x" * 8197 async def test_max_line_size_request_explicit(aiohttp_client: AiohttpClient) -> None: async def handler(request: web.Request) -> web.Response: - return web.Response(status=200, reason="x" * 8191) + return web.Response(status=200, reason="x" * 8197) app = web.Application() app.add_routes([web.get("/", handler)]) client = await aiohttp_client(app) - async with client.get("/", max_line_size=8191) as resp: - assert resp.reason == "x" * 8191 + async with client.get("/", max_line_size=8210) as resp: + assert resp.reason == "x" * 8197 async def test_rejected_upload( diff --git a/tests/test_http_exceptions.py b/tests/test_http_exceptions.py index 5aca60f8f91..74c7fb30505 100644 --- a/tests/test_http_exceptions.py +++ b/tests/test_http_exceptions.py @@ -77,32 +77,32 @@ def test_repr(self) -> None: class TestLineTooLong: def test_ctor(self) -> None: - err = http_exceptions.LineTooLong("spam", "10", "12") + err = http_exceptions.LineTooLong(b"spam", 10) assert err.code == 400 - assert err.message == "Got more than 10 bytes (12) when reading spam." + assert err.message == "Got more than 10 bytes when reading: b'spam'." assert err.headers is None def test_pickle(self) -> None: - err = http_exceptions.LineTooLong(line="spam", limit="10", actual_size="12") + err = http_exceptions.LineTooLong(line=b"spam", limit=10) err.foo = "bar" # type: ignore[attr-defined] for proto in range(pickle.HIGHEST_PROTOCOL + 1): pickled = pickle.dumps(err, proto) err2 = pickle.loads(pickled) assert err2.code == 400 - assert err2.message == ("Got more than 10 bytes (12) when reading spam.") + assert err2.message == ("Got more than 10 bytes when reading: b'spam'.") assert err2.headers is None assert err2.foo == "bar" def test_str(self) -> None: - err = http_exceptions.LineTooLong(line="spam", limit="10", actual_size="12") - expected = "400, message:\n Got more than 10 bytes (12) when reading spam." + err = http_exceptions.LineTooLong(line=b"spam", limit=10) + expected = "400, message:\n Got more than 10 bytes when reading: b'spam'." assert str(err) == expected def test_repr(self) -> None: - err = http_exceptions.LineTooLong(line="spam", limit="10", actual_size="12") + err = http_exceptions.LineTooLong(line=b"spam", limit=10) assert repr(err) == ( - "" + '" ) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 869799583e1..bfd84aae0d2 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -80,6 +80,7 @@ def parser( loop, 2**16, max_line_size=8190, + max_headers=128, max_field_size=8190, ) @@ -102,6 +103,7 @@ def response( loop, 2**16, max_line_size=8190, + max_headers=128, max_field_size=8190, read_until_eof=True, ) @@ -285,15 +287,6 @@ def test_whitespace_before_header(parser: HttpRequestParser) -> None: parser.feed_data(text) -def test_parse_headers_longline(parser: HttpRequestParser) -> None: - invalid_unicode_byte = b"\xd9" - header_name = b"Test" + invalid_unicode_byte + b"Header" + b"A" * 8192 - text = b"GET /test HTTP/1.1\r\n" + header_name + b": test\r\n" + b"\r\n" + b"\r\n" - with pytest.raises((http_exceptions.LineTooLong, http_exceptions.BadHttpMessage)): - # FIXME: `LineTooLong` doesn't seem to actually be happening - parser.feed_data(text) - - @pytest.fixture def xfail_c_parser_status(request: pytest.FixtureRequest) -> None: if isinstance(request.getfixturevalue("parser"), HttpRequestParserPy): @@ -728,13 +721,14 @@ def test_max_header_field_size(parser: HttpRequestParser, size: int) -> None: name = b"t" * size text = b"GET /test HTTP/1.1\r\n" + name + b":data\r\n\r\n" - match = f"400, message:\n Got more than 8190 bytes \\({size}\\) when reading" + match = "400, message:\n Got more than 8190 bytes when reading" with pytest.raises(http_exceptions.LineTooLong, match=match): - parser.feed_data(text) + for i in range(0, len(text), 5000): # pragma: no branch + parser.feed_data(text[i : i + 5000]) -def test_max_header_field_size_under_limit(parser: HttpRequestParser) -> None: - name = b"t" * 8190 +def test_max_header_size_under_limit(parser: HttpRequestParser) -> None: + name = b"t" * 8185 text = b"GET /test HTTP/1.1\r\n" + name + b":data\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) @@ -756,13 +750,67 @@ def test_max_header_value_size(parser: HttpRequestParser, size: int) -> None: name = b"t" * size text = b"GET /test HTTP/1.1\r\ndata:" + name + b"\r\n\r\n" - match = f"400, message:\n Got more than 8190 bytes \\({size}\\) when reading" + match = "400, message:\n Got more than 8190 bytes when reading" + with pytest.raises(http_exceptions.LineTooLong, match=match): + for i in range(0, len(text), 4000): # pragma: no branch + parser.feed_data(text[i : i + 4000]) + + +def test_max_header_combined_size(parser: HttpRequestParser) -> None: + k = b"t" * 4100 + text = b"GET /test HTTP/1.1\r\n" + k + b":" + k + b"\r\n\r\n" + + match = "400, message:\n Got more than 8190 bytes when reading" with pytest.raises(http_exceptions.LineTooLong, match=match): parser.feed_data(text) +@pytest.mark.parametrize("size", [40960, 8191]) +async def test_max_trailer_size(parser: HttpRequestParser, size: int) -> None: + value = b"t" * size + text = ( + b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" + + hex(4000)[2:].encode() + + b"\r\n" + + b"b" * 4000 + + b"\r\n0\r\ntest: " + + value + + b"\r\n\r\n" + ) + + match = "400, message:\n Got more than 8190 bytes when reading" + with pytest.raises(http_exceptions.LineTooLong, match=match): + payload = None + for i in range(0, len(text), 3000): # pragma: no branch + messages, upgrade, tail = parser.feed_data(text[i : i + 3000]) + if messages: + payload = messages[0][-1] + # Trailers are not seen until payload is read. + assert payload is not None + await payload.read() + + +@pytest.mark.parametrize("headers,trailers", ((129, 0), (0, 129), (64, 65))) +async def test_max_headers( + parser: HttpRequestParser, headers: int, trailers: int +) -> None: + text = ( + b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked" + + b"".join(b"\r\nHeader-%d: Value" % i for i in range(headers)) + + b"\r\n\r\n4\r\ntest\r\n0" + + b"".join(b"\r\nTrailer-%d: Value" % i for i in range(trailers)) + + b"\r\n\r\n" + ) + + match = "Too many (headers|trailers) received" + with pytest.raises(http_exceptions.BadHttpMessage, match=match): + messages, upgrade, tail = parser.feed_data(text) + # Trailers are not seen until payload is read. + await messages[0][-1].read() + + def test_max_header_value_size_under_limit(parser: HttpRequestParser) -> None: - value = b"A" * 8190 + value = b"A" * 8185 text = b"GET /test HTTP/1.1\r\ndata:" + value + b"\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) @@ -786,15 +834,16 @@ def test_max_header_value_size_continuation( name = b"T" * (size - 5) text = b"HTTP/1.1 200 Ok\r\ndata: test\r\n " + name + b"\r\n\r\n" - match = f"400, message:\n Got more than 8190 bytes \\({size}\\) when reading" + match = "400, message:\n Got more than 8190 bytes when reading" with pytest.raises(http_exceptions.LineTooLong, match=match): - response.feed_data(text) + for i in range(0, len(text), 9000): # pragma: no branch + response.feed_data(text[i : i + 9000]) def test_max_header_value_size_continuation_under_limit( response: HttpResponseParser, ) -> None: - value = b"A" * 8185 + value = b"A" * 8179 text = b"HTTP/1.1 200 Ok\r\ndata: test\r\n " + value + b"\r\n\r\n" messages, upgrade, tail = response.feed_data(text) @@ -1033,13 +1082,13 @@ def test_http_request_parser_bad_nonascii_uri(parser: HttpRequestParser) -> None @pytest.mark.parametrize("size", [40965, 8191]) def test_http_request_max_status_line(parser: HttpRequestParser, size: int) -> None: path = b"t" * (size - 5) - match = f"400, message:\n Got more than 8190 bytes \\({size}\\) when reading" + match = "400, message:\n Got more than 8190 bytes when reading" with pytest.raises(http_exceptions.LineTooLong, match=match): parser.feed_data(b"GET /path" + path + b" HTTP/1.1\r\n\r\n") def test_http_request_max_status_line_under_limit(parser: HttpRequestParser) -> None: - path = b"t" * (8190 - 5) + path = b"t" * 8172 messages, upgraded, tail = parser.feed_data( b"GET /path" + path + b" HTTP/1.1\r\n\r\n" ) @@ -1120,7 +1169,7 @@ def test_http_response_parser_bad_status_line_too_long( response: HttpResponseParser, size: int ) -> None: reason = b"t" * (size - 2) - match = f"400, message:\n Got more than 8190 bytes \\({size}\\) when reading" + match = "400, message:\n Got more than 8190 bytes when reading" with pytest.raises(http_exceptions.LineTooLong, match=match): response.feed_data(b"HTTP/1.1 200 Ok" + reason + b"\r\n\r\n") @@ -1128,7 +1177,7 @@ def test_http_response_parser_bad_status_line_too_long( def test_http_response_parser_status_line_under_limit( response: HttpResponseParser, ) -> None: - reason = b"O" * 8190 + reason = b"O" * 8177 messages, upgraded, tail = response.feed_data( b"HTTP/1.1 200 " + reason + b"\r\n\r\n" ) @@ -1662,6 +1711,7 @@ def test_parse_bad_method_for_c_parser_raises( loop, 2**16, max_line_size=8190, + max_headers=128, max_field_size=8190, ) @@ -1994,7 +2044,7 @@ async def test_streaming_decompress_large_payload( dbuf = DeflateBuffer(buf, "deflate") # Feed compressed data in chunks (simulating network streaming) - for i in range(0, len(compressed), chunk_size): + for i in range(0, len(compressed), chunk_size): # pragma: no branch chunk = compressed[i : i + chunk_size] dbuf.feed_data(chunk)