Decode non-ASCII header values with iso-8859-1.

Per RFC 9110, section 5.5:

> Historically, HTTP allowed field content with text
> in the ISO-8859-1 charset

This is as reversible and less surprising than surrogate escaping
(which was never documented in websockets and is quite uncommon).

Author: aaugustin

src/websockets/http11.py

@@ -149,7 +149,10 @@ def parse(
             )
         if method != b"GET":
             raise ValueError(f"unsupported HTTP method; expected GET; got {d(method)}")
-        path = raw_path.decode("ascii", "surrogateescape")
+
+        # RFC 9110 defers the definition of URIs to RFC 3986, which allows only
+        # a subset of ASCII. Non-ASCII IRIs must be UTF-8 then percent-encoded.
+        path = raw_path.decode("ascii")
 
         headers = yield from parse_headers(read_line)
 
@@ -272,7 +275,10 @@ def parse(
             )
         if not _value_re.fullmatch(raw_reason):
             raise ValueError(f"invalid HTTP reason phrase: {d(raw_reason)}")
-        reason = raw_reason.decode("ascii", "surrogateescape")
+
+        # RFC 2616 implies ISO-8859-1. It's easy to reverse and cannot crash.
+        # Non-ASCII never worked reliably and the reason isn't useful anyway.
+        reason = raw_reason.decode("iso-8859-1")
 
         headers = yield from parse_headers(read_line)
 
@@ -368,7 +374,10 @@ def parse_headers(
             raise ValueError(f"invalid HTTP header value: {d(raw_value)}")
 
         name = raw_name.decode("ascii")  # guaranteed to be ASCII at this point
-        value = raw_value.decode("ascii", "surrogateescape")
+        # Headers should be ASCII. Section 5.5 of RFC 9110 says: "Historically,
+        # HTTP allowed field content with text in the ISO-8859-1 charset."
+        # It's easy to reverse and cannot crash, making it a decent choice.
+        value = raw_value.decode("iso-8859-1")
 
         # Since we just validated raw_value, we don't need to revalidate it.
         headers.set_insecure(name, value)

tests/test_http11.py

@@ -397,30 +397,38 @@ def setUp(self):
         self.reader = StreamReader()
 
     def parse_headers(self):
-        return parse_headers(self.reader.read_line)
+        try:
+            next(parse_headers(self.reader.read_line))
+        except StopIteration as exc:
+            return exc.value
+
+    def test_parse_iso_8859_1(self):
+        self.reader.feed_data(b"X-Drink: caf\xe9\r\n\r\n")
+        headers = self.parse_headers()
+        self.assertEqual(headers["X-Drink"], "café")
 
     def test_parse_invalid_name(self):
         self.reader.feed_data(b"foo bar: baz qux\r\n\r\n")
         with self.assertRaises(ValueError):
-            next(self.parse_headers())
+            self.parse_headers()
 
     def test_parse_invalid_value(self):
         self.reader.feed_data(b"foo: \x00\x00\x0f\r\n\r\n")
         with self.assertRaises(ValueError):
-            next(self.parse_headers())
+            self.parse_headers()
 
     def test_parse_too_long_value(self):
         self.reader.feed_data(b"foo: bar\r\n" * 129 + b"\r\n")
         with self.assertRaises(SecurityError):
-            next(self.parse_headers())
+            self.parse_headers()
 
     def test_parse_too_long_line(self):
         # Header line contains 5 + 8186 + 2 = 8193 bytes.
         self.reader.feed_data(b"foo: " + b"a" * 8186 + b"\r\n\r\n")
         with self.assertRaises(SecurityError):
-            next(self.parse_headers())
+            self.parse_headers()
 
     def test_parse_invalid_line_ending(self):
         self.reader.feed_data(b"foo: bar\n\n")
         with self.assertRaises(EOFError):
-            next(self.parse_headers())
+            self.parse_headers()