From 648473cf640503fa145faa9d477d1789ed619efc Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Mon, 9 Mar 2026 12:04:10 -0700 Subject: [PATCH 1/2] Add test coverage for basic protocol types --- test/protocol/test_compact.py | 8 +++ test/protocol/test_schema.py | 16 +++++ test/protocol/test_types.py | 107 ++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 test/protocol/test_schema.py create mode 100644 test/protocol/test_types.py diff --git a/test/protocol/test_compact.py b/test/protocol/test_compact.py index c5940aa70..a050ce1e3 100644 --- a/test/protocol/test_compact.py +++ b/test/protocol/test_compact.py @@ -17,6 +17,14 @@ def test_compact_data_structs(): encoded = cs.encode("foobarbaz") assert cs.decode(io.BytesIO(encoded)) == "foobarbaz" + # Test custom encoding + cs_utf16 = CompactString('utf-16') + val = "好" + encoded = cs_utf16.encode(val) + assert len(encoded) == 5 + decoded = cs_utf16.decode(io.BytesIO(encoded)) + assert decoded == val + arr = CompactArray(CompactString()) assert arr.encode(None) == b'\x00' assert arr.decode(io.BytesIO(b'\x00')) is None diff --git a/test/protocol/test_schema.py b/test/protocol/test_schema.py new file mode 100644 index 000000000..37e7711d8 --- /dev/null +++ b/test/protocol/test_schema.py @@ -0,0 +1,16 @@ +import io + +import pytest + +from kafka.protocol.types import Schema, Int32, String + + +def test_schema_type(): + schema = Schema(('f1', Int32), ('f2', String())) + val = (123, "bar") + encoded = schema.encode(val) + assert encoded == b'\x00\x00\x00\x7b\x00\x03bar' + assert schema.decode(io.BytesIO(encoded)) == val + + with pytest.raises(ValueError): + schema.encode((123,)) diff --git a/test/protocol/test_types.py b/test/protocol/test_types.py new file mode 100644 index 000000000..8f8e35508 --- /dev/null +++ b/test/protocol/test_types.py @@ -0,0 +1,107 @@ +import io +import uuid +import struct + +import pytest + +from kafka.protocol.types import ( + Int8, Int16, Int32, Int64, Float64, Boolean, UUID, + String, Bytes, Array +) + + +@pytest.mark.parametrize("cls, value, expected", [ + (Int8, 0, b'\x00'), + (Int8, 127, b'\x7f'), + (Int8, -128, b'\x80'), + (Int16, 0, b'\x00\x00'), + (Int16, 32767, b'\x7f\xff'), + (Int16, -32768, b'\x80\x00'), + (Int32, 0, b'\x00\x00\x00\x00'), + (Int32, 2147483647, b'\x7f\xff\xff\xff'), + (Int32, -2147483648, b'\x80\x00\x00\x00'), + (Int64, 0, b'\x00\x00\x00\x00\x00\x00\x00\x00'), + (Int64, 9223372036854775807, b'\x7f\xff\xff\xff\xff\xff\xff\xff'), + (Int64, -9223372036854775808, b'\x80\x00\x00\x00\x00\x00\x00\x00'), + (Float64, 0.0, b'\x00\x00\x00\x00\x00\x00\x00\x00'), + (Float64, 1.0, b'\x3f\xf0\x00\x00\x00\x00\x00\x00'), + (Boolean, True, b'\x01'), + (Boolean, False, b'\x00'), +]) +def test_primitive_types(cls, value, expected): + encoded = cls.encode(value) + assert encoded == expected + decoded = cls.decode(io.BytesIO(encoded)) + assert decoded == value + + +def test_uuid_type(): + val = uuid.uuid4() + encoded = UUID.encode(val) + assert len(encoded) == 16 + assert encoded == val.bytes + decoded = UUID.decode(io.BytesIO(encoded)) + assert decoded == val + + # Test with string + val_str = str(val) + encoded = UUID.encode(val_str) + assert encoded == val.bytes + + +def test_string_type(): + s = String() + assert s.encode(None) == b'\xff\xff' + assert s.decode(io.BytesIO(b'\xff\xff')) is None + + val = "foo" + encoded = s.encode(val) + assert encoded == b'\x00\x03foo' + assert s.decode(io.BytesIO(encoded)) == val + + # Test custom encoding + s_utf16 = String('utf-16') + val = "好" + encoded = s_utf16.encode(val) + assert len(encoded) == 6 + decoded = s_utf16.decode(io.BytesIO(encoded)) + assert decoded == val + + +def test_bytes_type(): + assert Bytes.encode(None) == b'\xff\xff\xff\xff' + assert Bytes.decode(io.BytesIO(b'\xff\xff\xff\xff')) is None + + val = b"foo" + encoded = Bytes.encode(val) + assert encoded == b'\x00\x00\x00\x03foo' + assert Bytes.decode(io.BytesIO(encoded)) == val + + +def test_array_type(): + arr = Array(Int32) + assert arr.encode(None) == b'\xff\xff\xff\xff' + assert arr.decode(io.BytesIO(b'\xff\xff\xff\xff')) is None + + val = [1, 2, 3] + encoded = arr.encode(val) + assert encoded == b'\x00\x00\x00\x03\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03' + assert arr.decode(io.BytesIO(encoded)) == val + + # Array of Schema + arr_schema = Array(('f1', Int32), ('f2', Int32)) + val = [(1, 10), (2, 20)] + encoded = arr_schema.encode(val) + assert arr_schema.decode(io.BytesIO(encoded)) == val + + +def test_error_handling(): + with pytest.raises(ValueError): + Int8.encode(1000) # Too large + + with pytest.raises(ValueError): + Int8.decode(io.BytesIO(b'')) # Too short + + s = String() + with pytest.raises(ValueError): + s.decode(io.BytesIO(b'\x00\x05foo')) # length 5 but only 3 bytes From c25c8040e611515d6f0e49f339502a9067750af4 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Mon, 9 Mar 2026 11:16:34 -0700 Subject: [PATCH 2/2] Fix struct format error str in kafka.protocol.types --- kafka/protocol/types.py | 12 ++++++++++-- test/protocol/test_types.py | 7 +++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/kafka/protocol/types.py b/kafka/protocol/types.py index b0811c59b..1711b5def 100644 --- a/kafka/protocol/types.py +++ b/kafka/protocol/types.py @@ -9,9 +9,13 @@ def _pack(f, value): try: return f(value) except error as e: + try: + fmt = f.__self__.format + except AttributeError: + fmt = 'unknown' raise ValueError("Error encountered when attempting to convert value: " "{!r} to struct format: '{}', hit error: {}" - .format(value, f, e)) + .format(value, fmt, e)) def _unpack(f, data): @@ -19,9 +23,13 @@ def _unpack(f, data): (value,) = f(data) return value except error as e: + try: + fmt = f.__self__.format + except AttributeError: + fmt = 'unknown' raise ValueError("Error encountered when attempting to convert value: " "{!r} to struct format: '{}', hit error: {}" - .format(data, f, e)) + .format(data, fmt, e)) class Int8(AbstractType): diff --git a/test/protocol/test_types.py b/test/protocol/test_types.py index 8f8e35508..a296ffc36 100644 --- a/test/protocol/test_types.py +++ b/test/protocol/test_types.py @@ -96,10 +96,13 @@ def test_array_type(): def test_error_handling(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Error encountered when attempting to convert value: 1000 to struct format: '>b'"): Int8.encode(1000) # Too large - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Error encountered when attempting to convert value: None to struct format: '>h'"): + Int16.encode(None) + + with pytest.raises(ValueError, match="Error encountered when attempting to convert value: b'' to struct format: '>b'"): Int8.decode(io.BytesIO(b'')) # Too short s = String()