AVRO-4232: [Python] Modernize the Python tooling

.github/workflows/test-lang-py.yml

@@ -13,16 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: 'Test Python'
+name: "Test Python"
 on:
   workflow_dispatch:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
     paths:
-    - .github/workflows/test-lang-py.yml
-    - lang/py/**
+      - .github/workflows/test-lang-py.yml
+      - lang/py/**
 
 defaults:
   run:
@@ -38,16 +38,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: 
+        os:
           - ubuntu-latest
           - ubuntu-24.04-arm
         python:
-          - '3.13'
-          - '3.12'
-          - '3.11'
-          - '3.10'
-          - 'pypy-3.11'
-          - 'pypy-3.10'
+          - "3.14"
+          - "3.13"
+          - "3.12"
+          - "3.11"
+          - "3.10"
+          - "pypy-3.11"
+          - "pypy-3.10"
 
     steps:
       - uses: actions/checkout@v6
@@ -57,6 +58,9 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+
       - name: Apt Install Compression Libs
         run: |
           sudo apt-get update && \
@@ -67,33 +71,30 @@ jobs:
                                                             libzstd-dev
 
       - name: Install Dependencies
-        run: |
-          python3 -m pip install --upgrade pip setuptools tox
+        run: uv sync
 
       - name: Lint
         if: ${{ matrix.python == '3.10' }}
-        run: python3 -m tox -e lint
+        run: uv run ./build.sh lint
 
       - name: Typechecks
         if: ${{ matrix.python == '3.10' }}
-        run: python3 -m tox -e typechecks
+        run: uv run ./build.sh typechecks
 
       - name: Test
-        run: python3 -m tox -e py
+        run: uv run ./build.sh test
 
   interop:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: 
+        os:
           - ubuntu-latest
           - ubuntu-24.04-arm
         python:
-          - '3.11'
-          - '3.10'
-          - '3.9'
-          - 'pypy-3.10'
+          - "3.13"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v6
@@ -103,6 +104,9 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+
       - name: Apt Install Compression Libs
         run: |
           sudo apt-get update && \
@@ -113,9 +117,7 @@ jobs:
                                                             libzstd-dev
 
       - name: Install Dependencies
-        run: |
-          python3 -m pip install --upgrade pip setuptools tox
-          python3 -m pip install python-snappy zstandard
+        run: uv sync
 
       - name: Cache Local Maven Repository
         uses: actions/cache@v5
@@ -125,17 +127,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - name: 'Setup Temurin JDK 8, 11, 17 & 21'
+      - name: "Setup Temurin JDK 8, 11, 17 & 21"
         uses: actions/setup-java@v5
         with:
-          distribution: 'temurin'
+          distribution: "temurin"
           java-version: |
             8
             11
             17
             21
 
-      - name: 'Setup Maven 3.9.11'
+      - name: "Setup Maven 3.9.11"
         uses: stCarolas/setup-maven@d6af6abeda15e98926a57b5aa970a96bb37f97d1 # v5
         with:
           maven-version: 3.9.11
@@ -153,7 +155,7 @@ jobs:
         run: mvn -B -P interop-data-generate generate-resources
 
       - name: Generate Interop Data
-        run: ./build.sh interop-data-generate
+        run: uv run ./build.sh interop-data-generate
 
       - name: Run Interop Tests
-        run: ./build.sh interop-data-test
+        run: uv run ./build.sh interop-data-test

lang/py/avro/ipc.py

@@ -322,7 +322,7 @@ def respond(self, call_request):
             else:
                 writers_schema = local_message.errors
                 self.write_error(writers_schema, error, buffer_encoder)
-        except schema.AvroException as e:
+        except avro.schema.AvroException as e:
             error = avro.errors.AvroRemoteException(str(e))
             buffer_encoder = avro.io.BinaryEncoder(io.BytesIO())
             META_WRITER.write(response_metadata, buffer_encoder)

lang/py/avro/test/sample_http_server.py

@@ -32,11 +32,18 @@
             {
                 "name": "Message",
                 "type": "record",
-                "fields": [{"name": "to", "type": "string"}, {"name": "from", "type": "string"}, {"name": "body", "type": "string"}],
+                "fields": [
+                    {"name": "to", "type": "string"},
+                    {"name": "from", "type": "string"},
+                    {"name": "body", "type": "string"},
+                ],
             }
         ],
         "messages": {
-            "send": {"request": [{"name": "message", "type": "Message"}], "response": "string"},
+            "send": {
+                "request": [{"name": "message", "type": "Message"}],
+                "response": "string",
+            },
             "replay": {"request": [], "response": "string"},
         },
     }
@@ -71,7 +78,7 @@ def do_POST(self) -> None:
 
 
 def main():
-    mail_server = http_server.HTTPServer(SERVER_ADDRESS, MailHandler)
+    mail_server = http.server.HTTPServer(SERVER_ADDRESS, MailHandler)
     mail_server.allow_reuse_address = True
     mail_server.serve_forever()
 

lang/py/avro/test/test_io.py

@@ -179,11 +179,25 @@ class DefaultValueTestCaseType(TypedDict):
             {"value": {"car": {"value": "head"}, "cdr": {"value": None}}},
         ),
         (
-            {"type": "record", "name": "record", "fields": [{"name": "value", "type": "int"}, {"name": "next", "type": ["null", "record"]}]},
+            {
+                "type": "record",
+                "name": "record",
+                "fields": [
+                    {"name": "value", "type": "int"},
+                    {"name": "next", "type": ["null", "record"]},
+                ],
+            },
             {"value": 0, "next": {"value": 1, "next": None}},
         ),
         (
-            {"type": "record", "name": "ns.long", "fields": [{"name": "value", "type": "int"}, {"name": "next", "type": ["null", "ns.long"]}]},
+            {
+                "type": "record",
+                "name": "ns.long",
+                "fields": [
+                    {"name": "value", "type": "int"},
+                    {"name": "next", "type": ["null", "ns.long"]},
+                ],
+            },
             {"value": 0, "next": {"value": 1, "next": None}},
         ),
         # Optional logical types.
@@ -278,7 +292,11 @@ def write_datum(datum: object, writers_schema: avro.schema.Schema) -> Tuple[io.B
     return writer, encoder, datum_writer
 
 
-def read_datum(buffer: io.BytesIO, writers_schema: avro.schema.Schema, readers_schema: Optional[avro.schema.Schema] = None) -> object:
+def read_datum(
+    buffer: io.BytesIO,
+    writers_schema: avro.schema.Schema,
+    readers_schema: Optional[avro.schema.Schema] = None,
+) -> object:
     reader = io.BytesIO(buffer.getvalue())
     decoder = avro.io.BinaryDecoder(reader)
     datum_reader = avro.io.DatumReader(writers_schema, readers_schema)
@@ -302,7 +320,7 @@ def io_valid(self) -> None:
         """
         In these cases, the provided data should be valid with the given schema.
         """
-        with warnings.catch_warnings(record=True) as actual_warnings:
+        with warnings.catch_warnings(record=True) as _actual_warnings:
             self.assertTrue(
                 avro.io.validate(self.test_schema, self.test_datum),
                 f"{self.test_datum} did not validate in the schema {self.test_schema}",
@@ -326,7 +344,7 @@ def io_round_trip(self) -> None:
         """
         A datum should be the same after being encoded and then decoded.
         """
-        with warnings.catch_warnings(record=True) as actual_warnings:
+        with warnings.catch_warnings(record=True) as _actual_warnings:
             writer, encoder, datum_writer = write_datum(self.test_datum, self.test_schema)
             round_trip_datum = read_datum(writer, self.test_schema)
             expected: object
@@ -367,7 +385,7 @@ def __init__(self, skip: bool, test_type: str, test_datum: object, test_hex: byt
         warnings.simplefilter("always")
 
     def check_binary_encoding(self) -> None:
-        with warnings.catch_warnings(record=True) as actual_warnings:
+        with warnings.catch_warnings(record=True) as _actual_warnings:
             writer, encoder, datum_writer = write_datum(self.test_datum, self.writers_schema)
             writer.seek(0)
             hex_val = avro_hexlify(writer)
@@ -379,7 +397,7 @@ def check_binary_encoding(self) -> None:
 
     def check_skip_encoding(self) -> None:
         VALUE_TO_READ = 6253
-        with warnings.catch_warnings(record=True) as actual_warnings:
+        with warnings.catch_warnings(record=True) as _actual_warnings:
             # write the value to skip and a known value
             writer, encoder, datum_writer = write_datum(self.test_datum, self.writers_schema)
             datum_writer.write(VALUE_TO_READ, encoder)
@@ -418,7 +436,7 @@ def check_schema_promotion(self) -> None:
         # note that checking writers_schema.type in read_data
         # allows us to handle promotion correctly
         DATUM_TO_WRITE = 219
-        with warnings.catch_warnings(record=True) as actual_warnings:
+        with warnings.catch_warnings(record=True) as _actual_warnings:
             writer, enc, dw = write_datum(DATUM_TO_WRITE, self.writers_schema)
             datum_read = read_datum(writer, self.writers_schema, self.readers_schema)
             self.assertEqual(
@@ -429,7 +447,11 @@ def check_schema_promotion(self) -> None:
 
 
 class DefaultValueTestCase(unittest.TestCase):
-    def __init__(self, field_type: Collection[str], default: Union[Dict[str, int], List[int], None, float, str]) -> None:
+    def __init__(
+        self,
+        field_type: Collection[str],
+        default: Union[Dict[str, int], List[int], None, float, str],
+    ) -> None:
         """Ignore the normal signature for unittest.TestCase because we are generating
         many test cases from this one class. This is safe as long as the autoloader
         ignores this class. The autoloader will ignore this class as long as it has
@@ -443,7 +465,7 @@ def __init__(self, field_type: Collection[str], default: Union[Dict[str, int], L
 
     def check_default_value(self) -> None:
         datum_read: DefaultValueTestCaseType
-        with warnings.catch_warnings(record=True) as actual_warnings:
+        with warnings.catch_warnings(record=True) as _actual_warnings:
             datum_to_read = cast(DefaultValueTestCaseType, {"H": self.default})
             readers_schema = avro.schema.parse(
                 json.dumps(
@@ -461,7 +483,10 @@ def check_default_value(self) -> None:
                 )
             )
             writer, _, _ = write_datum(LONG_RECORD_DATUM, LONG_RECORD_SCHEMA)
-            datum_read_ = cast(DefaultValueTestCaseType, read_datum(writer, LONG_RECORD_SCHEMA, readers_schema))
+            datum_read_ = cast(
+                DefaultValueTestCaseType,
+                read_datum(writer, LONG_RECORD_SCHEMA, readers_schema),
+            )
             datum_read = {"H": cast(bytes, datum_read_["H"]).decode()} if isinstance(datum_read_["H"], bytes) else datum_read_
             self.assertEqual(datum_to_read, datum_read)
 
@@ -505,7 +530,11 @@ def test_deserialization_fails(self) -> None:
             enc_bytes = writer_bio.getvalue()
         reader = avro.io.DatumReader(reader_schema)
         with io.BytesIO(enc_bytes) as reader_bio:
-            self.assertRaises(avro.errors.InvalidAvroBinaryEncoding, reader.read, avro.io.BinaryDecoder(reader_bio))
+            self.assertRaises(
+                avro.errors.InvalidAvroBinaryEncoding,
+                reader.read,
+                avro.io.BinaryDecoder(reader_bio),
+            )
 
         incompatibleUserRecord = {"name": -10, "age": 21, "location": "Woodford"}
         with io.BytesIO() as writer_bio:
@@ -514,7 +543,11 @@ def test_deserialization_fails(self) -> None:
             enc_bytes = writer_bio.getvalue()
         reader = avro.io.DatumReader(reader_schema)
         with io.BytesIO(enc_bytes) as reader_bio:
-            self.assertRaises(avro.errors.InvalidAvroBinaryEncoding, reader.read, avro.io.BinaryDecoder(reader_bio))
+            self.assertRaises(
+                avro.errors.InvalidAvroBinaryEncoding,
+                reader.read,
+                avro.io.BinaryDecoder(reader_bio),
+            )
 
 
 class TestMisc(unittest.TestCase):
@@ -648,18 +681,37 @@ def test_type_exception_int(self) -> None:
         assert str(exc.exception) == 'The datum "Bad" provided for "F" is not an example of the schema "int"'
 
     def test_type_exception_long(self) -> None:
-        writers_schema = avro.schema.parse(json.dumps({"type": "record", "name": "Test", "fields": [{"name": "foo", "type": "long"}]}))
+        writers_schema = avro.schema.parse(
+            json.dumps(
+                {
+                    "type": "record",
+                    "name": "Test",
+                    "fields": [{"name": "foo", "type": "long"}],
+                }
+            )
+        )
         datum_to_write = {"foo": 5.0}
 
         with self.assertRaises(avro.errors.AvroTypeException) as exc:
             write_datum(datum_to_write, writers_schema)
         assert str(exc.exception) == 'The datum "5.0" provided for "foo" is not an example of the schema "long"'
 
     def test_type_exception_record(self) -> None:
-        writers_schema = avro.schema.parse(json.dumps({"type": "record", "name": "Test", "fields": [{"name": "foo", "type": "long"}]}))
+        writers_schema = avro.schema.parse(
+            json.dumps(
+                {
+                    "type": "record",
+                    "name": "Test",
+                    "fields": [{"name": "foo", "type": "long"}],
+                }
+            )
+        )
         datum_to_write = ("foo", 5.0)
 
-        with self.assertRaisesRegex(avro.errors.AvroTypeException, r"The datum \".*\" provided for \".*\" is not an example of the schema [\s\S]*"):
+        with self.assertRaisesRegex(
+            avro.errors.AvroTypeException,
+            r"The datum \".*\" provided for \".*\" is not an example of the schema [\s\S]*",
+        ):
             write_datum(datum_to_write, writers_schema)