AVRO-4232: [Python] Modernize the Python tooling (#3658)

* AVRO-4232: [Python] Modernize the Python tooling

Use `uv` for management and `ruff` for linting and formatting

* Ignore uv.lock for ASFv2 licence check

* Add librt as a dev dependency

* Drop librt as dev-dep. It fails at CI with another error

* Do not test Python 3.9 since it is not supported

Test 3.14 instead

* Try with mypy<1.19

python/mypy#20454

* Use uv for the Java interop tests too

* Remove the flake8 settings. Now we use ruff

* Update uv.lock

* Set cwd to lang/py for `uv sync`. Do not use uv for C# scripts

* Update the usage. Wrap variables in quotes

* Move more entries from setup.cfg to pyproject.toml

* Remove typechecks dependency. It seems to be tox specific

* Do not shadow Python built-in `type`

* Use `--frozen` for `uv sync` in CI

* Update setup-uv Github action to v7

* Fix the package name for AvroException

* First declare the local variable and then assign it a value

* Use Python 3.10 for mypy

* Move "scripts" from setup.cfg to pyproject.toml

* Re-add the optional dependencies from setup.cfg to pyproject.toml

* Re-add the package-data from setup.cfg to pyproject.toml

* Use SHA for the external actions

Policy by Infra: https://infra.apache.org/github-actions-policy.html > External actions
Last allowed version: 7.1.6 (https://github.com/apache/infrastructure-actions/blob/2943971f47fadefa6d86af85c2c24c2cee262ee1/actions.yml#L25-L26)

Co-authored-by: Ryan Skraba <ryan@skraba.com>

* No need to call `build.sh` thru `uv run`

---------

Co-authored-by: Ryan Skraba <ryan@skraba.com>

Author: martin-g

.github/workflows/test-lang-java.yml

@@ -13,17 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: 'Test Java'
+name: "Test Java"
 on:
   workflow_dispatch:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
     paths:
-    - .github/workflows/test-lang-java.yml
-    - lang/java/**
-    - pom.xml
+      - .github/workflows/test-lang-java.yml
+      - lang/java/**
+      - pom.xml
 
 defaults:
   run:
@@ -35,137 +35,139 @@ concurrency:
 
 jobs:
   test:
-    name: 'Java Test'
+    name: "Java Test"
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: 
+        os:
           - ubuntu-latest
           - ubuntu-24.04-arm
     steps:
-      - name: 'Checkout sourcecode'
+      - name: "Checkout sourcecode"
         uses: actions/checkout@v6
 
-      - name: 'Cache Local Maven Repository'
+      - name: "Cache Local Maven Repository"
         uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - name: 'Setup Temurin JDK 11, 17 & 21'
+      - name: "Setup Temurin JDK 11, 17 & 21"
         uses: actions/setup-java@v5
         with:
-          distribution: 'temurin'
+          distribution: "temurin"
           java-version: |
             11
             17
             21
 
-      - name: 'Setup Maven 3.9.11'
+      - name: "Setup Maven 3.9.11"
         uses: stCarolas/setup-maven@d6af6abeda15e98926a57b5aa970a96bb37f97d1 # v5
         with:
           maven-version: 3.9.11
 
-      - name: 'Install Java Avro Toplevel'
+      - name: "Install Java Avro Toplevel"
         working-directory: ./
         run: mvn -B install -PskipQuality -DskipTests
 
-      - name: 'Java Lint'
+      - name: "Java Lint"
         run: ./build.sh lint
 
-      - name: 'Java Test'
+      - name: "Java Test"
         run: ./build.sh test
 
-      - name: 'Install Java Avro'
+      - name: "Install Java Avro"
         working-directory: .
         run: mvn -B clean install -PskipQuality -DskipTests
 
-      - name: 'Test Reproducible Build'
+      - name: "Test Reproducible Build"
         working-directory: .
         run: mvn clean verify -PskipQuality artifact:compare
 
   interop:
-    name: 'Java Interop'
+    name: "Java Interop"
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: 
+        os:
           - ubuntu-latest
           - ubuntu-24.04-arm
 
     steps:
       - uses: actions/checkout@v6
 
-      - name: 'Cache Local Maven Repository'
+      - name: "Cache Local Maven Repository"
         uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - name: 'Setup Temurin JDK 11, 17 & 21'
+      - name: "Setup Temurin JDK 11, 17 & 21"
         uses: actions/setup-java@v5
         with:
-          distribution: 'temurin'
+          distribution: "temurin"
           java-version: |
             11
             17
             21
 
-      - name: 'Setup Maven 3.9.11'
+      - name: "Setup Maven 3.9.11"
         uses: stCarolas/setup-maven@d6af6abeda15e98926a57b5aa970a96bb37f97d1 # v5
         with:
           maven-version: 3.9.11
 
-      - name: 'Setup Python for Generating Input Data'
+      - name: "Setup Python for Generating Input Data"
         uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
-          python-version: '3.12'
+          python-version: "3.12"
 
-      - name: 'Apt Install Compression Libs Required by Python'
+      - name: Setup uv
+        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867
+
+      - name: "Apt Install Compression Libs Required by Python"
         run: |
           sudo apt-get update && \
           sudo apt-get install -qqy --no-install-recommends libbz2-dev \
                                                             liblzma-dev \
                                                             libsnappy-dev \
                                                             libzstd-dev
 
-      - name: 'Install Python Dependencies'
-        run: |
-          python3 -m pip install --break-system-packages --upgrade pip setuptools tox
-          python3 -m pip install --break-system-packages python-snappy zstandard
+      - name: "Install Python Dependencies"
+        working-directory: lang/py
+        run: uv sync --frozen
 
-      - name: 'Setup C# for Generating Interop Data'
+      - name: "Setup C# for Generating Interop Data"
         uses: actions/setup-dotnet@v5
         with:
           dotnet-version: |
             6.0.x
             7.0.x
             8.0.x
 
-      - name: 'Create Interop Data Directory'
+      - name: "Create Interop Data Directory"
         working-directory: .
         run: mkdir -p build/interop/data
 
-      - name: 'Generate Interop Data using Python'
+      - name: "Generate Interop Data using Python"
         working-directory: lang/py
         run: ./build.sh interop-data-generate
 
-      - name: 'Generate Interop Data using C#'
+      - name: "Generate Interop Data using C#"
         working-directory: lang/csharp
         run: ./build.sh interop-data-generate
 
-      - name: 'Install Java Avro for other tests'
+      - name: "Install Java Avro for other tests"
         working-directory: .
         run: mvn -B install -PskipQuality
 
-      - name: 'Generate Interop Data using Java 11, 17 & 21'
+      - name: "Generate Interop Data using Java 11, 17 & 21"
         working-directory: lang/java/interop-data-test
         run: mvn -B verify -Pgenerate-test-data
 
-      - name: 'Run Interop Tests using Java 11, 17 & 21'
+      - name: "Run Interop Tests using Java 11, 17 & 21"
         working-directory: lang/java/interop-data-test
         run: mvn -B verify -Pcheck-test-data

.github/workflows/test-lang-py.yml

@@ -13,16 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: 'Test Python'
+name: "Test Python"
 on:
   workflow_dispatch:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
     paths:
-    - .github/workflows/test-lang-py.yml
-    - lang/py/**
+      - .github/workflows/test-lang-py.yml
+      - lang/py/**
 
 defaults:
   run:
@@ -38,16 +38,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: 
+        os:
           - ubuntu-latest
           - ubuntu-24.04-arm
         python:
-          - '3.13'
-          - '3.12'
-          - '3.11'
-          - '3.10'
-          - 'pypy-3.11'
-          - 'pypy-3.10'
+          - "3.14"
+          - "3.13"
+          - "3.12"
+          - "3.11"
+          - "3.10"
+          - "pypy-3.11"
+          - "pypy-3.10"
 
     steps:
       - uses: actions/checkout@v6
@@ -57,6 +58,9 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
+      - name: Setup uv
+        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867
+
       - name: Apt Install Compression Libs
         run: |
           sudo apt-get update && \
@@ -67,33 +71,30 @@ jobs:
                                                             libzstd-dev
 
       - name: Install Dependencies
-        run: |
-          python3 -m pip install --upgrade pip setuptools tox
+        run: uv sync --frozen
 
       - name: Lint
         if: ${{ matrix.python == '3.10' }}
-        run: python3 -m tox -e lint
+        run: ./build.sh lint
 
       - name: Typechecks
         if: ${{ matrix.python == '3.10' }}
-        run: python3 -m tox -e typechecks
+        run: ./build.sh typechecks
 
       - name: Test
-        run: python3 -m tox -e py
+        run: ./build.sh test
 
   interop:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: 
+        os:
           - ubuntu-latest
           - ubuntu-24.04-arm
         python:
-          - '3.11'
-          - '3.10'
-          - '3.9'
-          - 'pypy-3.10'
+          - "3.13"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v6
@@ -103,6 +104,9 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
+      - name: Setup uv
+        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867
+
       - name: Apt Install Compression Libs
         run: |
           sudo apt-get update && \
@@ -113,9 +117,7 @@ jobs:
                                                             libzstd-dev
 
       - name: Install Dependencies
-        run: |
-          python3 -m pip install --upgrade pip setuptools tox
-          python3 -m pip install python-snappy zstandard
+        run: uv sync --frozen
 
       - name: Cache Local Maven Repository
         uses: actions/cache@v5
@@ -125,17 +127,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - name: 'Setup Temurin JDK 8, 11, 17 & 21'
+      - name: "Setup Temurin JDK 8, 11, 17 & 21"
         uses: actions/setup-java@v5
         with:
-          distribution: 'temurin'
+          distribution: "temurin"
           java-version: |
             8
             11
             17
             21
 
-      - name: 'Setup Maven 3.9.11'
+      - name: "Setup Maven 3.9.11"
         uses: stCarolas/setup-maven@d6af6abeda15e98926a57b5aa970a96bb37f97d1 # v5
         with:
           maven-version: 3.9.11

lang/py/avro/ipc.py

@@ -322,7 +322,7 @@ def respond(self, call_request):
             else:
                 writers_schema = local_message.errors
                 self.write_error(writers_schema, error, buffer_encoder)
-        except schema.AvroException as e:
+        except avro.errors.AvroException as e:
             error = avro.errors.AvroRemoteException(str(e))
             buffer_encoder = avro.io.BinaryEncoder(io.BytesIO())
             META_WRITER.write(response_metadata, buffer_encoder)

lang/py/avro/test/sample_http_server.py

@@ -32,11 +32,18 @@
             {
                 "name": "Message",
                 "type": "record",
-                "fields": [{"name": "to", "type": "string"}, {"name": "from", "type": "string"}, {"name": "body", "type": "string"}],
+                "fields": [
+                    {"name": "to", "type": "string"},
+                    {"name": "from", "type": "string"},
+                    {"name": "body", "type": "string"},
+                ],
             }
         ],
         "messages": {
-            "send": {"request": [{"name": "message", "type": "Message"}], "response": "string"},
+            "send": {
+                "request": [{"name": "message", "type": "Message"}],
+                "response": "string",
+            },
             "replay": {"request": [], "response": "string"},
         },
     }
@@ -71,7 +78,7 @@ def do_POST(self) -> None:
 
 
 def main():
-    mail_server = http_server.HTTPServer(SERVER_ADDRESS, MailHandler)
+    mail_server = http.server.HTTPServer(SERVER_ADDRESS, MailHandler)
     mail_server.allow_reuse_address = True
     mail_server.serve_forever()