From 8ce988fae81c5cf64d3936bb46aefe6e8e69e5c2 Mon Sep 17 00:00:00 2001 From: shaoyijie Date: Sun, 26 Jul 2026 21:34:19 -0700 Subject: [PATCH 1/2] [python] Support pypaimon-rust 0.4 --- .github/workflows/paimon-python-checks.yml | 5 +++-- paimon-python/setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/paimon-python-checks.yml b/.github/workflows/paimon-python-checks.yml index 00540533f191..51b5ad4ddc3b 100755 --- a/.github/workflows/paimon-python-checks.yml +++ b/.github/workflows/paimon-python-checks.yml @@ -34,7 +34,7 @@ env: JDK_VERSION: 8 MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true LUMINA_DATA_VERSION: 0.1.0 - PYPAIMON_RUST_REV: 7c568274a4c5df6bd00e8d60edea21395412202b + PYPAIMON_RUST_REV: 6b6782923a59eeba766cbd45dde6989705b282bc concurrency: @@ -133,8 +133,9 @@ jobs: pip install torch --index-url https://download.pytorch.org/whl/cpu python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 'isal>=1.8,<2' pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' 'datafusion>=52' datasketches if [[ "${{ matrix.python-version }}" == "3.11" ]]; then - # Exercise the split-planning API in one lane until the compatible 0.3 wheel is published. + # Exercise the 0.4 API in one lane until its wheel is published. python -m pip install "git+https://github.com/apache/paimon-rust.git@${PYPAIMON_RUST_REV}#subdirectory=bindings/python" + python -m pip install "./paimon-python[sql]" python -c "from pypaimon_rust.datafusion import PaimonCatalog; assert hasattr(PaimonCatalog, 'get_table')" else python -m pip install pypaimon-rust==0.2.0 diff --git a/paimon-python/setup.py b/paimon-python/setup.py index 214d81313c5c..178b4a47f798 100644 --- a/paimon-python/setup.py +++ b/paimon-python/setup.py @@ -187,7 +187,7 @@ def read_requirements(): 'paimon-ftindex==0.1.0; python_version>="3.8"', ], 'sql': [ - 'pypaimon-rust>=0.3.0,<0.4.0; python_version>="3.10"', + 'pypaimon-rust>=0.3.0,<0.5.0; python_version>="3.10"', 'datafusion>=52; python_version>="3.10"', ], 'hdfs': [ From a15c0c4729e544a94e72df25a8bd6941dd8b5221 Mon Sep 17 00:00:00 2001 From: shaoyijie Date: Mon, 27 Jul 2026 04:20:38 -0700 Subject: [PATCH 2/2] [python] Align DataFusion ABI for pypaimon-rust 0.4 --- .github/workflows/paimon-python-checks.yml | 4 ++-- paimon-python/setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/paimon-python-checks.yml b/.github/workflows/paimon-python-checks.yml index 51b5ad4ddc3b..5b1ec0c64929 100755 --- a/.github/workflows/paimon-python-checks.yml +++ b/.github/workflows/paimon-python-checks.yml @@ -131,12 +131,12 @@ jobs: else python -m pip install --upgrade pip pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 'isal>=1.8,<2' pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' 'datafusion>=52' datasketches + python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 'isal>=1.8,<2' pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' 'datafusion>=54,<55' datasketches if [[ "${{ matrix.python-version }}" == "3.11" ]]; then # Exercise the 0.4 API in one lane until its wheel is published. python -m pip install "git+https://github.com/apache/paimon-rust.git@${PYPAIMON_RUST_REV}#subdirectory=bindings/python" python -m pip install "./paimon-python[sql]" - python -c "from pypaimon_rust.datafusion import PaimonCatalog; assert hasattr(PaimonCatalog, 'get_table')" + python -c "import tempfile; from datafusion import SessionContext; from pypaimon_rust.datafusion import PaimonCatalog; warehouse = tempfile.TemporaryDirectory(); ctx = SessionContext(); ctx.register_catalog_provider('paimon', PaimonCatalog({'warehouse': warehouse.name}))" else python -m pip install pypaimon-rust==0.2.0 fi diff --git a/paimon-python/setup.py b/paimon-python/setup.py index 178b4a47f798..ea31de98fe28 100644 --- a/paimon-python/setup.py +++ b/paimon-python/setup.py @@ -187,8 +187,8 @@ def read_requirements(): 'paimon-ftindex==0.1.0; python_version>="3.8"', ], 'sql': [ - 'pypaimon-rust>=0.3.0,<0.5.0; python_version>="3.10"', - 'datafusion>=52; python_version>="3.10"', + 'pypaimon-rust>=0.3.0; python_version>="3.10"', + 'datafusion>=54,<55; python_version>="3.10"', ], 'hdfs': [ 'hdfs-native>=0.13,<1; python_version >= "3.10" and platform_system != "Windows"',