ci: fix publish paimon-datafusion and python issue (#253)

Author: luoyuxia

.github/workflows/release-rust.yml

@@ -53,10 +53,23 @@ jobs:
           rustup default stable
 
       - name: Dry run
+        # Only dry-run paimon since paimon-datafusion depends on it
+        if: matrix.package == 'paimon'
         run: cargo publish -p ${{ matrix.package }} --all-features --dry-run
 
       - name: Publish ${{ matrix.package }} to crates.io
         if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-')
         run: cargo publish -p ${{ matrix.package }} --all-features
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+
+      - name: Wait for crates.io to update
+        if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') && matrix.package == 'paimon'
+        run: |
+          VERSION=${GITHUB_REF_NAME#v}
+          for i in {1..10}; do
+            CURRENT=$(curl -s https://crates.io/api/v1/crates/paimon | jq -r '.crate.max_version')
+            [ "$CURRENT" = "$VERSION" ] && echo "✅ paimon $VERSION available" && exit 0
+            echo "Waiting for $VERSION (currently $CURRENT)... $i/10"
+            sleep 30
+          done

.github/workflows/release_python_binding.yml

@@ -49,7 +49,7 @@ jobs:
           command: sdist
           args: -o dist
           before-script-linux: |
-            yum install -y openssl-devel 2>/dev/null || apt-get update && apt-get install -y libssl-dev
+            sudo apt-get update && sudo apt-get install -y libssl-dev
 
       - name: Upload sdist
         uses: actions/upload-artifact@v4
@@ -78,7 +78,58 @@ jobs:
           args: --release -o dist
           manylinux: ${{ matrix.manylinux || 'auto' }}
           before-script-linux: |
-            yum install -y openssl-devel 2>/dev/null || apt-get update && apt-get install -y libssl-dev
+            # Install OpenSSL for x86_64 (manylinux2014 with OpenSSL 1.1)
+            if [ "${{ matrix.target }}" = "x86_64" ]; then
+              yum install -y openssl11-devel
+              # Create symlinks so pkg-config finds openssl11 as openssl
+              ln -sf /usr/lib64/pkgconfig/openssl11.pc /usr/lib64/pkgconfig/openssl.pc
+              ln -sf /usr/lib64/pkgconfig/libssl11.pc /usr/lib64/pkgconfig/libssl.pc
+              ln -sf /usr/lib64/pkgconfig/libcrypto11.pc /usr/lib64/pkgconfig/libcrypto.pc
+            fi
+
+            # Install precompiled OpenSSL from Debian packages for aarch64
+            if [ "${{ matrix.target }}" = "aarch64" ]; then
+              set -e
+              echo "Installing OpenSSL for aarch64 from Debian packages..."
+
+              (cd /tmp
+              # Debian Bullseye arm64 packages (OpenSSL 1.1.1)
+              DEBIAN_MIRROR="http://ftp.debian.org/debian/pool/main/o/openssl"
+              curl -sLO "${DEBIAN_MIRROR}/libssl1.1_1.1.1w-0+deb11u1_arm64.deb"
+              curl -sLO "${DEBIAN_MIRROR}/libssl-dev_1.1.1w-0+deb11u1_arm64.deb"
+
+              # Extract .deb packages (ar format)
+              for deb in *.deb; do
+                ar x "$deb"
+                tar xf data.tar.*
+                rm -f debian-binary control.tar.* data.tar.*
+              done
+
+              # Install to maturin's expected location
+              TARGET_DIR="/usr/aarch64-unknown-linux-gnu"
+              mkdir -p "${TARGET_DIR}/include/openssl" "${TARGET_DIR}/lib"
+
+              # Copy main OpenSSL headers (architecture-independent)
+              cp -r usr/include/openssl/* "${TARGET_DIR}/include/openssl/"
+
+              # Overwrite with architecture-specific headers
+              cp -f usr/include/aarch64-linux-gnu/openssl/configuration.h "${TARGET_DIR}/include/openssl/" || true
+              cp -f usr/include/aarch64-linux-gnu/openssl/opensslconf.h "${TARGET_DIR}/include/openssl/" || true
+
+              # Copy libraries
+              cp -a usr/lib/aarch64-linux-gnu/libssl* "${TARGET_DIR}/lib/"
+              cp -a usr/lib/aarch64-linux-gnu/libcrypto* "${TARGET_DIR}/lib/"
+
+              # Verify
+              echo "✓ OpenSSL installed"
+              ls "${TARGET_DIR}/lib/libssl.a" && echo "✓ libssl.a found"
+              ls "${TARGET_DIR}/include/openssl/opensslv.h" && echo "✓ opensslv.h found"
+              ls "${TARGET_DIR}/include/openssl/opensslconf.h" && echo "✓ opensslconf.h found"
+              ) # End subshell
+
+              # Fix ring crate build: define ARM architecture version for aarch64
+              export CFLAGS_aarch64_unknown_linux_gnu="-D__ARM_ARCH=8"
+            fi
 
       - name: Upload wheels
         uses: actions/upload-artifact@v4

Cargo.toml

@@ -37,6 +37,7 @@ arrow-ord = "58.0"
 arrow-select = "58.0"
 datafusion = "53.0.0"
 datafusion-ffi = "53.0.0"
+paimon = { version = "0.1.0", path = "crates/paimon" }
 parquet = "58.0"
 tokio = "1.39.2"
 tokio-util = "0.7"

crates/integrations/datafusion/Cargo.toml

@@ -34,7 +34,7 @@ fulltext = ["paimon/fulltext"]
 async-trait = "0.1"
 chrono = "0.4"
 datafusion = { workspace = true }
-paimon = { path = "../../paimon" }
+paimon = { workspace = true }
 futures = "0.3"
 tokio = { workspace = true, features = ["rt", "time", "fs"] }