diff --git a/.github/workflows/iceberg_spark_test.yml b/.github/workflows/iceberg_spark_test.yml index ec764279a6..eec2c99466 100644 --- a/.github/workflows/iceberg_spark_test.yml +++ b/.github/workflows/iceberg_spark_test.yml @@ -102,138 +102,6 @@ jobs: path: native/target/ci/libcomet.so retention-days: 1 - iceberg-spark: - needs: build-native - if: contains(github.event.pull_request.title, '[iceberg]') - strategy: - matrix: - os: [ubuntu-24.04] - java-version: [11, 17] - iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}] - spark-version: [{short: '3.5', full: '3.5.8'}] - scala-version: ['2.13'] - fail-fast: false - name: iceberg-spark/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v6 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java-version }} - - name: Download native library - uses: actions/download-artifact@v8 - with: - name: native-lib-iceberg - path: native/target/release/ - - name: Build Comet - run: | - ./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }} - - name: Setup Iceberg - uses: ./.github/actions/setup-iceberg-builder - with: - iceberg-version: ${{ matrix.iceberg-version.full }} - - name: Run Iceberg Spark tests - run: | - cd apache-iceberg - rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups - ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \ - :iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:test \ - -Pquick=true -x javadoc - - iceberg-spark-extensions: - needs: build-native - if: contains(github.event.pull_request.title, '[iceberg]') - strategy: - matrix: - os: [ubuntu-24.04] - java-version: [11, 17] - iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}] - spark-version: [{short: '3.5', full: '3.5.8'}] - scala-version: ['2.13'] - fail-fast: false - name: iceberg-spark-extensions/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v6 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java-version }} - - name: Download native library - uses: actions/download-artifact@v8 - with: - name: native-lib-iceberg - path: native/target/release/ - - name: Build Comet - run: | - ./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }} - - name: Setup Iceberg - uses: ./.github/actions/setup-iceberg-builder - with: - iceberg-version: ${{ matrix.iceberg-version.full }} - - name: Run Iceberg Spark extensions tests - run: | - cd apache-iceberg - rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups - ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \ - :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:test \ - -Pquick=true -x javadoc - - iceberg-spark-runtime: - needs: build-native - if: contains(github.event.pull_request.title, '[iceberg]') - strategy: - matrix: - os: [ubuntu-24.04] - java-version: [11, 17] - iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}] - spark-version: [{short: '3.5', full: '3.5.8'}] - scala-version: ['2.13'] - fail-fast: false - name: iceberg-spark-runtime/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v6 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java-version }} - - name: Download native library - uses: actions/download-artifact@v8 - with: - name: native-lib-iceberg - path: native/target/release/ - - name: Build Comet - run: | - ./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }} - - name: Setup Iceberg - uses: ./.github/actions/setup-iceberg-builder - with: - iceberg-version: ${{ matrix.iceberg-version.full }} - - name: Run Iceberg Spark runtime tests - run: | - cd apache-iceberg - rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups - ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \ - :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:integrationTest \ - -Pquick=true -x javadoc - iceberg-spark-rust: needs: build-native if: contains(github.event.pull_request.title, '[iceberg]')