diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6d8e4d84e..aeda353d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,7 +114,7 @@ Auron provides a unified build script `auron-build.sh` that supports both local ```bash # Build inside Docker container -./auron-build.sh --docker true --image centos7 --release \ +./auron-build.sh --docker true --image rockylinux8 --release \ --sparkver 3.5 --scalaver 2.12 ``` @@ -130,6 +130,28 @@ Run `./auron-build.sh --help` to see all available options, including: - `--sparktests`: Run Spark integration tests - `--threads`: Maven build threads (e.g. 1, 4, 1C). Defaults to single-threaded local builds; Docker defaults to 8 unless overridden. +### Build Compatibility Matrix + +The examples printed by `./auron-build.sh --help` use the following documented build +configurations based on the current compatibility checks. A dash (`—`) means that the +integration is not supported for that Spark version and must not be passed to +`auron-build.sh`. + +| Spark | Scala | JDK | Docker image | Flink | Celeborn | Uniffle | Paimon | Iceberg | Hudi | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 3.5 | 2.12 | 8+ | `rockylinux8` | 1.18 | 0.6 | 0.10 | 1.2 | 1.10.1 | 0.15 | +| 4.0 | 2.13 | 17+ | `rockylinux8` | 1.18 | 0.6 | 0.10 | 1.2 | 1.10.1 | — | +| 4.1 | 2.13 | 17+ | `rockylinux8` | 1.18 | 0.6 | 0.10 | 1.2 | — | — | + +Spark 4.x requires Scala 2.13 and JDK 17 or later. In Docker mode, +`auron-build.sh` selects JDK 17 for Spark 4.x, and the `rockylinux8` image applies that +selection. The `centos7` image only provides JDK 8 and must not be used for Spark 4.x. + +Iceberg 1.10.1 supports Spark 3.4, 3.5, and 4.0 in Auron. Spark 4.1 support requires a +future upgrade to Iceberg 1.11.0 or later. Hudi 0.15 supports Spark 3.0 through 3.5. +When a supported version or compatibility rule changes, update this matrix and the +examples in `auron-build.sh` together. + ### Running Tests By default, the build script skips unit tests (`--skiptests true`). To run them, you must explicitly set `--skiptests false`, as shown in the examples below: diff --git a/auron-build.sh b/auron-build.sh index 2fc673a9e..033b9b540 100755 --- a/auron-build.sh +++ b/auron-build.sh @@ -72,16 +72,33 @@ print_help() { echo "Examples:" echo " $0 --pre --sparkver ${SUPPORTED_SPARK_VERSIONS[*]: -1}" \ "--scalaver ${SUPPORTED_SCALA_VERSIONS[*]: -1} -DskipBuildNative" - echo " $0 --docker true --image ${SUPPORTED_OS_IMAGES[*]:0:1}" \ + echo " # Spark 3.5 with all supported third-party integrations" + echo " $0 --docker true --image rockylinux8" \ "--clean true --skiptests true --release" \ - "--sparkver ${SUPPORTED_SPARK_VERSIONS[*]: -1}" \ + "--sparkver 3.5 --scalaver 2.12" \ "--flinkver ${SUPPORTED_FLINK_VERSIONS[*]: -1}" \ - "--scalaver ${SUPPORTED_SCALA_VERSIONS[*]: -1}" \ "--celeborn ${SUPPORTED_CELEBORN_VERSIONS[*]: -1}" \ "--uniffle ${SUPPORTED_UNIFFLE_VERSIONS[*]: -1}" \ "--paimon ${SUPPORTED_PAIMON_VERSIONS[*]: -1}" \ "--iceberg ${SUPPORTED_ICEBERG_VERSIONS[*]: -1}" \ "--hudi ${SUPPORTED_HUDI_VERSIONS[*]: -1}" + echo " # Spark 4.0 (Hudi is not supported)" + echo " $0 --docker true --image rockylinux8" \ + "--clean true --skiptests true --release" \ + "--sparkver 4.0 --scalaver 2.13" \ + "--flinkver ${SUPPORTED_FLINK_VERSIONS[*]: -1}" \ + "--celeborn ${SUPPORTED_CELEBORN_VERSIONS[*]: -1}" \ + "--uniffle ${SUPPORTED_UNIFFLE_VERSIONS[*]: -1}" \ + "--paimon ${SUPPORTED_PAIMON_VERSIONS[*]: -1}" \ + "--iceberg ${SUPPORTED_ICEBERG_VERSIONS[*]: -1}" + echo " # Spark 4.1 (Iceberg and Hudi are not supported)" + echo " $0 --docker true --image rockylinux8" \ + "--clean true --skiptests true --release" \ + "--sparkver 4.1 --scalaver 2.13" \ + "--flinkver ${SUPPORTED_FLINK_VERSIONS[*]: -1}" \ + "--celeborn ${SUPPORTED_CELEBORN_VERSIONS[*]: -1}" \ + "--uniffle ${SUPPORTED_UNIFFLE_VERSIONS[*]: -1}" \ + "--paimon ${SUPPORTED_PAIMON_VERSIONS[*]: -1}" exit 0 }