Skip to content

[AURON #1961] Fix Spark 4.0+: unit test catalyst codegen failure due to session artifact isolation #1971

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
richox merged 1 commit into from
Jan 30, 2026
+13 −21
Merged

[AURON #1961] Fix Spark 4.0+: unit test catalyst codegen failure due to session artifact isolation #1971

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import java.text.SimpleDateFormat

import org.apache.spark.sql.{AuronQueryTest, Row}

import org.apache.auron.util.{AuronTestUtils, SparkVersionUtil}
import org.apache.auron.util.AuronTestUtils

class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {

Expand Down Expand Up @@ -83,9 +83,6 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
}

test("spark hash function") {
// TODO: Fix flaky codegen cache failures in SPARK-4.x, https://github.com/apache/auron/issues/1961
assume(!SparkVersionUtil.isSparkV40OrGreater)

withTable("t1") {
sql("create table t1 using parquet as select array(1, 2) as arr")
val functions =
Expand All @@ -97,9 +94,6 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
}

test("expm1 function") {
// TODO: Fix flaky codegen cache failures in SPARK-4.x, https://github.com/apache/auron/issues/1961
assume(!SparkVersionUtil.isSparkV40OrGreater)

withTable("t1") {
sql("create table t1(c1 double) using parquet")
sql("insert into t1 values(0.0), (1.1), (2.2)")
Expand Down
11 changes: 1 addition & 10 deletions spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import org.apache.spark.sql.{AuronQueryTest, Row}
import org.apache.spark.sql.execution.joins.auron.plan.NativeBroadcastJoinExec

import org.apache.auron.spark.configuration.SparkAuronConfiguration
import org.apache.auron.util.{AuronTestUtils, SparkVersionUtil}
import org.apache.auron.util.AuronTestUtils

class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQLTestHelper {
import testImplicits._
Expand All @@ -42,9 +42,6 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
}

test("test filter with year function") {
// TODO: Fix flaky codegen cache failures in SPARK-4.x, https://github.com/apache/auron/issues/1961
assume(!SparkVersionUtil.isSparkV40OrGreater)

withTable("t1") {
sql("create table t1 using parquet as select '2024-12-18' as event_time")
checkSparkAnswerAndOperator(s"""
Expand All @@ -57,9 +54,6 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
}

test("test select multiple spark ext functions with the same signature") {
// TODO: Fix flaky codegen cache failures in SPARK-4.x, https://github.com/apache/auron/issues/1961
assume(!SparkVersionUtil.isSparkV40OrGreater)

withTable("t1") {
sql("create table t1 using parquet as select '2024-12-18' as event_time")
checkSparkAnswerAndOperator("select year(event_time), month(event_time) from t1")
Expand Down Expand Up @@ -177,9 +171,6 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
}

test("floor function with long input") {
// TODO: Fix flaky codegen cache failures in SPARK-4.x, https://github.com/apache/auron/issues/1961
assume(!SparkVersionUtil.isSparkV40OrGreater)

withTable("t1") {
sql("create table t1 using parquet as select 1L as c1, 2.2 as c2")
checkSparkAnswerAndOperator("select floor(c1), floor(c2) from t1")
Expand Down
15 changes: 11 additions & 4 deletions spark-extension-shims-spark/src/test/scala/org/apache/auron/BaseAuronSQLSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import org.apache.commons.io.FileUtils
import org.apache.spark.SparkConf
import org.apache.spark.sql.test.SharedSparkSession

import org.apache.auron.util.SparkVersionUtil

trait BaseAuronSQLSuite extends SharedSparkSession {
protected val suiteWorkspace: String = getClass.getResource("/").getPath + "auron-tests-workdir"
protected val warehouseDir: String = suiteWorkspace + "/spark-warehouse"
Expand Down Expand Up @@ -49,7 +51,7 @@ trait BaseAuronSQLSuite extends SharedSparkSession {
}

override protected def sparkConf: SparkConf = {
super.sparkConf
val conf = super.sparkConf
.set("spark.sql.extensions", "org.apache.spark.sql.auron.AuronSparkSessionExtension")
.set(
"spark.shuffle.manager",
Expand All @@ -58,8 +60,13 @@ trait BaseAuronSQLSuite extends SharedSparkSession {
.set("spark.auron.enable", "true")
.set("spark.ui.enabled", "false")
.set("spark.sql.warehouse.dir", warehouseDir)
// Avoid the code size overflow error in Spark code generation.
.set("spark.sql.codegen.wholeStage", "false")
.set("spark.sql.codegen.factoryMode", "NO_CODEGEN")

if (SparkVersionUtil.isSparkV40OrGreater) {
// Spark 4.0+: Disable session artifact isolation, align with Spark 3.x behavior
// Fix Catalyst codegen failure: prevent org.apache.spark.sql.catalyst.expressions.Object
// in isolated dirs from REPL classloader lookup failure
conf.set("spark.sql.artifact.isolation.enabled", "false")
}
conf
}
}
Loading