From bf84fad478dcd0ae24c47808e59ca20cedaadda0 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Sun, 15 Mar 2026 15:04:24 +0200 Subject: [PATCH 1/3] Use the loaded Comet extension too Signed-off-by: Martin Tzvetanov Grigorov --- dev/diffs/3.5.8.diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/diffs/3.5.8.diff b/dev/diffs/3.5.8.diff index 3aaecdecb1..abb25a8cb3 100644 --- a/dev/diffs/3.5.8.diff +++ b/dev/diffs/3.5.8.diff @@ -76,7 +76,7 @@ index 27ae10b3d59..78e69902dfd 100644 val extensionConfClassNames = sparkContext.getConf.get(StaticSQLConf.SPARK_SESSION_EXTENSIONS) .getOrElse(Seq.empty) + val extensionClassNames = extensionConfClassNames ++ loadCometExtension(sparkContext) - extensionConfClassNames.foreach { extensionConfClassName => + extensionClassNames.foreach { extensionConfClassName => try { val extensionConfClass = Utils.classForName(extensionConfClassName) @@ -1396,4 +1405,12 @@ object SparkSession extends Logging { From 0e0f0b95248c7724429991a2d465fbeb8805ecd9 Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Mon, 16 Mar 2026 14:07:32 +0200 Subject: [PATCH 2/3] Fix extension class names loading logic --- dev/diffs/3.5.8.diff | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/diffs/3.5.8.diff b/dev/diffs/3.5.8.diff index abb25a8cb3..78fb7f2d2f 100644 --- a/dev/diffs/3.5.8.diff +++ b/dev/diffs/3.5.8.diff @@ -76,7 +76,8 @@ index 27ae10b3d59..78e69902dfd 100644 val extensionConfClassNames = sparkContext.getConf.get(StaticSQLConf.SPARK_SESSION_EXTENSIONS) .getOrElse(Seq.empty) + val extensionClassNames = extensionConfClassNames ++ loadCometExtension(sparkContext) - extensionClassNames.foreach { extensionConfClassName => +- extensionConfClassNames.foreach { extensionConfClassName => ++ extensionClassNames.foreach { extensionConfClassName => try { val extensionConfClass = Utils.classForName(extensionConfClassName) @@ -1396,4 +1405,12 @@ object SparkSession extends Logging { From 7a68b8d6ab32a43b7047e6191fe4bb3a1608cbc9 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 19 Mar 2026 11:56:40 +0200 Subject: [PATCH 3/3] Disable few tests in SparkSessionExtensionSuite https://github.com/apache/datafusion-comet/pull/3707#issuecomment-4081584084 --- dev/diffs/3.5.8.diff | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/dev/diffs/3.5.8.diff b/dev/diffs/3.5.8.diff index 32381106ed..e1e3850598 100644 --- a/dev/diffs/3.5.8.diff +++ b/dev/diffs/3.5.8.diff @@ -3272,7 +3272,38 @@ index f3be79f9022..b4b1ea8dbc4 100644 + IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3312")) { withTempDir { tempDir => - + +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala +index 8b4ac474..3f79f208 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala +@@ -223,6 +223,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt + withSession(extensions) { session => + session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true) + session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") ++ // https://github.com/apache/datafusion-comet/issues/1197 ++ session.conf.set("spark.comet.enabled", false) + assert(session.sessionState.columnarRules.contains( + MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) + import session.sqlContext.implicits._ +@@ -281,6 +283,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt + } + withSession(extensions) { session => + session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE) ++ // https://github.com/apache/datafusion-comet/issues/1197 ++ session.conf.set("spark.comet.enabled", false) + assert(session.sessionState.columnarRules.contains( + MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) + import session.sqlContext.implicits._ +@@ -319,6 +323,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt + val session = SparkSession.builder() + .master("local[1]") + .config(COLUMN_BATCH_SIZE.key, 2) ++ // https://github.com/apache/datafusion-comet/issues/1197 ++ .config("spark.comet.enabled", false) + .withExtensions { extensions => + extensions.injectColumnar(session => + MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 6160c3e5f6c..0956d7d9edc 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala