diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index b7a1e172b2c..556c036d041 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -1134,7 +1134,13 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with Logging { Sig[AssertNotNull](ExpressionNames.ASSERT_NOT_NULL), // For test purpose. Sig[VeloxDummyExpression](VeloxDummyExpression.VELOX_DUMMY_EXPRESSION) - ) + ) ++ scala.util.Try( + // scalastyle:off classforname + Sig( + Class.forName("org.apache.spark.sql.catalyst.expressions.BitmapConstructAgg"), + ExpressionNames.BITMAP_CONSTRUCT_AGG) + // scalastyle:on classforname + ).toOption.toSeq } override def rewriteSpillPath(path: String): String = { diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index 3ea800b60ce..1b1b2cbe854 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -1323,7 +1323,8 @@ bool SubstraitToVeloxPlanValidator::validate(const ::substrait::AggregateRel& ag "regr_slope", "regr_intercept", "regr_sxy", - "regr_replacement"}; + "regr_replacement", + "bitmap_construct_agg"}; auto udafFuncs = UdfLoader::getInstance()->getRegisteredUdafNames(); diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 904bb5ef94f..0680080345f 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -85,6 +85,10 @@ class VeloxTestSettings extends BackendTestSettings { "INCONSISTENT_BEHAVIOR_CROSS_VERSION: compatibility with Spark 2.4/3.2 in reading/writing dates") // Doesn't support unhex with failOnError=true. .exclude("CONVERSION_INVALID_INPUT: to_binary conversion function hex") + // bitmap_construct_agg offloaded to Velox throws GlutenException instead of + // SparkArrayIndexOutOfBoundsException. + .exclude("INVALID_BITMAP_POSITION: position out of bounds") + .exclude("INVALID_BITMAP_POSITION: negative position") enableSuite[GlutenQueryParsingErrorsSuite] enableSuite[GlutenArithmeticExpressionSuite] .exclude("SPARK-45786: Decimal multiply, divide, remainder, quot") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala index e07821857a5..97072bfeff7 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala @@ -16,6 +16,25 @@ */ package org.apache.spark.sql +import org.apache.gluten.execution.HashAggregateExecBaseTransformer + +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + class GlutenBitmapExpressionsQuerySuite extends BitmapExpressionsQuerySuite - with GlutenSQLTestsTrait {} + with GlutenSQLTestsTrait + with AdaptiveSparkPlanHelper { + + test("bitmap_construct_agg routes to native") { + val df = spark.sql( + "SELECT bitmap_construct_agg(bitmap_bit_position(col)) " + + "FROM values (1L), (2L), (3L) AS t(col)") + df.collect() + assert( + collectWithSubqueries(df.queryExecution.executedPlan) { + case h: HashAggregateExecBaseTransformer => h + }.nonEmpty, + "Expected native HashAggregateExecBaseTransformer in plan" + ) + } +} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index f564324b141..3692322e6e9 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -282,6 +282,10 @@ class VeloxTestSettings extends BackendTestSettings { "INCONSISTENT_BEHAVIOR_CROSS_VERSION: compatibility with Spark 2.4/3.2 in reading/writing dates") // Doesn't support unhex with failOnError=true. .exclude("CONVERSION_INVALID_INPUT: to_binary conversion function hex") + // bitmap_construct_agg offloaded to Velox throws GlutenException instead of + // SparkArrayIndexOutOfBoundsException. + .exclude("INVALID_BITMAP_POSITION: position out of bounds") + .exclude("INVALID_BITMAP_POSITION: negative position") enableSuite[GlutenQueryParsingErrorsSuite] enableSuite[GlutenQueryContextSuite] enableSuite[GlutenQueryExecutionAnsiErrorsSuite] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala index e07821857a5..97072bfeff7 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala @@ -16,6 +16,25 @@ */ package org.apache.spark.sql +import org.apache.gluten.execution.HashAggregateExecBaseTransformer + +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + class GlutenBitmapExpressionsQuerySuite extends BitmapExpressionsQuerySuite - with GlutenSQLTestsTrait {} + with GlutenSQLTestsTrait + with AdaptiveSparkPlanHelper { + + test("bitmap_construct_agg routes to native") { + val df = spark.sql( + "SELECT bitmap_construct_agg(bitmap_bit_position(col)) " + + "FROM values (1L), (2L), (3L) AS t(col)") + df.collect() + assert( + collectWithSubqueries(df.queryExecution.executedPlan) { + case h: HashAggregateExecBaseTransformer => h + }.nonEmpty, + "Expected native HashAggregateExecBaseTransformer in plan" + ) + } +} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index d43199709c2..d0c0a144442 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -293,6 +293,10 @@ class VeloxTestSettings extends BackendTestSettings { "INCONSISTENT_BEHAVIOR_CROSS_VERSION: compatibility with Spark 2.4/3.2 in reading/writing dates") // Doesn't support unhex with failOnError=true. .exclude("CONVERSION_INVALID_INPUT: to_binary conversion function hex") + // bitmap_construct_agg offloaded to Velox throws GlutenException instead of + // SparkArrayIndexOutOfBoundsException. + .exclude("INVALID_BITMAP_POSITION: position out of bounds") + .exclude("INVALID_BITMAP_POSITION: negative position") enableSuite[GlutenQueryParsingErrorsSuite] enableSuite[GlutenQueryContextSuite] enableSuite[GlutenQueryExecutionAnsiErrorsSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala index e07821857a5..97072bfeff7 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala @@ -16,6 +16,25 @@ */ package org.apache.spark.sql +import org.apache.gluten.execution.HashAggregateExecBaseTransformer + +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + class GlutenBitmapExpressionsQuerySuite extends BitmapExpressionsQuerySuite - with GlutenSQLTestsTrait {} + with GlutenSQLTestsTrait + with AdaptiveSparkPlanHelper { + + test("bitmap_construct_agg routes to native") { + val df = spark.sql( + "SELECT bitmap_construct_agg(bitmap_bit_position(col)) " + + "FROM values (1L), (2L), (3L) AS t(col)") + df.collect() + assert( + collectWithSubqueries(df.queryExecution.executedPlan) { + case h: HashAggregateExecBaseTransformer => h + }.nonEmpty, + "Expected native HashAggregateExecBaseTransformer in plan" + ) + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index d4afb7ff739..edd285b0f84 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -32,6 +32,7 @@ object ExpressionNames { final val COLLECT_LIST = "collect_list" final val COLLECT_SET = "collect_set" final val BLOOM_FILTER_AGG = "bloom_filter_agg" + final val BITMAP_CONSTRUCT_AGG = "bitmap_construct_agg" final val VAR_SAMP = "var_samp" final val VAR_POP = "var_pop" final val BIT_AND_AGG = "bit_and"