Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/source/user-guide/latest/configs.md
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ These settings can be used to determine which parts of the plan are accelerated
| `spark.comet.expression.Log10.enabled` | Enable Comet acceleration for `Log10` | true |
| `spark.comet.expression.Log2.enabled` | Enable Comet acceleration for `Log2` | true |
| `spark.comet.expression.Lower.enabled` | Enable Comet acceleration for `Lower` | true |
| `spark.comet.expression.MakeDate.enabled` | Enable Comet acceleration for `MakeDate` | true |
| `spark.comet.expression.MakeDecimal.enabled` | Enable Comet acceleration for `MakeDecimal` | true |
| `spark.comet.expression.MapEntries.enabled` | Enable Comet acceleration for `MapEntries` | true |
| `spark.comet.expression.MapFromArrays.enabled` | Enable Comet acceleration for `MapFromArrays` | true |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ object QueryPlanSerde extends Logging with CometExprShim {
classOf[DateSub] -> CometDateSub,
classOf[FromUnixTime] -> CometFromUnixTime,
classOf[Hour] -> CometHour,
classOf[MakeDate] -> CometMakeDate,
classOf[Minute] -> CometMinute,
classOf[Second] -> CometSecond,
classOf[TruncDate] -> CometTruncDate,
Expand Down
4 changes: 3 additions & 1 deletion spark/src/main/scala/org/apache/comet/serde/datetime.scala
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ package org.apache.comet.serde

import java.util.Locale

import org.apache.spark.sql.catalyst.expressions.{Attribute, DateAdd, DateSub, DayOfMonth, DayOfWeek, DayOfYear, GetDateField, Hour, Literal, Minute, Month, Quarter, Second, TruncDate, TruncTimestamp, WeekDay, WeekOfYear, Year}
import org.apache.spark.sql.catalyst.expressions.{Attribute, DateAdd, DateSub, DayOfMonth, DayOfWeek, DayOfYear, GetDateField, Hour, Literal, MakeDate, Minute, Month, Quarter, Second, TruncDate, TruncTimestamp, WeekDay, WeekOfYear, Year}
import org.apache.spark.sql.types.{DateType, IntegerType}
import org.apache.spark.unsafe.types.UTF8String

Expand Down Expand Up @@ -258,6 +258,8 @@ object CometDateAdd extends CometScalarFunction[DateAdd]("date_add")

object CometDateSub extends CometScalarFunction[DateSub]("date_sub")

object CometMakeDate extends CometScalarFunction[MakeDate]("make_date")

object CometTruncDate extends CometExpressionSerde[TruncDate] {

val supportedFormats: Seq[String] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,44 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
StructField("fmt", DataTypes.StringType, true)))
FuzzDataGenerator.generateDataFrame(r, spark, schema, 1000, DataGenOptions())
}

test("make_date") {
import org.apache.spark.sql.Row

// Create test data with year, month, day columns (no nulls to avoid edge cases)
val r = new Random(42)
val testData = (1 to 1000).map { _ =>
// Generate reasonable date components
val year = 1900 + r.nextInt(200) // 1900-2099
val month = 1 + r.nextInt(12) // 1-12
val day = 1 + r.nextInt(28) // 1-28 to avoid invalid dates
Row(year, month, day)
}
val schema = StructType(
Seq(
StructField("year", DataTypes.IntegerType, false),
StructField("month", DataTypes.IntegerType, false),
StructField("day", DataTypes.IntegerType, false)))
val df = spark.createDataFrame(spark.sparkContext.parallelize(testData), schema)
df.createOrReplaceTempView("tbl")

// Test with column values
checkSparkAnswerAndOperator(
"SELECT year, month, day, make_date(year, month, day) FROM tbl ORDER BY year, month, day")

// Disable constant folding to ensure literal expressions are executed by Comet
withSQLConf(
SQLConf.OPTIMIZER_EXCLUDED_RULES.key ->
"org.apache.spark.sql.catalyst.optimizer.ConstantFolding") {
// Test with literal values
checkSparkAnswerAndOperator("SELECT make_date(2023, 12, 25)")
checkSparkAnswerAndOperator("SELECT make_date(1970, 1, 1)")
checkSparkAnswerAndOperator("SELECT make_date(2000, 2, 29)") // Leap year

// Test null handling
checkSparkAnswerAndOperator("SELECT make_date(NULL, 1, 1)")
checkSparkAnswerAndOperator("SELECT make_date(2023, NULL, 1)")
checkSparkAnswerAndOperator("SELECT make_date(2023, 1, NULL)")
}
}
}
Loading