diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index fd4324babc..192eae807e 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -277,7 +277,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | `localtimestamp` | ✅ | — | | | `make_date` | ✅ | Native | | | `make_dt_interval` | ✅ | Codegen dispatch | | -| `make_interval` | 🔜 | — | Produces legacy CalendarInterval; tracked by [#5061](https://github.com/apache/datafusion-comet/issues/5061) | +| `make_interval` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by default; intervals outside Arrow's nanosecond range are tracked by [#5279](https://github.com/apache/datafusion-comet/issues/5279); the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) | | `make_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) | | `make_timestamp` | ✅ | Hybrid | | | `make_timestamp_ltz` | ✅ | — | 2-arg TIME form falls back | @@ -308,7 +308,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | `to_unix_timestamp` | ✅ | Hybrid | | | `to_utc_timestamp` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by default (handles all timezone forms); the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) | | `trunc` | ✅ | Hybrid | | -| `try_make_interval` | 🔜 | — | Produces legacy CalendarInterval; tracked by [#5061](https://github.com/apache/datafusion-comet/issues/5061) | +| `try_make_interval` | ✅ | — | Rewrites to `MakeInterval`; same support as `make_interval` (Spark 4.0+) | | `try_make_timestamp` | ✅ | — | | | `try_to_date` | ✅ | — | Rewrites to `Cast`/`GetTimestamp` before Comet sees the plan; same support as `to_date` | | `try_to_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) | diff --git a/native/Cargo.lock b/native/Cargo.lock index eeaa9da5ce..ae8a2f7be7 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -2066,6 +2066,7 @@ dependencies = [ "datafusion", "datafusion-comet-common", "datafusion-comet-jni-bridge", + "datafusion-spark", "futures", "jni 0.22.4", "num", diff --git a/native/spark-expr/Cargo.toml b/native/spark-expr/Cargo.toml index 2333d2f0e6..6faa9fec4e 100644 --- a/native/spark-expr/Cargo.toml +++ b/native/spark-expr/Cargo.toml @@ -30,6 +30,7 @@ edition = { workspace = true } arrow = { workspace = true } chrono = { workspace = true } datafusion = { workspace = true } +datafusion-spark = { workspace = true } chrono-tz = { workspace = true } num = { workspace = true } regex = { workspace = true } diff --git a/native/spark-expr/src/comet_scalar_funcs.rs b/native/spark-expr/src/comet_scalar_funcs.rs index 50732959c8..e90e01de95 100644 --- a/native/spark-expr/src/comet_scalar_funcs.rs +++ b/native/spark-expr/src/comet_scalar_funcs.rs @@ -28,8 +28,8 @@ use crate::{ spark_isnan, spark_lpad, spark_make_decimal, spark_month_name, spark_read_side_padding, spark_round, spark_rpad, spark_to_time, spark_unhex, spark_unscaled_value, EvalMode, SparkArrayPositionFunc, SparkArraySlice, SparkArraysOverlap, SparkContains, SparkDateDiff, - SparkDateFromUnixDate, SparkDateTrunc, SparkFlatten, SparkMakeDate, SparkMakeTime, - SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc, + SparkDateFromUnixDate, SparkDateTrunc, SparkFlatten, SparkMakeDate, SparkMakeInterval, + SparkMakeTime, SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc, }; use arrow::datatypes::DataType; use datafusion::common::{DataFusionError, Result as DataFusionResult}; @@ -250,6 +250,9 @@ pub fn create_comet_physical_fun_with_eval_mode( "make_date" => Ok(Arc::new(ScalarUDF::new_from_impl(SparkMakeDate::new( fail_on_error, )))), + "make_interval" => Ok(Arc::new(ScalarUDF::new_from_impl(SparkMakeInterval::new( + fail_on_error, + )))), "next_day" => Ok(Arc::new(ScalarUDF::new_from_impl(SparkNextDay::new( fail_on_error, )))), diff --git a/native/spark-expr/src/datetime_funcs/make_interval.rs b/native/spark-expr/src/datetime_funcs/make_interval.rs new file mode 100644 index 0000000000..9f7b21ca2a --- /dev/null +++ b/native/spark-expr/src/datetime_funcs/make_interval.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::arithmetic_overflow_error; +use arrow::array::Array; +use arrow::datatypes::DataType; +use datafusion::common::Result; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature}; +use datafusion_spark::function::datetime::make_interval::SparkMakeInterval as DataFusionMakeInterval; + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkMakeInterval { + inner: DataFusionMakeInterval, + fail_on_error: bool, +} + +impl SparkMakeInterval { + pub fn new(fail_on_error: bool) -> Self { + Self { + inner: DataFusionMakeInterval::new(), + fail_on_error, + } + } +} + +impl ScalarUDFImpl for SparkMakeInterval { + fn name(&self) -> &str { + self.inner.name() + } + + fn signature(&self) -> &Signature { + self.inner.signature() + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + self.inner.return_type(arg_types) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let inputs = if self.fail_on_error { + Some(args.args.clone()) + } else { + None + }; + let result = self.inner.invoke_with_args(args)?; + + if let Some(inputs) = inputs { + let inputs_are_valid = |i| { + inputs.iter().all(|input| match input { + ColumnarValue::Array(values) => values.is_valid(i), + ColumnarValue::Scalar(value) => !value.is_null(), + }) + }; + let overflow = match &result { + ColumnarValue::Array(values) => values.nulls().is_some_and(|nulls| { + nulls.null_count() != 0 + && nulls + .iter() + .enumerate() + .any(|(i, is_valid)| !is_valid && inputs_are_valid(i)) + }), + ColumnarValue::Scalar(value) => value.is_null() && inputs_are_valid(0), + }; + if overflow { + // Spark identifies the integer or long operation that overflowed. The native + // wrapper only sees the result null mask, so it can only report interval overflow. + return Err(arithmetic_overflow_error("interval").into()); + } + } + + Ok(result) + } +} diff --git a/native/spark-expr/src/datetime_funcs/mod.rs b/native/spark-expr/src/datetime_funcs/mod.rs index 37c5fa5dd2..05530f29c2 100644 --- a/native/spark-expr/src/datetime_funcs/mod.rs +++ b/native/spark-expr/src/datetime_funcs/mod.rs @@ -22,6 +22,7 @@ mod day_month_name; mod extract_date_part; mod hours; mod make_date; +mod make_interval; mod make_time; mod next_day; mod seconds_to_timestamp; @@ -38,6 +39,7 @@ pub use extract_date_part::SparkMinute; pub use extract_date_part::SparkSecond; pub use hours::SparkHoursTransform; pub use make_date::SparkMakeDate; +pub use make_interval::SparkMakeInterval; pub use make_time::SparkMakeTime; pub use next_day::SparkNextDay; pub use seconds_to_timestamp::SparkSecondsToTimestamp; diff --git a/native/spark-expr/src/lib.rs b/native/spark-expr/src/lib.rs index 42ab43361a..2b5c29befc 100644 --- a/native/spark-expr/src/lib.rs +++ b/native/spark-expr/src/lib.rs @@ -77,8 +77,9 @@ pub use comet_scalar_funcs::{ pub use csv_funcs::*; pub use datetime_funcs::{ spark_day_name, spark_month_name, spark_to_time, SparkDateDiff, SparkDateFromUnixDate, - SparkDateTrunc, SparkHour, SparkHoursTransform, SparkMakeDate, SparkMakeTime, SparkMinute, - SparkNextDay, SparkSecond, SparkSecondsToTimestamp, SparkUnixTimestamp, TimestampTruncExpr, + SparkDateTrunc, SparkHour, SparkHoursTransform, SparkMakeDate, SparkMakeInterval, + SparkMakeTime, SparkMinute, SparkNextDay, SparkSecond, SparkSecondsToTimestamp, + SparkUnixTimestamp, TimestampTruncExpr, }; pub use error::{decimal_overflow_error, SparkError, SparkErrorWithContext, SparkResult}; pub use hash_funcs::*; diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 4299eb14ba..85601a9e0c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -303,6 +303,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[MakeTimestamp] -> CometMakeTimestamp, classOf[MakeYMInterval] -> CometMakeYMInterval, classOf[MakeDTInterval] -> CometMakeDTInterval, + classOf[MakeInterval] -> CometMakeInterval, classOf[MultiplyDTInterval] -> CometMultiplyDTInterval, classOf[TimestampAdd] -> CometTimestampAdd, classOf[TimestampDiff] -> CometTimestampDiff, diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index 946c34a857..a74a158e97 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -21,9 +21,9 @@ package org.apache.comet.serde import java.util.Locale -import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, TimestampAdd, TimestampDiff, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} +import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, TimestampAdd, TimestampDiff, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometConf @@ -963,6 +963,39 @@ object CometMakeYMInterval extends CometCodegenDispatch[MakeYMInterval] object CometMakeDTInterval extends CometCodegenDispatch[MakeDTInterval] +object CometMakeInterval extends CometExpressionSerde[MakeInterval] with CodegenDispatchFallback { + private val incompatReason = + "The native implementation converts seconds to `Float64`, which can lose microsecond" + + " precision, and stores time in nanoseconds, which overflows for large time components" + + " (hours, minutes, seconds) that Spark can represent." + + override def getCompatibleNotes(): Seq[String] = Seq( + "Both the default JVM codegen-dispatch path and the native path currently limit the" + + " elapsed-time component to about 292 years in either direction. This only affects" + + " extreme intervals and is tracked in" + + " [#5279](https://github.com/apache/datafusion-comet/issues/5279).") + + override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason) + + override def getSupportLevel(expr: MakeInterval): SupportLevel = + Incompatible(Some(incompatReason)) + + override def convert( + expr: MakeInterval, + inputs: Seq[Attribute], + binding: Boolean): Option[Expr] = { + // The explicit return type skips DataFusion's registry coercion, but its kernel needs Float64. + val children = expr.children.updated(6, Cast(expr.secs, DoubleType)) + val childExprs = children.map(exprToProtoInternal(_, inputs, binding)) + val optExpr = scalarFunctionExprToProtoWithReturnType( + "make_interval", + CalendarIntervalType, + expr.failOnError, + childExprs: _*) + optExpr + } +} + object CometMultiplyDTInterval extends CometCodegenDispatch[MultiplyDTInterval] object CometTimestampAdd extends CometCodegenDispatch[TimestampAdd] diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql new file mode 100644 index 0000000000..952a9085ee --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql @@ -0,0 +1,67 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.expression.MakeInterval.allowIncompatible=true + +statement +CREATE TABLE test_make_interval( + years int, + months int, + weeks int, + days int, + hours int, + mins int, + secs decimal(18, 6)) USING parquet + +statement +INSERT INTO test_make_interval VALUES + (1, 2, 3, 4, 5, 6, 7.123456), + (0, 1, 0, 1, 0, 0, 100.000001), + (-1, -2, -1, -1, -1, -1, -1.500000), + (NULL, 1, 2, 3, 4, 5, 6.000000), + (2, NULL, 2, 3, 4, 5, 6.000000), + (3, 1, 2, 3, 4, 5, NULL), + (-2147483648, 0, 0, 0, 0, 0, 0.000000) + +query +SELECT make_interval(years, months, weeks, days, hours, mins, secs) +FROM test_make_interval +ORDER BY years + +query +SELECT make_interval(1, 2), make_interval(3), make_interval() + +query +SELECT make_interval(0, 1, 0, 1, 0, 0, 100.000001) + +query +SELECT make_interval(2147483647) + +query ignore(https://github.com/apache/datafusion-comet/issues/5131) +SELECT make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456) + +query +SELECT make_interval(0, 0, 0, 0, 0, 0, 999999999.999999) + +query ignore(https://github.com/apache/datafusion-comet/issues/5131) +SELECT make_interval(0, 0, 0, 0, 0, 0, 999999999.000001) + +query ignore(https://github.com/apache/datafusion-comet/issues/5131) +SELECT make_interval(0, 0, 0, 0, 2562048) + +query +SELECT make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql new file mode 100644 index 0000000000..175b9ecd08 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql @@ -0,0 +1,41 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Native ANSI execution must preserve Spark's overflow exception. +-- Config: spark.sql.ansi.enabled=true +-- Config: spark.comet.expression.MakeInterval.allowIncompatible=true + +statement +CREATE TABLE test_make_interval_ansi(years int) USING parquet + +statement +INSERT INTO test_make_interval_ansi VALUES (NULL) + +query +SELECT make_interval(1, 2, 3, 4, 5, 6, 7.123456) + +query +SELECT make_interval(years) FROM test_make_interval_ansi + +query expect_error(overflow. If necessary set) +SELECT make_interval(2147483647) + +query expect_error(overflow. If necessary set) +SELECT make_interval(0, 0, 2147483647) + +query ignore(https://github.com/apache/datafusion-comet/issues/5131) +SELECT make_interval(0, 0, 0, 0, 2562048) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql new file mode 100644 index 0000000000..ee2d5e8160 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql @@ -0,0 +1,49 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- With allowIncompatible unset, MakeInterval uses Spark's JVM codegen dispatcher. + +statement +CREATE TABLE test_make_interval_dispatch( + years int, + months int, + weeks int, + days int, + hours int, + mins int, + secs decimal(18, 6)) USING parquet + +statement +INSERT INTO test_make_interval_dispatch VALUES + (1, 2, 3, 4, 5, 6, 7.123456), + (0, 1, 0, 1, 0, 0, 100.000001), + (-1, -2, -1, -1, -1, -1, -1.500000), + (NULL, 1, 2, 3, 4, 5, 6.000000), + (2, NULL, 2, 3, 4, 5, 6.000000), + (3, 1, 2, 3, 4, 5, NULL), + (0, 0, 0, 0, 2562048, 0, 0.000000) + +query +SELECT make_interval(years, months, weeks, days, hours, mins, secs) +FROM test_make_interval_dispatch +WHERE hours != 2562048 +ORDER BY years + +query ignore(https://github.com/apache/datafusion-comet/issues/5279) +SELECT make_interval(0, 0, 0, 0, hours) +FROM test_make_interval_dispatch +WHERE hours = 2562048 diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql new file mode 100644 index 0000000000..f199a8fb66 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql @@ -0,0 +1,48 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.sql.ansi.enabled=true + +statement +CREATE TABLE test_make_interval_dispatch_ansi(years int, weeks int, hours int) USING parquet + +statement +INSERT INTO test_make_interval_dispatch_ansi VALUES + (1, 0, 5), + (2147483647, 0, 0), + (0, 2147483647, 0), + (0, 0, 2562048) + +query +SELECT make_interval(years, 0, weeks, 0, hours) +FROM test_make_interval_dispatch_ansi +WHERE years = 1 + +query expect_error(overflow. If necessary set) +SELECT make_interval(years) +FROM test_make_interval_dispatch_ansi +WHERE years = 2147483647 + +query expect_error(overflow. If necessary set) +SELECT make_interval(0, 0, weeks) +FROM test_make_interval_dispatch_ansi +WHERE weeks = 2147483647 + +query ignore(https://github.com/apache/datafusion-comet/issues/5279) +SELECT make_interval(0, 0, 0, 0, hours) +FROM test_make_interval_dispatch_ansi +WHERE hours = 2562048 diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql new file mode 100644 index 0000000000..5e481cb9ff --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 4.0 +-- ConfigMatrix: spark.sql.ansi.enabled=true,false + +statement +CREATE TABLE test_try_make_interval( + years int, + months int, + weeks int, + days int, + hours int, + mins int, + secs decimal(18, 6)) USING parquet + +statement +INSERT INTO test_try_make_interval VALUES + (1, 2, 3, 4, 5, 6, 7.123456), + (2147483647, 0, 0, 0, 0, 0, 0.000000) + +query +SELECT try_make_interval(years, months, weeks, days, hours, mins, secs) +FROM test_try_make_interval +ORDER BY years diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala index 224f2ad344..a9d5b35e17 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala @@ -19,9 +19,12 @@ package org.apache.spark.sql.benchmark +import org.apache.spark.benchmark.Benchmark import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA} import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.CometConf + // spotless:off /** * Benchmark to measure Comet execution performance. To run this benchmark: @@ -131,6 +134,54 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { } } + def makeIntervalBenchmark(values: Int): Unit = { + withTempPath { dir => + withTempTable("parquetV1Table") { + prepareTable( + dir, + spark.sql(s"""SELECT + | CAST(ABS(value) % 10 AS INT) AS y, + | CAST(ABS(value) % 12 AS INT) AS mo, + | CAST(ABS(value) % 4 AS INT) AS w, + | CAST(ABS(value) % 28 AS INT) AS d, + | CAST(ABS(value) % 24 AS INT) AS h, + | CAST(ABS(value) % 60 AS INT) AS mi, + | CAST(ABS(value) % 60 AS DECIMAL(18, 6)) AS s + |FROM $tbl""".stripMargin)) + + val query = "SELECT make_interval(y, mo, w, d, h, mi, s) FROM parquetV1Table" + def consumeIntervals(): Unit = { + spark.sql(query).queryExecution.toRdd.foreachPartition(_.foreach(_.getInterval(0))) + } + val benchmark = new Benchmark("MakeInterval", values, output = output) + val cometConfigs = Map( + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + "spark.sql.optimizer.excludedRules" -> + "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") + + benchmark.addCase("Spark") { _ => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + consumeIntervals() + } + } + benchmark.addCase("Comet (codegen dispatch)") { _ => + withSQLConf(cometConfigs.toSeq: _*) { + consumeIntervals() + } + } + benchmark.addCase("Comet (native)") { _ => + val configs = + cometConfigs ++ Map(CometConf.getExprAllowIncompatConfigKey("MakeInterval") -> "true") + withSQLConf(configs.toSeq: _*) { + consumeIntervals() + } + } + benchmark.run() + } + } + } + override def runCometBenchmark(mainArgs: Array[String]): Unit = { val values = 1024 * 1024; @@ -167,6 +218,10 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { makeTimeBenchmark(v) } } + + runBenchmarkWithTable("MakeInterval", values) { v => + makeIntervalBenchmark(v) + } } }