From e269eb24170c162be82eda7fcdaad64b7612710f Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 26 Jul 2026 02:40:02 +0800 Subject: [PATCH 1/5] Add native make_interval support --- docs/source/user-guide/latest/expressions.md | 2 +- native/Cargo.lock | 1 + native/core/src/execution/jni_api.rs | 2 + native/spark-expr/Cargo.toml | 1 + native/spark-expr/src/comet_scalar_funcs.rs | 7 +- .../src/datetime_funcs/make_interval.rs | 82 +++++++++++++++++++ native/spark-expr/src/datetime_funcs/mod.rs | 2 + native/spark-expr/src/lib.rs | 5 +- .../apache/comet/serde/QueryPlanSerde.scala | 10 ++- .../org/apache/comet/serde/datetime.scala | 13 ++- .../expressions/datetime/make_interval.sql | 46 +++++++++++ .../datetime/make_interval_ansi.sql | 35 ++++++++ 12 files changed, 199 insertions(+), 7 deletions(-) create mode 100644 native/spark-expr/src/datetime_funcs/make_interval.rs create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index c292700dbc..f987642bee 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -277,7 +277,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | `localtimestamp` | ✅ | — | | | `make_date` | ✅ | Native | | | `make_dt_interval` | ✅ | Codegen dispatch | | -| `make_interval` | 🔜 | — | Produces legacy CalendarInterval; tracked by [#4540](https://github.com/apache/datafusion-comet/issues/4540) | +| `make_interval` | ✅ | Native | | | `make_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) | | `make_timestamp` | ✅ | Hybrid | | | `make_timestamp_ltz` | ✅ | — | 2-arg TIME form falls back | diff --git a/native/Cargo.lock b/native/Cargo.lock index 8194fdd70a..4dfeb9c074 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -2065,6 +2065,7 @@ dependencies = [ "datafusion", "datafusion-comet-common", "datafusion-comet-jni-bridge", + "datafusion-spark", "futures", "jni 0.22.4", "num", diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 9a82eb6746..da0d880dc1 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -43,6 +43,7 @@ use datafusion::{ }; use datafusion_comet_proto::spark_operator::Operator; use datafusion_comet_spark_expr::url_funcs::{CometParseUrl, CometTryParseUrl}; +use datafusion_comet_spark_expr::SparkMakeInterval; use datafusion_spark::function::array::array_contains::SparkArrayContains; use datafusion_spark::function::array::repeat::SparkArrayRepeat; use datafusion_spark::function::bitwise::bit_count::SparkBitCount; @@ -618,6 +619,7 @@ fn register_datafusion_spark_function(session_ctx: &SessionContext) { session_ctx.register_udf(ScalarUDF::new_from_impl(SparkDateSub::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkFromUtcTimestamp::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkLastDay::default())); + session_ctx.register_udf(ScalarUDF::new_from_impl(SparkMakeInterval::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkToUtcTimestamp::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSha1::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkConcat::default())); diff --git a/native/spark-expr/Cargo.toml b/native/spark-expr/Cargo.toml index c05ae89793..70eedf810c 100644 --- a/native/spark-expr/Cargo.toml +++ b/native/spark-expr/Cargo.toml @@ -30,6 +30,7 @@ edition = { workspace = true } arrow = { workspace = true } chrono = { workspace = true } datafusion = { workspace = true } +datafusion-spark = { workspace = true } chrono-tz = { workspace = true } num = { workspace = true } regex = { workspace = true } diff --git a/native/spark-expr/src/comet_scalar_funcs.rs b/native/spark-expr/src/comet_scalar_funcs.rs index 8e913dabab..ed85c8cee0 100644 --- a/native/spark-expr/src/comet_scalar_funcs.rs +++ b/native/spark-expr/src/comet_scalar_funcs.rs @@ -27,8 +27,8 @@ use crate::{ spark_isnan, spark_lpad, spark_make_decimal, spark_month_name, spark_read_side_padding, spark_round, spark_rpad, spark_to_time, spark_unhex, spark_unscaled_value, EvalMode, SparkArrayPositionFunc, SparkArraySlice, SparkArraysOverlap, SparkContains, SparkDateDiff, - SparkDateFromUnixDate, SparkDateTrunc, SparkFlatten, SparkMakeDate, SparkMakeTime, - SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc, + SparkDateFromUnixDate, SparkDateTrunc, SparkFlatten, SparkMakeDate, SparkMakeInterval, + SparkMakeTime, SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc, }; use arrow::datatypes::DataType; use datafusion::common::{DataFusionError, Result as DataFusionResult}; @@ -230,6 +230,9 @@ pub fn create_comet_physical_fun_with_eval_mode( "make_date" => Ok(Arc::new(ScalarUDF::new_from_impl(SparkMakeDate::new( fail_on_error, )))), + "make_interval" => Ok(Arc::new(ScalarUDF::new_from_impl(SparkMakeInterval::new( + fail_on_error, + )))), "next_day" => Ok(Arc::new(ScalarUDF::new_from_impl(SparkNextDay::new( fail_on_error, )))), diff --git a/native/spark-expr/src/datetime_funcs/make_interval.rs b/native/spark-expr/src/datetime_funcs/make_interval.rs new file mode 100644 index 0000000000..ad9996959e --- /dev/null +++ b/native/spark-expr/src/datetime_funcs/make_interval.rs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::arithmetic_overflow_error; +use arrow::array::Array; +use arrow::datatypes::DataType; +use datafusion::common::Result; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature}; +use datafusion_spark::function::datetime::make_interval::SparkMakeInterval as DataFusionMakeInterval; + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkMakeInterval { + inner: DataFusionMakeInterval, + fail_on_error: bool, +} + +impl SparkMakeInterval { + pub fn new(fail_on_error: bool) -> Self { + Self { + inner: DataFusionMakeInterval::new(), + fail_on_error, + } + } +} + +impl Default for SparkMakeInterval { + fn default() -> Self { + Self::new(false) + } +} + +impl ScalarUDFImpl for SparkMakeInterval { + fn name(&self) -> &str { + self.inner.name() + } + + fn signature(&self) -> &Signature { + self.inner.signature() + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + self.inner.return_type(arg_types) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let inputs = if self.fail_on_error && !args.args.is_empty() { + Some(ColumnarValue::values_to_arrays(&args.args)?) + } else { + None + }; + let result = self.inner.invoke_with_args(args)?; + + if let Some(inputs) = inputs { + let overflow = match &result { + ColumnarValue::Array(values) => (0..values.len()) + .any(|i| values.is_null(i) && inputs.iter().all(|a| !a.is_null(i))), + ColumnarValue::Scalar(value) => { + value.is_null() && inputs.iter().all(|a| !a.is_null(0)) + } + }; + if overflow { + return Err(arithmetic_overflow_error("interval").into()); + } + } + + Ok(result) + } +} diff --git a/native/spark-expr/src/datetime_funcs/mod.rs b/native/spark-expr/src/datetime_funcs/mod.rs index 37c5fa5dd2..05530f29c2 100644 --- a/native/spark-expr/src/datetime_funcs/mod.rs +++ b/native/spark-expr/src/datetime_funcs/mod.rs @@ -22,6 +22,7 @@ mod day_month_name; mod extract_date_part; mod hours; mod make_date; +mod make_interval; mod make_time; mod next_day; mod seconds_to_timestamp; @@ -38,6 +39,7 @@ pub use extract_date_part::SparkMinute; pub use extract_date_part::SparkSecond; pub use hours::SparkHoursTransform; pub use make_date::SparkMakeDate; +pub use make_interval::SparkMakeInterval; pub use make_time::SparkMakeTime; pub use next_day::SparkNextDay; pub use seconds_to_timestamp::SparkSecondsToTimestamp; diff --git a/native/spark-expr/src/lib.rs b/native/spark-expr/src/lib.rs index 40422cde07..19a0775291 100644 --- a/native/spark-expr/src/lib.rs +++ b/native/spark-expr/src/lib.rs @@ -77,8 +77,9 @@ pub use comet_scalar_funcs::{ pub use csv_funcs::*; pub use datetime_funcs::{ spark_day_name, spark_month_name, spark_to_time, SparkDateDiff, SparkDateFromUnixDate, - SparkDateTrunc, SparkHour, SparkHoursTransform, SparkMakeDate, SparkMakeTime, SparkMinute, - SparkNextDay, SparkSecond, SparkSecondsToTimestamp, SparkUnixTimestamp, TimestampTruncExpr, + SparkDateTrunc, SparkHour, SparkHoursTransform, SparkMakeDate, SparkMakeInterval, + SparkMakeTime, SparkMinute, SparkNextDay, SparkSecond, SparkSecondsToTimestamp, + SparkUnixTimestamp, TimestampTruncExpr, }; pub use error::{decimal_overflow_error, SparkError, SparkErrorWithContext, SparkResult}; pub use hash_funcs::*; diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 6f8306862a..961431838e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -301,6 +301,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[MakeTimestamp] -> CometMakeTimestamp, classOf[MakeYMInterval] -> CometMakeYMInterval, classOf[MakeDTInterval] -> CometMakeDTInterval, + classOf[MakeInterval] -> CometMakeInterval, classOf[MultiplyDTInterval] -> CometMultiplyDTInterval, classOf[MicrosToTimestamp] -> CometMicrosToTimestamp, classOf[MillisToTimestamp] -> CometMillisToTimestamp, @@ -1068,9 +1069,16 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { } def scalarFunctionExprToProto(funcName: String, args: Option[Expr]*): Option[Expr] = { + scalarFunctionExprToProto(funcName, false, args: _*) + } + + def scalarFunctionExprToProto( + funcName: String, + failOnError: Boolean, + args: Option[Expr]*): Option[Expr] = { val builder = ExprOuterClass.ScalarFunc.newBuilder() builder.setFunc(funcName) - builder.setFailOnError(false) + builder.setFailOnError(failOnError) scalarFunctionExprToProto0(builder, args: _*) } diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index d968c00cd1..a32627c83a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -21,7 +21,7 @@ package org.apache.comet.serde import java.util.Locale -import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} +import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String @@ -954,6 +954,17 @@ object CometMakeYMInterval extends CometCodegenDispatch[MakeYMInterval] object CometMakeDTInterval extends CometCodegenDispatch[MakeDTInterval] +object CometMakeInterval extends CometExpressionSerde[MakeInterval] { + override def convert( + expr: MakeInterval, + inputs: Seq[Attribute], + binding: Boolean): Option[Expr] = { + val childExprs = expr.children.map(exprToProtoInternal(_, inputs, binding)) + val optExpr = scalarFunctionExprToProto("make_interval", expr.failOnError, childExprs: _*) + optExprWithFallbackReason(optExpr, expr, expr.children: _*) + } +} + object CometMultiplyDTInterval extends CometCodegenDispatch[MultiplyDTInterval] /** diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql new file mode 100644 index 0000000000..924bbd5fa6 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false + +statement +CREATE TABLE test_make_interval( + years int, + months int, + weeks int, + days int, + hours int, + mins int, + secs decimal(38, 6)) USING parquet + +statement +INSERT INTO test_make_interval VALUES + (1, 2, 3, 4, 5, 6, 7.123456), + (0, 1, 0, 1, 0, 0, 100.000001), + (-1, -2, -1, -1, -1, -1, -1.500000), + (NULL, 1, 2, 3, 4, 5, 6.000000) + +query +SELECT make_interval(years, months, weeks, days, hours, mins, secs) +FROM test_make_interval +ORDER BY years + +query +SELECT make_interval(1, 2), make_interval(3), make_interval() + +query +SELECT make_interval(2147483647) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql new file mode 100644 index 0000000000..1e521cff68 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Native ANSI execution must preserve Spark's overflow exception. +-- Config: spark.sql.ansi.enabled=true +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false + +statement +CREATE TABLE test_make_interval_ansi(years int) USING parquet + +statement +INSERT INTO test_make_interval_ansi VALUES (NULL) + +query +SELECT make_interval(1, 2, 3, 4, 5, 6, 7.123456) + +query +SELECT make_interval(years) FROM test_make_interval_ansi + +query expect_error(overflow) +SELECT make_interval(2147483647) From 69eb4aa061117b6b7b1d7c7df451d5b2693bd4ed Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 29 Jul 2026 23:21:37 +0800 Subject: [PATCH 2/5] review --- docs/source/user-guide/latest/expressions.md | 2 +- .../src/datetime_funcs/make_interval.rs | 23 +++++++++++----- .../org/apache/comet/serde/datetime.scala | 26 +++++++++++++++---- .../expressions/datetime/make_interval.sql | 24 ++++++++++++++--- .../datetime/make_interval_ansi.sql | 5 +++- 5 files changed, 63 insertions(+), 17 deletions(-) diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 28f7a9a7c4..43b69442e1 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -277,7 +277,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | `localtimestamp` | ✅ | — | | | `make_date` | ✅ | Native | | | `make_dt_interval` | ✅ | Codegen dispatch | | -| `make_interval` | ✅ | Native | | +| `make_interval` | ✅ | Native | Falls back by default; opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) | | `make_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) | | `make_timestamp` | ✅ | Hybrid | | | `make_timestamp_ltz` | ✅ | — | 2-arg TIME form falls back | diff --git a/native/spark-expr/src/datetime_funcs/make_interval.rs b/native/spark-expr/src/datetime_funcs/make_interval.rs index ad9996959e..d6105ba87f 100644 --- a/native/spark-expr/src/datetime_funcs/make_interval.rs +++ b/native/spark-expr/src/datetime_funcs/make_interval.rs @@ -57,20 +57,29 @@ impl ScalarUDFImpl for SparkMakeInterval { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let inputs = if self.fail_on_error && !args.args.is_empty() { - Some(ColumnarValue::values_to_arrays(&args.args)?) + let inputs = if self.fail_on_error { + Some(args.args.clone()) } else { None }; let result = self.inner.invoke_with_args(args)?; if let Some(inputs) = inputs { + let inputs_are_valid = |i| { + inputs.iter().all(|input| match input { + ColumnarValue::Array(values) => values.is_valid(i), + ColumnarValue::Scalar(value) => !value.is_null(), + }) + }; let overflow = match &result { - ColumnarValue::Array(values) => (0..values.len()) - .any(|i| values.is_null(i) && inputs.iter().all(|a| !a.is_null(i))), - ColumnarValue::Scalar(value) => { - value.is_null() && inputs.iter().all(|a| !a.is_null(0)) - } + ColumnarValue::Array(values) => values.nulls().is_some_and(|nulls| { + nulls.null_count() != 0 + && nulls + .iter() + .enumerate() + .any(|(i, is_valid)| !is_valid && inputs_are_valid(i)) + }), + ColumnarValue::Scalar(value) => value.is_null() && inputs_are_valid(0), }; if overflow { return Err(arithmetic_overflow_error("interval").into()); diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index a32627c83a..570853924c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -21,9 +21,9 @@ package org.apache.comet.serde import java.util.Locale -import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} +import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometConf @@ -955,13 +955,29 @@ object CometMakeYMInterval extends CometCodegenDispatch[MakeYMInterval] object CometMakeDTInterval extends CometCodegenDispatch[MakeDTInterval] object CometMakeInterval extends CometExpressionSerde[MakeInterval] { + private val incompatReason = + "The native implementation converts seconds to `Float64`, which can lose microsecond" + + " precision, and stores time in nanoseconds, which overflows for large seconds values" + + " that Spark can represent." + + override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason) + + override def getSupportLevel(expr: MakeInterval): SupportLevel = + Incompatible(Some(incompatReason)) + override def convert( expr: MakeInterval, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = { - val childExprs = expr.children.map(exprToProtoInternal(_, inputs, binding)) - val optExpr = scalarFunctionExprToProto("make_interval", expr.failOnError, childExprs: _*) - optExprWithFallbackReason(optExpr, expr, expr.children: _*) + // The explicit return type skips DataFusion's registry coercion, but its kernel needs Float64. + val children = expr.children.updated(6, Cast(expr.children(6), DoubleType)) + val childExprs = children.map(exprToProtoInternal(_, inputs, binding)) + val optExpr = scalarFunctionExprToProtoWithReturnType( + "make_interval", + CalendarIntervalType, + expr.failOnError, + childExprs: _*) + optExprWithFallbackReason(optExpr, expr, children: _*) } } diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql index 924bbd5fa6..5f7bc6be2f 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql @@ -15,7 +15,7 @@ -- specific language governing permissions and limitations -- under the License. --- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.comet.expression.MakeInterval.allowIncompatible=true statement CREATE TABLE test_make_interval( @@ -25,14 +25,17 @@ CREATE TABLE test_make_interval( days int, hours int, mins int, - secs decimal(38, 6)) USING parquet + secs decimal(18, 6)) USING parquet statement INSERT INTO test_make_interval VALUES (1, 2, 3, 4, 5, 6, 7.123456), (0, 1, 0, 1, 0, 0, 100.000001), (-1, -2, -1, -1, -1, -1, -1.500000), - (NULL, 1, 2, 3, 4, 5, 6.000000) + (NULL, 1, 2, 3, 4, 5, 6.000000), + (2, NULL, 2, 3, 4, 5, 6.000000), + (3, 1, 2, 3, 4, 5, NULL), + (-2147483648, 0, 0, 0, 0, 0, 0.000000) query SELECT make_interval(years, months, weeks, days, hours, mins, secs) @@ -42,5 +45,20 @@ ORDER BY years query SELECT make_interval(1, 2), make_interval(3), make_interval() +query +SELECT make_interval(0, 1, 0, 1, 0, 0, 100.000001) + query SELECT make_interval(2147483647) + +query ignore(https://github.com/apache/datafusion-comet/issues/5131) +SELECT make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456) + +query +SELECT make_interval(0, 0, 0, 0, 0, 0, 999999999.999999) + +query ignore(https://github.com/apache/datafusion-comet/issues/5131) +SELECT make_interval(0, 0, 0, 0, 0, 0, 999999999.000001) + +query +SELECT make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql index 1e521cff68..a56dbfc7d1 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql @@ -17,7 +17,7 @@ -- Native ANSI execution must preserve Spark's overflow exception. -- Config: spark.sql.ansi.enabled=true --- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.comet.expression.MakeInterval.allowIncompatible=true statement CREATE TABLE test_make_interval_ansi(years int) USING parquet @@ -33,3 +33,6 @@ SELECT make_interval(years) FROM test_make_interval_ansi query expect_error(overflow) SELECT make_interval(2147483647) + +query expect_error(overflow) +SELECT make_interval(0, 0, 2147483647) From 8739e746cd7fcbea1ada50fe9f23ad4a0b4a0f91 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 5 Aug 2026 22:58:54 +0800 Subject: [PATCH 3/5] address make_interval review feedback --- docs/source/user-guide/latest/expressions.md | 4 +- native/core/src/execution/jni_api.rs | 2 - .../src/datetime_funcs/make_interval.rs | 8 +-- .../org/apache/comet/serde/datetime.scala | 6 +- .../expressions/datetime/make_interval.sql | 3 + .../datetime/make_interval_ansi.sql | 4 +- .../datetime/try_make_interval.sql | 23 ++++++++ .../CometDatetimeExpressionBenchmark.scala | 55 +++++++++++++++++++ 8 files changed, 90 insertions(+), 15 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index a6cbd37a6f..3e2fbb948e 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -277,7 +277,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | `localtimestamp` | ✅ | — | | | `make_date` | ✅ | Native | | | `make_dt_interval` | ✅ | Codegen dispatch | | -| `make_interval` | ✅ | Native | Falls back by default; opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) | +| `make_interval` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by default; the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) | | `make_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) | | `make_timestamp` | ✅ | Hybrid | | | `make_timestamp_ltz` | ✅ | — | 2-arg TIME form falls back | @@ -305,7 +305,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | `to_unix_timestamp` | ✅ | Hybrid | | | `to_utc_timestamp` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by default (handles all timezone forms); the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) | | `trunc` | ✅ | Hybrid | | -| `try_make_interval` | 🔜 | — | Produces legacy CalendarInterval; tracked by [#5061](https://github.com/apache/datafusion-comet/issues/5061) | +| `try_make_interval` | ✅ | — | Rewrites to `MakeInterval`; same support as `make_interval` (Spark 4.0+) | | `try_make_timestamp` | ✅ | — | | | `try_to_date` | ✅ | — | Rewrites to `Cast`/`GetTimestamp` before Comet sees the plan; same support as `to_date` | | `try_to_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) | diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 6cc19bdf73..f798b80335 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -43,7 +43,6 @@ use datafusion::{ }; use datafusion_comet_proto::spark_operator::Operator; use datafusion_comet_spark_expr::url_funcs::{CometParseUrl, CometTryParseUrl}; -use datafusion_comet_spark_expr::SparkMakeInterval; use datafusion_spark::function::array::array_contains::SparkArrayContains; use datafusion_spark::function::array::repeat::SparkArrayRepeat; use datafusion_spark::function::bitwise::bit_count::SparkBitCount; @@ -624,7 +623,6 @@ fn register_datafusion_spark_function(session_ctx: &SessionContext) { session_ctx.register_udf(ScalarUDF::new_from_impl(SparkDateSub::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkFromUtcTimestamp::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkLastDay::default())); - session_ctx.register_udf(ScalarUDF::new_from_impl(SparkMakeInterval::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkToUtcTimestamp::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSha1::default())); session_ctx.register_udf(ScalarUDF::new_from_impl(SparkConcat::default())); diff --git a/native/spark-expr/src/datetime_funcs/make_interval.rs b/native/spark-expr/src/datetime_funcs/make_interval.rs index d6105ba87f..9f7b21ca2a 100644 --- a/native/spark-expr/src/datetime_funcs/make_interval.rs +++ b/native/spark-expr/src/datetime_funcs/make_interval.rs @@ -37,12 +37,6 @@ impl SparkMakeInterval { } } -impl Default for SparkMakeInterval { - fn default() -> Self { - Self::new(false) - } -} - impl ScalarUDFImpl for SparkMakeInterval { fn name(&self) -> &str { self.inner.name() @@ -82,6 +76,8 @@ impl ScalarUDFImpl for SparkMakeInterval { ColumnarValue::Scalar(value) => value.is_null() && inputs_are_valid(0), }; if overflow { + // Spark identifies the integer or long operation that overflowed. The native + // wrapper only sees the result null mask, so it can only report interval overflow. return Err(arithmetic_overflow_error("interval").into()); } } diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index 041a439ab5..7426f9428f 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -968,11 +968,11 @@ object CometMakeYMInterval extends CometCodegenDispatch[MakeYMInterval] object CometMakeDTInterval extends CometCodegenDispatch[MakeDTInterval] -object CometMakeInterval extends CometExpressionSerde[MakeInterval] { +object CometMakeInterval extends CometExpressionSerde[MakeInterval] with CodegenDispatchFallback { private val incompatReason = "The native implementation converts seconds to `Float64`, which can lose microsecond" + - " precision, and stores time in nanoseconds, which overflows for large seconds values" + - " that Spark can represent." + " precision, and stores time in nanoseconds, which overflows for large time components" + + " (hours, minutes, seconds) that Spark can represent." override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql index 5f7bc6be2f..952a9085ee 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql @@ -60,5 +60,8 @@ SELECT make_interval(0, 0, 0, 0, 0, 0, 999999999.999999) query ignore(https://github.com/apache/datafusion-comet/issues/5131) SELECT make_interval(0, 0, 0, 0, 0, 0, 999999999.000001) +query ignore(https://github.com/apache/datafusion-comet/issues/5131) +SELECT make_interval(0, 0, 0, 0, 2562048) + query SELECT make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql index a56dbfc7d1..a13eba375f 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql @@ -31,8 +31,8 @@ SELECT make_interval(1, 2, 3, 4, 5, 6, 7.123456) query SELECT make_interval(years) FROM test_make_interval_ansi -query expect_error(overflow) +query expect_error(ARITHMETIC_OVERFLOW) SELECT make_interval(2147483647) -query expect_error(overflow) +query expect_error(ARITHMETIC_OVERFLOW) SELECT make_interval(0, 0, 2147483647) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql new file mode 100644 index 0000000000..b9a2a77b47 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql @@ -0,0 +1,23 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 4.0 +-- Config: spark.sql.ansi.enabled=true +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true + +query +SELECT try_make_interval(2147483647) diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala index 224f2ad344..a9d5b35e17 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala @@ -19,9 +19,12 @@ package org.apache.spark.sql.benchmark +import org.apache.spark.benchmark.Benchmark import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA} import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.CometConf + // spotless:off /** * Benchmark to measure Comet execution performance. To run this benchmark: @@ -131,6 +134,54 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { } } + def makeIntervalBenchmark(values: Int): Unit = { + withTempPath { dir => + withTempTable("parquetV1Table") { + prepareTable( + dir, + spark.sql(s"""SELECT + | CAST(ABS(value) % 10 AS INT) AS y, + | CAST(ABS(value) % 12 AS INT) AS mo, + | CAST(ABS(value) % 4 AS INT) AS w, + | CAST(ABS(value) % 28 AS INT) AS d, + | CAST(ABS(value) % 24 AS INT) AS h, + | CAST(ABS(value) % 60 AS INT) AS mi, + | CAST(ABS(value) % 60 AS DECIMAL(18, 6)) AS s + |FROM $tbl""".stripMargin)) + + val query = "SELECT make_interval(y, mo, w, d, h, mi, s) FROM parquetV1Table" + def consumeIntervals(): Unit = { + spark.sql(query).queryExecution.toRdd.foreachPartition(_.foreach(_.getInterval(0))) + } + val benchmark = new Benchmark("MakeInterval", values, output = output) + val cometConfigs = Map( + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + "spark.sql.optimizer.excludedRules" -> + "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") + + benchmark.addCase("Spark") { _ => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + consumeIntervals() + } + } + benchmark.addCase("Comet (codegen dispatch)") { _ => + withSQLConf(cometConfigs.toSeq: _*) { + consumeIntervals() + } + } + benchmark.addCase("Comet (native)") { _ => + val configs = + cometConfigs ++ Map(CometConf.getExprAllowIncompatConfigKey("MakeInterval") -> "true") + withSQLConf(configs.toSeq: _*) { + consumeIntervals() + } + } + benchmark.run() + } + } + } + override def runCometBenchmark(mainArgs: Array[String]): Unit = { val values = 1024 * 1024; @@ -167,6 +218,10 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { makeTimeBenchmark(v) } } + + runBenchmarkWithTable("MakeInterval", values) { v => + makeIntervalBenchmark(v) + } } } From 8693243d52cf1bf5a272096e16f772d08cd3a473 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 5 Aug 2026 23:27:44 +0800 Subject: [PATCH 4/5] fix Spark 4.0 make_interval compile --- spark/src/main/scala/org/apache/comet/serde/datetime.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index 4689f7761b..b1dc04cc16 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -986,7 +986,7 @@ object CometMakeInterval extends CometExpressionSerde[MakeInterval] with Codegen CalendarIntervalType, expr.failOnError, childExprs: _*) - optExprWithFallbackReason(optExpr, expr, children: _*) + optExpr } } From 6e28936c7fccc20408d42e411aade366f7c139f2 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Aug 2026 04:10:53 +0800 Subject: [PATCH 5/5] Address make_interval review feedback --- docs/source/user-guide/latest/expressions.md | 2 +- .../apache/comet/serde/QueryPlanSerde.scala | 9 +--- .../org/apache/comet/serde/datetime.scala | 8 ++- .../datetime/make_interval_ansi.sql | 7 ++- .../datetime/make_interval_dispatch.sql | 49 +++++++++++++++++++ .../datetime/make_interval_dispatch_ansi.sql | 48 ++++++++++++++++++ .../datetime/try_make_interval.sql | 22 +++++++-- 7 files changed, 130 insertions(+), 15 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 0ac0353d9e..192eae807e 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -277,7 +277,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci | `localtimestamp` | ✅ | — | | | `make_date` | ✅ | Native | | | `make_dt_interval` | ✅ | Codegen dispatch | | -| `make_interval` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by default; the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) | +| `make_interval` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by default; intervals outside Arrow's nanosecond range are tracked by [#5279](https://github.com/apache/datafusion-comet/issues/5279); the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) | | `make_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) | | `make_timestamp` | ✅ | Hybrid | | | `make_timestamp_ltz` | ✅ | — | 2-arg TIME form falls back | diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index b3014d6651..85601a9e0c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -1132,16 +1132,9 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { } def scalarFunctionExprToProto(funcName: String, args: Option[Expr]*): Option[Expr] = { - scalarFunctionExprToProto(funcName, false, args: _*) - } - - def scalarFunctionExprToProto( - funcName: String, - failOnError: Boolean, - args: Option[Expr]*): Option[Expr] = { val builder = ExprOuterClass.ScalarFunc.newBuilder() builder.setFunc(funcName) - builder.setFailOnError(failOnError) + builder.setFailOnError(false) scalarFunctionExprToProto0(builder, args: _*) } diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala index b1dc04cc16..579a22dac8 100644 --- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala @@ -969,6 +969,12 @@ object CometMakeInterval extends CometExpressionSerde[MakeInterval] with Codegen " precision, and stores time in nanoseconds, which overflows for large time components" + " (hours, minutes, seconds) that Spark can represent." + override def getCompatibleNotes(): Seq[String] = Seq( + "Both the default JVM codegen-dispatch path and the native path encode elapsed time as" + + " Arrow nanoseconds. Spark intervals whose microseconds cannot be multiplied by 1,000" + + " in an `i64` are not supported" + + " ([#5279](https://github.com/apache/datafusion-comet/issues/5279)).") + override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason) override def getSupportLevel(expr: MakeInterval): SupportLevel = @@ -979,7 +985,7 @@ object CometMakeInterval extends CometExpressionSerde[MakeInterval] with Codegen inputs: Seq[Attribute], binding: Boolean): Option[Expr] = { // The explicit return type skips DataFusion's registry coercion, but its kernel needs Float64. - val children = expr.children.updated(6, Cast(expr.children(6), DoubleType)) + val children = expr.children.updated(6, Cast(expr.secs, DoubleType)) val childExprs = children.map(exprToProtoInternal(_, inputs, binding)) val optExpr = scalarFunctionExprToProtoWithReturnType( "make_interval", diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql index a13eba375f..175b9ecd08 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql @@ -31,8 +31,11 @@ SELECT make_interval(1, 2, 3, 4, 5, 6, 7.123456) query SELECT make_interval(years) FROM test_make_interval_ansi -query expect_error(ARITHMETIC_OVERFLOW) +query expect_error(overflow. If necessary set) SELECT make_interval(2147483647) -query expect_error(ARITHMETIC_OVERFLOW) +query expect_error(overflow. If necessary set) SELECT make_interval(0, 0, 2147483647) + +query ignore(https://github.com/apache/datafusion-comet/issues/5131) +SELECT make_interval(0, 0, 0, 0, 2562048) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql new file mode 100644 index 0000000000..ee2d5e8160 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql @@ -0,0 +1,49 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- With allowIncompatible unset, MakeInterval uses Spark's JVM codegen dispatcher. + +statement +CREATE TABLE test_make_interval_dispatch( + years int, + months int, + weeks int, + days int, + hours int, + mins int, + secs decimal(18, 6)) USING parquet + +statement +INSERT INTO test_make_interval_dispatch VALUES + (1, 2, 3, 4, 5, 6, 7.123456), + (0, 1, 0, 1, 0, 0, 100.000001), + (-1, -2, -1, -1, -1, -1, -1.500000), + (NULL, 1, 2, 3, 4, 5, 6.000000), + (2, NULL, 2, 3, 4, 5, 6.000000), + (3, 1, 2, 3, 4, 5, NULL), + (0, 0, 0, 0, 2562048, 0, 0.000000) + +query +SELECT make_interval(years, months, weeks, days, hours, mins, secs) +FROM test_make_interval_dispatch +WHERE hours != 2562048 +ORDER BY years + +query ignore(https://github.com/apache/datafusion-comet/issues/5279) +SELECT make_interval(0, 0, 0, 0, hours) +FROM test_make_interval_dispatch +WHERE hours = 2562048 diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql new file mode 100644 index 0000000000..f199a8fb66 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql @@ -0,0 +1,48 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.sql.ansi.enabled=true + +statement +CREATE TABLE test_make_interval_dispatch_ansi(years int, weeks int, hours int) USING parquet + +statement +INSERT INTO test_make_interval_dispatch_ansi VALUES + (1, 0, 5), + (2147483647, 0, 0), + (0, 2147483647, 0), + (0, 0, 2562048) + +query +SELECT make_interval(years, 0, weeks, 0, hours) +FROM test_make_interval_dispatch_ansi +WHERE years = 1 + +query expect_error(overflow. If necessary set) +SELECT make_interval(years) +FROM test_make_interval_dispatch_ansi +WHERE years = 2147483647 + +query expect_error(overflow. If necessary set) +SELECT make_interval(0, 0, weeks) +FROM test_make_interval_dispatch_ansi +WHERE weeks = 2147483647 + +query ignore(https://github.com/apache/datafusion-comet/issues/5279) +SELECT make_interval(0, 0, 0, 0, hours) +FROM test_make_interval_dispatch_ansi +WHERE hours = 2562048 diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql index b9a2a77b47..5e481cb9ff 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql @@ -16,8 +16,24 @@ -- under the License. -- MinSparkVersion: 4.0 --- Config: spark.sql.ansi.enabled=true --- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- ConfigMatrix: spark.sql.ansi.enabled=true,false + +statement +CREATE TABLE test_try_make_interval( + years int, + months int, + weeks int, + days int, + hours int, + mins int, + secs decimal(18, 6)) USING parquet + +statement +INSERT INTO test_try_make_interval VALUES + (1, 2, 3, 4, 5, 6, 7.123456), + (2147483647, 0, 0, 0, 0, 0, 0.000000) query -SELECT try_make_interval(2147483647) +SELECT try_make_interval(years, months, weeks, days, hours, mins, secs) +FROM test_try_make_interval +ORDER BY years