fixes

Author: parthchandra

native/spark-expr/src/conversion_funcs/string.rs

@@ -1103,31 +1103,31 @@ fn timestamp_parser<T: TimeZone>(
     // Define regex patterns and corresponding parsing functions
     let patterns = &[
         (
-            Regex::new(r"^\d{4,5}$").unwrap(),
+            Regex::new(r"^\d{4,7}$").unwrap(),
             parse_str_to_year_timestamp as fn(&str, &T) -> SparkResult<Option<i64>>,
         ),
         (
-            Regex::new(r"^\d{4,5}-\d{2}$").unwrap(),
+            Regex::new(r"^\d{4,7}-\d{2}$").unwrap(),
             parse_str_to_month_timestamp,
         ),
         (
-            Regex::new(r"^\d{4,5}-\d{2}-\d{2}$").unwrap(),
+            Regex::new(r"^\d{4,7}-\d{2}-\d{2}$").unwrap(),
             parse_str_to_day_timestamp,
         ),
         (
-            Regex::new(r"^\d{4,5}-\d{2}-\d{2}T\d{1,2}$").unwrap(),
+            Regex::new(r"^\d{4,7}-\d{2}-\d{2}T\d{1,2}$").unwrap(),
             parse_str_to_hour_timestamp,
         ),
         (
-            Regex::new(r"^\d{4,5}-\d{2}-\d{2}T\d{2}:\d{2}$").unwrap(),
+            Regex::new(r"^\d{4,7}-\d{2}-\d{2}T\d{2}:\d{2}$").unwrap(),
             parse_str_to_minute_timestamp,
         ),
         (
-            Regex::new(r"^\d{4,5}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$").unwrap(),
+            Regex::new(r"^\d{4,7}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$").unwrap(),
             parse_str_to_second_timestamp,
         ),
         (
-            Regex::new(r"^\d{4,5}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{1,6}$").unwrap(),
+            Regex::new(r"^\d{4,7}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{1,6}$").unwrap(),
             parse_str_to_microsecond_timestamp,
         ),
         (

spark/src/main/scala/org/apache/comet/expressions/CometCast.scala

@@ -218,7 +218,7 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
       case DataTypes.TimestampType if timeZoneId.exists(tz => tz != "UTC") =>
         Incompatible(Some(s"Cast will use UTC instead of $timeZoneId"))
       case DataTypes.TimestampType =>
-        Compatible(Some("Only supports years between 262143 BC and 262142 AD"))
+        Incompatible(Some("Not all valid formats are supported"))
       case _ =>
         unsupported(DataTypes.StringType, toType)
     }

spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala

@@ -176,7 +176,7 @@ trait ShimSparkErrorConverter {
               "expression" -> expression,
               "sourceType" -> sourceType,
               "targetType" -> targetType,
-              "ansiConfig" -> "`spark.sql.ansi.enabled`"),
+              "ansiConfig" -> "\"spark.sql.ansi.enabled\""),
             context = context,
             summary = summary))
 

spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala

@@ -174,7 +174,7 @@ trait ShimSparkErrorConverter {
               "expression" -> expression,
               "sourceType" -> sourceType,
               "targetType" -> targetType,
-              "ansiConfig" -> "`spark.sql.ansi.enabled`"),
+              "ansiConfig" -> "\"spark.sql.ansi.enabled\""),
             context = context,
             summary = summary))
 

spark/src/test/scala/org/apache/comet/CometCastSuite.scala

@@ -979,14 +979,21 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
         "10000-01-01T12",
         "10000-01-01T12:34",
         "10000-01-01T12:34:56",
-        "10000-01-01T12:34:56.123456")
-      castTimestampTest(values.toDF("a"), DataTypes.TimestampType, assertNative = true)
+        "10000-01-01T12:34:56.123456",
+        "213170",
+        "213170-06",
+        "213170-06-15",
+        "213170-06-15T12",
+        "213170-06-15T12:34",
+        "213170-06-15T12:34:56",
+        "213170-06-15T12:34:56.123456")
+      castTimestampTest(values.toDF("a"), DataTypes.TimestampType)
     }
   }
 
   ignore("cast StringType to TimestampType") {
-    // TODO: enable once all Spark timestamp formats are supported natively
-    // (e.g. time-only formats like "T12:34" are accepted by Spark but not yet by Comet)
+    // TODO: enable once all Spark timestamp formats are supported natively.
+    // Currently missing: time-only formats with colon (e.g. "T12:34", "4:4").
     withSQLConf((SQLConf.SESSION_LOCAL_TIMEZONE.key, "UTC")) {
       val values = Seq("2020-01-01T12:34:56.123456", "T2") ++ gen.generateStrings(
         dataSize,
@@ -1041,12 +1048,6 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
         "10000-01-01T12:34:56.123456")
       castTimestampTest(values.toDF("a"), DataTypes.TimestampType)
     }
-
-    // test for invalid inputs with UTC - asserts native execution
-    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
-      val values = Seq("-9?", "1-", "0.5")
-      castTimestampTest(values.toDF("a"), DataTypes.TimestampType, assertNative = true)
-    }
   }
 
   // CAST from BinaryType