From 8864057df96f0927c8ba4e56b1a6e8bb45e37c7e Mon Sep 17 00:00:00 2001 From: dor-bernstein <160585775+dor-bernstein@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:22:55 +0300 Subject: [PATCH 1/3] fix(glue): map Timestamptz/TimestamptzNs to Glue timestamp types Glue has no timezone-aware timestamp type. Previously, schemas containing Timestamptz or TimestamptzNs columns returned FeatureUnsupported during catalog operations (e.g. committing rewrites), causing failures for any table with timestamp-with-timezone fields. Map Timestamptz -> "timestamp" and TimestamptzNs -> "timestamp_ns", matching the behavior of Spark's Glue catalog integration which also drops timezone information when serializing to the Glue type system. Adds a regression test covering both Timestamptz and TimestamptzNs fields. --- crates/catalog/glue/src/schema.rs | 46 +++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/crates/catalog/glue/src/schema.rs b/crates/catalog/glue/src/schema.rs index 59d51f3dc1..f584fbc4ce 100644 --- a/crates/catalog/glue/src/schema.rs +++ b/crates/catalog/glue/src/schema.rs @@ -157,12 +157,10 @@ impl SchemaVisitor for GlueSchemaBuilder { PrimitiveType::Date => "date".to_string(), PrimitiveType::Timestamp => "timestamp".to_string(), PrimitiveType::TimestampNs => "timestamp_ns".to_string(), - PrimitiveType::Timestamptz | PrimitiveType::TimestamptzNs => { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - format!("Conversion from {p:?} is not supported"), - )); - } + // Glue has no timezone-aware timestamp type; map to the equivalent non-tz type, + // matching the behavior of Spark's Glue catalog integration. + PrimitiveType::Timestamptz => "timestamp".to_string(), + PrimitiveType::TimestamptzNs => "timestamp_ns".to_string(), PrimitiveType::Time | PrimitiveType::String | PrimitiveType::Uuid => { "string".to_string() } @@ -342,6 +340,42 @@ mod tests { Ok(()) } + #[test] + fn test_schema_with_timestamptz_fields() -> Result<()> { + let record = r#"{ + "type": "struct", + "schema-id": 1, + "fields": [ + { + "id": 1, + "name": "ts_tz", + "required": true, + "type": "timestamptz" + }, + { + "id": 2, + "name": "ts_tz_ns", + "required": true, + "type": "timestamptz_ns" + } + ] + }"#; + + let schema = serde_json::from_str::(record)?; + let metadata = create_metadata(schema)?; + + let result = GlueSchemaBuilder::from_iceberg(&metadata)?.build(); + + let expected = vec![ + create_column("ts_tz", "timestamp", "1", false)?, + create_column("ts_tz_ns", "timestamp_ns", "2", false)?, + ]; + + assert_eq!(result, expected); + + Ok(()) + } + #[test] fn test_schema_with_structs() -> Result<()> { let record = r#"{ From ca55745b7a030b54ab34e0bdcb6c8da6bf8a5aff Mon Sep 17 00:00:00 2001 From: dor-bernstein <160585775+dor-bernstein@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:38:52 +0300 Subject: [PATCH 2/3] Address CR: remove unused imports, fix comment wording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop Error and ErrorKind from iceberg import (unused after removing the FeatureUnsupported arm) — fixes clippy unused_imports warning - Update comment to reference apache/iceberg AWS Glue module instead of Spark's integration Co-Authored-By: Claude Sonnet 4.6 --- crates/catalog/glue/src/schema.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/catalog/glue/src/schema.rs b/crates/catalog/glue/src/schema.rs index f584fbc4ce..d9779d27da 100644 --- a/crates/catalog/glue/src/schema.rs +++ b/crates/catalog/glue/src/schema.rs @@ -26,7 +26,7 @@ use std::collections::HashMap; use aws_sdk_glue::types::Column; use iceberg::spec::{PrimitiveType, SchemaVisitor, TableMetadata, VariantType, visit_schema}; -use iceberg::{Error, ErrorKind, Result}; +use iceberg::Result; use crate::error::from_aws_build_error; @@ -158,7 +158,7 @@ impl SchemaVisitor for GlueSchemaBuilder { PrimitiveType::Timestamp => "timestamp".to_string(), PrimitiveType::TimestampNs => "timestamp_ns".to_string(), // Glue has no timezone-aware timestamp type; map to the equivalent non-tz type, - // matching the behavior of Spark's Glue catalog integration. + // matching the behavior of the AWS Glue catalog module in apache/iceberg. PrimitiveType::Timestamptz => "timestamp".to_string(), PrimitiveType::TimestamptzNs => "timestamp_ns".to_string(), PrimitiveType::Time | PrimitiveType::String | PrimitiveType::Uuid => { From 7c0366324f26e0b982356430c1ea0950843051f2 Mon Sep 17 00:00:00 2001 From: dor-bernstein <160585775+dor-bernstein@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:53:28 +0300 Subject: [PATCH 3/3] fix(glue): fix import order and fold timestamptz coverage into existing test - run cargo fmt so `use iceberg::Result` sorts before `iceberg::spec` (CI lint) - merge the timestamptz case into test_schema_with_simple_fields per review - cover the nanosecond variants in a small V3 test, since timestamp_ns / timestamptz_ns are rejected by the V2 metadata validation Co-Authored-By: Claude Opus 5 (1M context) --- crates/catalog/glue/src/schema.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/catalog/glue/src/schema.rs b/crates/catalog/glue/src/schema.rs index d9779d27da..113a27878d 100644 --- a/crates/catalog/glue/src/schema.rs +++ b/crates/catalog/glue/src/schema.rs @@ -25,8 +25,8 @@ pub(crate) const ICEBERG_FIELD_CURRENT: &str = "iceberg.field.current"; use std::collections::HashMap; use aws_sdk_glue::types::Column; -use iceberg::spec::{PrimitiveType, SchemaVisitor, TableMetadata, VariantType, visit_schema}; use iceberg::Result; +use iceberg::spec::{PrimitiveType, SchemaVisitor, TableMetadata, VariantType, visit_schema}; use crate::error::from_aws_build_error; @@ -310,6 +310,12 @@ mod tests { "name": "c13", "required": true, "type": "binary" + }, + { + "id": 14, + "name": "c14", + "required": true, + "type": "timestamptz" } ] }"#; @@ -333,6 +339,7 @@ mod tests { create_column("c11", "string", "11", false)?, create_column("c12", "binary", "12", false)?, create_column("c13", "binary", "13", false)?, + create_column("c14", "timestamp", "14", false)?, ]; assert_eq!(result, expected); @@ -341,20 +348,20 @@ mod tests { } #[test] - fn test_schema_with_timestamptz_fields() -> Result<()> { + fn test_schema_with_nanosecond_timestamps() -> Result<()> { let record = r#"{ "type": "struct", "schema-id": 1, "fields": [ { "id": 1, - "name": "ts_tz", + "name": "c1", "required": true, - "type": "timestamptz" + "type": "timestamp_ns" }, { "id": 2, - "name": "ts_tz_ns", + "name": "c2", "required": true, "type": "timestamptz_ns" } @@ -362,13 +369,14 @@ mod tests { }"#; let schema = serde_json::from_str::(record)?; - let metadata = create_metadata(schema)?; + // Nanosecond timestamps are only valid from format version 3 onwards. + let metadata = create_metadata_with_format_version(schema, FormatVersion::V3)?; let result = GlueSchemaBuilder::from_iceberg(&metadata)?.build(); let expected = vec![ - create_column("ts_tz", "timestamp", "1", false)?, - create_column("ts_tz_ns", "timestamp_ns", "2", false)?, + create_column("c1", "timestamp_ns", "1", false)?, + create_column("c2", "timestamp_ns", "2", false)?, ]; assert_eq!(result, expected);