[WIP] Bump Apache DF to 55

Cargo.toml

@@ -102,16 +102,16 @@ anyhow = "1.0.100"
 arbitrary = "1.3.2"
 arc-swap = "1.9"
 arcref = "0.2.0"
-arrow-arith = "58.3"
-arrow-array = "58.3"
-arrow-buffer = "58.3"
-arrow-cast = "58.3"
-arrow-data = "58.3"
-arrow-ipc = "58.3"
-arrow-row = "58.3"
-arrow-schema = "58.3"
-arrow-select = "58.3"
-arrow-string = "58.3"
+arrow-arith = "59.2"
+arrow-array = "59.2"
+arrow-buffer = "59.2"
+arrow-cast = "59.2"
+arrow-data = "59.2"
+arrow-ipc = "59.2"
+arrow-row = "59.2"
+arrow-schema = "59.2"
+arrow-select = "59.2"
+arrow-string = "59.2"
 async-fs = "2.2.0"
 async-lock = "3.4"
 async-stream = "0.3.6"
@@ -139,21 +139,23 @@ cudarc = { version = "0.19.0", features = [
 ] }
 custom-labels = "0.4.4"
 dashmap = "6.1.0"
-datafusion = { version = "54", default-features = false, features = ["sql"] }
-datafusion-catalog = { version = "54" }
-datafusion-common = { version = "54" }
-datafusion-common-runtime = { version = "54" }
-datafusion-datasource = { version = "54", default-features = false }
-datafusion-execution = { version = "54" }
-datafusion-expr = { version = "54" }
-datafusion-functions = { version = "54" }
-datafusion-functions-nested = { version = "54" }
-datafusion-physical-expr = { version = "54" }
-datafusion-physical-expr-adapter = { version = "54" }
-datafusion-physical-expr-common = { version = "54" }
-datafusion-physical-plan = { version = "54" }
-datafusion-pruning = { version = "54" }
-datafusion-sqllogictest = { version = "54" }
+datafusion = { version = "55.0.0", default-features = false, features = [
+    "sql",
+] }
+datafusion-catalog = { version = "55.0.0" }
+datafusion-common = { version = "55.0.0" }
+datafusion-common-runtime = { version = "55.0.0" }
+datafusion-datasource = { version = "55.0.0", default-features = false }
+datafusion-execution = { version = "55.0.0" }
+datafusion-expr = { version = "55.0.0" }
+datafusion-functions = { version = "55.0.0" }
+datafusion-functions-nested = { version = "55.0.0" }
+datafusion-physical-expr = { version = "55.0.0" }
+datafusion-physical-expr-adapter = { version = "55.0.0" }
+datafusion-physical-expr-common = { version = "55.0.0" }
+datafusion-physical-plan = { version = "55.0.0" }
+datafusion-pruning = { version = "55.0.0" }
+datafusion-sqllogictest = { version = "55.0.0" }
 divan = { package = "codspeed-divan-compat", version = "5.0.0" }
 enum-iterator = "2.0.0"
 env_logger = "0.11"
@@ -205,9 +207,9 @@ opentelemetry = "0.32.0"
 opentelemetry-otlp = "0.32.0"
 opentelemetry_sdk = "0.32.0"
 parking_lot = { version = "0.12.3", features = ["nightly"] }
-parquet = "58.3"
-parquet-variant = "58.3"
-parquet-variant-compute = "58.3"
+parquet = "59.2"
+parquet-variant = "59.2"
+parquet-variant-compute = "59.2"
 paste = "1.0.15"
 pco = "1.0.1"
 percent-encoding = "2.3.2"
@@ -276,8 +278,8 @@ tokio-stream = "0.1.17"
 tokio-util = "0.7.17"
 vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" }
 # Pull these into non-public crates to support DF 58
-tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
-tpchgen-arrow = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
+tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" }
+tpchgen-arrow = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" }
 tracing = { version = "0.1.41", default-features = false }
 tracing-perfetto = "0.1.5"
 tracing-subscriber = "0.3"
@@ -442,3 +444,43 @@ incremental = false
 # This improved build times significantly for default common cases that we use locally
 [profile.dev.package.vortex-fastlanes]
 debug = false
+
+[patch.crates-io]
+datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-catalog = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-catalog-listing = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-common-runtime = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-datasource = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-datasource-arrow = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-datasource-avro = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-datasource-csv = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-datasource-json = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-datasource-parquet = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-doc = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-execution = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-expr = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-expr-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-functions = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-functions-nested = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-functions-table = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-functions-window = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-functions-window-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-macros = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-optimizer = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-physical-expr = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-physical-optimizer = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-physical-plan = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-pruning = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-session = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-spark = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-sql = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+datafusion-sqllogictest = { git = "https://github.com/apache/datafusion", branch = "branch-55" }
+geoarrow = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
+geoarrow-array = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
+geoarrow-cast = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
+geoarrow-schema = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }

benchmarks/datafusion-bench/src/lib.rs

@@ -11,9 +11,7 @@ use datafusion::datasource::file_format::csv::CsvFormat;
 use datafusion::datasource::file_format::parquet::ParquetFormat;
 use datafusion::datasource::provider::DefaultTableFactory;
 use datafusion::execution::SessionStateBuilder;
-use datafusion::execution::cache::DefaultListFilesCache;
 use datafusion::execution::cache::cache_manager::CacheManagerConfig;
-use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache;
 use datafusion::execution::runtime_env::RuntimeEnvBuilder;
 use datafusion::prelude::SessionConfig;
 use datafusion::prelude::SessionContext;
@@ -33,12 +31,7 @@ use vortex_datafusion::VortexTableOptions;
 pub fn get_session_context() -> SessionContext {
     let mut rt_builder = RuntimeEnvBuilder::new();
 
-    let file_static_cache = Arc::new(DefaultFileStatisticsCache::default());
-    let list_file_cache = Arc::new(DefaultListFilesCache::default());
-    let cache_config = CacheManagerConfig::default()
-        .with_file_statistics_cache(Some(file_static_cache))
-        .with_list_files_cache(Some(list_file_cache));
-    rt_builder = rt_builder.with_cache_manager(cache_config);
+    rt_builder = rt_builder.with_cache_manager(CacheManagerConfig::default());
 
     let rt = rt_builder
         .build_arc()

benchmarks/datafusion-bench/src/main.rs

@@ -247,8 +247,7 @@ async fn register_benchmark_tables<B: Benchmark + ?Sized>(
             let table_url = ListingTableUrl::try_new(benchmark_base.clone(), pattern)?
                 .with_table_ref(table_ref.clone());
 
-            let listing_options = ListingOptions::new(Arc::clone(&file_format))
-                .with_session_config_options(session.state().config());
+            let listing_options = ListingOptions::new(Arc::clone(&file_format));
             let mut config =
                 ListingTableConfig::new(table_url).with_listing_options(listing_options);
 

benchmarks/lance-bench/src/random_access.rs

@@ -114,8 +114,7 @@ impl RandomAccessor for LanceRandomAccessor {
     }
 
     async fn take(&self, indices: &[u64]) -> anyhow::Result<RandomAccessorRet> {
-        Ok(RandomAccessorRet::RecordBatch(
-            self.dataset.take(indices, self.projection.clone()).await?,
-        ))
+        let batch = self.dataset.take(indices, self.projection.clone()).await?;
+        Ok(RandomAccessorRet::Native(Box::new(batch)))
     }
 }

encodings/parquet-variant/src/array.rs

@@ -137,15 +137,15 @@ impl ParquetVariant {
                 Validity::NonNullable
             });
         let metadata = session
-            .from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_field()), false)?;
+            .from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_column()), false)?;
 
-        let value = arrow_variant
-            .value_field()
-            .map(|v| session.from_arrow_array(ArrowArrayRef::clone(v), value_nullable))
-            .transpose()?;
+        let value = Some(session.from_arrow_array(
+            ArrowArrayRef::clone(arrow_variant.value_column()),
+            value_nullable,
+        )?);
 
         let typed_value = arrow_variant
-            .typed_value_field()
+            .typed_value_column()
             .map(|tv| session.from_arrow_array(ArrowArrayRef::clone(tv), typed_value_nullable))
             .transpose()?;
         ParquetVariant::try_new(validity, metadata, value, typed_value).map(IntoArray::into_array)
@@ -692,7 +692,7 @@ mod tests {
     }
 
     #[test]
-    fn test_arrow_variant_import_typed_value_only_preserves_storage() -> VortexResult<()> {
+    fn test_arrow_variant_import_typed_value_only_normalizes_storage() -> VortexResult<()> {
         let metadata = binary_view_array([b"\x01\x00", b"\x01\x00", b"\x01\x00"]);
         let typed_value: ArrowArrayRef = Arc::new(Int32Array::from(vec![10, 20, 30]));
 
@@ -711,17 +711,21 @@ mod tests {
         let parquet_array = vortex_arr
             .as_opt::<ParquetVariant>()
             .ok_or_else(|| vortex_err!("expected parquet variant array"))?;
-        assert!(parquet_array.value().is_none());
+        let mut ctx = SESSION.create_execution_ctx();
+        let value = parquet_array
+            .value()
+            .ok_or_else(|| vortex_err!("expected synthesized value child"))?;
+        assert!(value.all_invalid(&mut ctx)?);
 
         let typed_value = parquet_array
             .typed_value()
             .ok_or_else(|| vortex_err!("expected typed_value child"))?
             .clone()
-            .execute::<PrimitiveArray>(&mut SESSION.create_execution_ctx())?;
+            .execute::<PrimitiveArray>(&mut ctx)?;
         assert_arrays_eq!(
             typed_value,
-            PrimitiveArray::from_iter([10i32, 20, 30]),
-            &mut SESSION.create_execution_ctx()
+            PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]),
+            &mut ctx
         );
         Ok(())
     }

encodings/parquet-variant/src/arrow.rs

@@ -715,7 +715,6 @@ mod tests {
 
         let actual = session.arrow().from_arrow_array(exported, &field)?;
 
-        assert_arrays_eq!(actual, expected, &mut ctx);
-        Ok(())
+        assert_variant_scalars_eq(&actual, &expected, &session)
     }
 }

encodings/parquet-variant/src/kernel.rs

@@ -841,17 +841,13 @@ mod tests {
 
         let metadata = SESSION
             .arrow()
-            .from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_field()), false)?;
-        let value = arrow_variant
-            .value_field()
-            .map(|value| {
-                SESSION
-                    .arrow()
-                    .from_arrow_array(ArrowArrayRef::clone(value), value_nullable)
-            })
-            .transpose()?;
+            .from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_column()), false)?;
+        let value = Some(SESSION.arrow().from_arrow_array(
+            ArrowArrayRef::clone(arrow_variant.value_column()),
+            value_nullable,
+        )?);
         let typed_value = arrow_variant
-            .typed_value_field()
+            .typed_value_column()
             .map(|typed_value| {
                 SESSION
                     .arrow()

vortex-arrow/src/dtype.rs

@@ -335,13 +335,12 @@ pub(crate) fn to_data_type_naive(dtype: &DType) -> VortexResult<DataType> {
             let scale = dt.scale();
 
             match precision {
-                // This code is commented out until DataFusion improves its support for smaller decimals.
-                // // DECIMAL32_MAX_PRECISION
-                // 0..=9 => DataType::Decimal32(precision, scale),
-                // // DECIMAL64_MAX_PRECISION
-                // 10..=18 => DataType::Decimal64(precision, scale),
+                // DECIMAL32_MAX_PRECISION
+                0..=9 => DataType::Decimal32(precision, scale),
+                // DECIMAL64_MAX_PRECISION
+                10..=18 => DataType::Decimal64(precision, scale),
                 // DECIMAL128_MAX_PRECISION
-                0..=38 => DataType::Decimal128(precision, scale),
+                19..=38 => DataType::Decimal128(precision, scale),
                 // DECIMAL256_MAX_PRECISION
                 39.. => DataType::Decimal256(precision, scale),
             }
@@ -601,7 +600,11 @@ mod test {
     }
 
     #[rstest]
-    #[case(1, DataType::Decimal128(1, 0))]
+    #[case(1, DataType::Decimal32(1, 0))]
+    #[case(9, DataType::Decimal32(9, 0))]
+    #[case(10, DataType::Decimal64(10, 0))]
+    #[case(18, DataType::Decimal64(18, 0))]
+    #[case(19, DataType::Decimal128(19, 0))]
     #[case(38, DataType::Decimal128(38, 0))]
     #[case(39, DataType::Decimal256(39, 0))]
     #[case(76, DataType::Decimal256(76, 0))]