chore: Update to latest main on upstream DataFusion (#1662)

* Update to main on the upstream datafusion to prepare for DF55 release

* Deprecate singular external table location

AI Disclosure: This code was written in part by an AI agent.:

* Fix tests for updated DataFusion behavior

AI Disclosure: This code was written in part by an AI agent.:

* create table requires a location

* Remove unused Python token from Substrait deserialization

AI Disclosure: This code was written in part by an AI agent.:

* test: cover external table locations binding

AI Disclosure: This code was written in part by an AI agent.:

Author: timsaucer

Cargo.lock

@@ -32,24 +32,24 @@ resolver = "3"
 
 [workspace.dependencies]
 tokio = { version = "1.52" }
-pyo3 = { version = "0.28" }
-pyo3-async-runtimes = { version = "0.28" }
+pyo3 = { version = "0.29" }
+pyo3-async-runtimes = { version = "0.29" }
 pyo3-log = "0.13.3"
 chrono = { version = "0.4", default-features = false }
-arrow = { version = "58" }
-arrow-array = { version = "58" }
-arrow-schema = { version = "58" }
-arrow-select = { version = "58" }
-datafusion = { version = "54" }
-datafusion-substrait = { version = "54" }
-datafusion-proto = { version = "54" }
-datafusion-ffi = { version = "54" }
-datafusion-catalog = { version = "54", default-features = false }
-datafusion-common = { version = "54", default-features = false }
-datafusion-functions-aggregate = { version = "54" }
-datafusion-functions-window = { version = "54" }
-datafusion-spark = { version = "54" }
-datafusion-expr = { version = "54" }
+arrow = { version = "59" }
+arrow-array = { version = "59" }
+arrow-schema = { version = "59" }
+arrow-select = { version = "59" }
+datafusion = { version = "54.1.0" }
+datafusion-substrait = { version = "54.1.0" }
+datafusion-proto = { version = "54.1.0" }
+datafusion-ffi = { version = "54.1.0" }
+datafusion-catalog = { version = "54.1.0", default-features = false }
+datafusion-common = { version = "54.1.0", default-features = false }
+datafusion-functions-aggregate = { version = "54.1.0" }
+datafusion-functions-window = { version = "54.1.0" }
+datafusion-spark = { version = "54.1.0" }
+datafusion-expr = { version = "54.1.0" }
 prost = "0.14.3"
 serde_json = "1"
 uuid = { version = "1.23" }
@@ -62,7 +62,7 @@ url = "2"
 log = "0.4.29"
 parking_lot = "0.12"
 prost-types = "0.14.3"                                                # keep in line with `datafusion-substrait`
-pyo3-build-config = "0.28"
+pyo3-build-config = "0.29"
 datafusion-python-util = { path = "crates/util", version = "54.0.0" }
 
 [profile.release]
@@ -72,3 +72,13 @@ codegen-units = 2
 # We cannot publish to crates.io with any patches in the below section. Developers
 # must remove any entries in this section before creating a release candidate.
 [patch.crates-io]
+datafusion = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-substrait = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-proto = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-ffi = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-catalog = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-common = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-functions-window = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-expr = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }

Cargo.toml

@@ -53,7 +53,7 @@ datafusion = { workspace = true, features = ["avro", "unicode_expressions"] }
 datafusion-substrait = { workspace = true, optional = true }
 datafusion-proto = { workspace = true }
 datafusion-ffi = { workspace = true }
-datafusion-spark = { workspace = true }
+datafusion-spark = { workspace = true, features = ["core"] }
 prost = { workspace = true } # keep in line with `datafusion-substrait`
 serde_json = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }

crates/core/Cargo.toml

@@ -63,10 +63,10 @@ impl PyArrowArrayExportable {
         };
 
         let ffi_schema = FFI_ArrowSchema::try_from(&field)?;
-        let schema_capsule = PyCapsule::new(py, ffi_schema, Some(cr"arrow_schema".into()))?;
+        let schema_capsule = PyCapsule::new_with_value(py, ffi_schema, cr"arrow_schema")?;
 
         let ffi_array = FFI_ArrowArray::new(&self.array.to_data());
-        let array_capsule = PyCapsule::new(py, ffi_array, Some(cr"arrow_array".into()))?;
+        let array_capsule = PyCapsule::new_with_value(py, ffi_array, cr"arrow_array")?;
 
         Ok((schema_capsule, array_capsule))
     }

crates/core/src/array.rs

@@ -100,9 +100,13 @@ use datafusion::logical_expr::{
     TypeSignature, Volatility, WindowUDF, WindowUDFImpl,
 };
 use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
+use datafusion::physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion_proto::logical_plan::{DefaultLogicalExtensionCodec, LogicalExtensionCodec};
-use datafusion_proto::physical_plan::{DefaultPhysicalExtensionCodec, PhysicalExtensionCodec};
+use datafusion_proto::physical_plan::{
+    DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalProtoConverterExtension,
+};
 use pyo3::prelude::*;
 use pyo3::sync::PyOnceLock;
 use pyo3::types::{PyBytes, PyTuple};
@@ -483,12 +487,18 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec {
         buf: &[u8],
         inputs: &[Arc<dyn ExecutionPlan>],
         ctx: &TaskContext,
+        proto_converter: &dyn PhysicalProtoConverterExtension,
     ) -> Result<Arc<dyn ExecutionPlan>> {
-        self.inner.try_decode(buf, inputs, ctx)
+        self.inner.try_decode(buf, inputs, ctx, proto_converter)
     }
 
-    fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
-        self.inner.try_encode(node, buf)
+    fn try_encode(
+        &self,
+        node: Arc<dyn ExecutionPlan>,
+        buf: &mut Vec<u8>,
+        proto_converter: &dyn PhysicalProtoConverterExtension,
+    ) -> Result<()> {
+        self.inner.try_encode(node, buf, proto_converter)
     }
 
     fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec<u8>) -> Result<()> {
@@ -509,16 +519,22 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec {
         self.inner.try_decode_udf(name, buf)
     }
 
-    fn try_encode_expr(&self, node: &Arc<dyn PhysicalExpr>, buf: &mut Vec<u8>) -> Result<()> {
-        self.inner.try_encode_expr(node, buf)
+    fn try_encode_expr(
+        &self,
+        node: &Arc<dyn PhysicalExpr>,
+        buf: &mut Vec<u8>,
+        ctx: &PhysicalExprEncodeCtx<'_>,
+    ) -> Result<()> {
+        self.inner.try_encode_expr(node, buf, ctx)
     }
 
     fn try_decode_expr(
         &self,
         buf: &[u8],
         inputs: &[Arc<dyn PhysicalExpr>],
+        ctx: &PhysicalExprDecodeCtx<'_>,
     ) -> Result<Arc<dyn PhysicalExpr>> {
-        self.inner.try_decode_expr(buf, inputs)
+        self.inner.try_decode_expr(buf, inputs, ctx)
     }
 
     fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec<u8>) -> Result<()> {

crates/core/src/codec.rs

@@ -1371,12 +1371,10 @@ impl PySessionContext {
         &self,
         py: Python<'py>,
     ) -> PyResult<Bound<'py, PyCapsule>> {
-        let name = cr"datafusion_task_context_provider".into();
-
         let ctx_provider = Arc::clone(&self.ctx) as Arc<dyn TaskContextProvider>;
         let ffi_ctx_provider = FFI_TaskContextProvider::from(&ctx_provider);
 
-        PyCapsule::new(py, ffi_ctx_provider, Some(name))
+        PyCapsule::new_with_value(py, ffi_ctx_provider, cr"datafusion_task_context_provider")
     }
 
     pub fn __datafusion_logical_extension_codec__<'py>(

crates/core/src/context.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use std::collections::HashMap;
-use std::ffi::{CStr, CString};
+use std::ffi::CStr;
 use std::ptr::NonNull;
 use std::str::FromStr;
 use std::sync::Arc;
@@ -1237,8 +1237,7 @@ impl PyDataFrame {
         // destructor provided by PyO3 will drop the stream unless ownership is
         // transferred to PyArrow during import.
         let stream = FFI_ArrowArrayStream::new(reader);
-        let name = CString::new(ARROW_ARRAY_STREAM_NAME.to_bytes()).unwrap();
-        let capsule = PyCapsule::new(py, stream, Some(name))?;
+        let capsule = PyCapsule::new_with_value(py, stream, ARROW_ARRAY_STREAM_NAME)?;
         Ok(capsule)
     }
 
@@ -1317,7 +1316,8 @@ impl PyDataFrame {
             None => Vec::new(), // Empty vector means fill null for all columns
         };
 
-        let df = self.df.as_ref().clone().fill_null(scalar_value.0, cols)?;
+        let cols = cols.iter().map(String::as_str).collect::<Vec<_>>();
+        let df = self.df.as_ref().fill_null(&scalar_value.0, &cols)?;
         Ok(Self::new(df))
     }
 }

crates/core/src/dataframe.rs

@@ -88,7 +88,7 @@ impl PyCreateExternalTable {
         let create = CreateExternalTable {
             schema: Arc::new(schema.into()),
             name: name.into(),
-            location,
+            locations: vec![location],
             file_type,
             table_partition_cols,
             if_not_exists,
@@ -118,8 +118,8 @@ impl PyCreateExternalTable {
         Ok(self.create.name.to_string())
     }
 
-    pub fn location(&self) -> String {
-        self.create.location.clone()
+    pub fn locations(&self) -> Vec<String> {
+        self.create.locations.clone()
     }
 
     pub fn file_type(&self) -> String {

crates/core/src/expr/create_external_table.rs

@@ -18,6 +18,7 @@
 use datafusion::logical_expr::dml::InsertOp;
 use datafusion::logical_expr::{DmlStatement, WriteOp};
 use pyo3::IntoPyObjectExt;
+use pyo3::exceptions::PyNotImplementedError;
 use pyo3::prelude::*;
 
 use super::logical_node::LogicalNode;
@@ -71,8 +72,8 @@ impl PyDmlStatement {
         })
     }
 
-    pub fn op(&self) -> PyWriteOp {
-        self.dml.op.clone().into()
+    pub fn op(&self) -> PyResult<PyWriteOp> {
+        self.dml.op.clone().try_into()
     }
 
     pub fn input(&self) -> PyLogicalPlan {
@@ -112,16 +113,21 @@ pub enum PyWriteOp {
     Truncate,
 }
 
-impl From<WriteOp> for PyWriteOp {
-    fn from(write_op: WriteOp) -> Self {
+impl TryFrom<WriteOp> for PyWriteOp {
+    type Error = PyErr;
+
+    fn try_from(write_op: WriteOp) -> Result<Self, Self::Error> {
         match write_op {
-            WriteOp::Insert(InsertOp::Append) => PyWriteOp::Append,
-            WriteOp::Insert(InsertOp::Overwrite) => PyWriteOp::Overwrite,
-            WriteOp::Insert(InsertOp::Replace) => PyWriteOp::Replace,
-            WriteOp::Update => PyWriteOp::Update,
-            WriteOp::Delete => PyWriteOp::Delete,
-            WriteOp::Ctas => PyWriteOp::Ctas,
-            WriteOp::Truncate => PyWriteOp::Truncate,
+            WriteOp::Insert(InsertOp::Append) => Ok(PyWriteOp::Append),
+            WriteOp::Insert(InsertOp::Overwrite) => Ok(PyWriteOp::Overwrite),
+            WriteOp::Insert(InsertOp::Replace) => Ok(PyWriteOp::Replace),
+            WriteOp::Update => Ok(PyWriteOp::Update),
+            WriteOp::Delete => Ok(PyWriteOp::Delete),
+            WriteOp::Ctas => Ok(PyWriteOp::Ctas),
+            WriteOp::Truncate => Ok(PyWriteOp::Truncate),
+            unsupported => Err(PyNotImplementedError::new_err(format!(
+                "DataFusion write operation {unsupported:?} is not supported"
+            ))),
         }
     }
 }

crates/core/src/expr/dml.rs

@@ -18,9 +18,8 @@
 use std::fmt::{self, Display, Formatter};
 use std::sync::Arc;
 
-use datafusion::common::SchemaReference;
+use datafusion::common::{SchemaReference, TableReference};
 use datafusion::logical_expr::DropCatalogSchema;
-use datafusion::sql::TableReference;
 use pyo3::IntoPyObjectExt;
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;