chore(deps): bump substrait from 0.62 to 0.63.0

datafusion/expr/src/logical_plan/plan.rs

@@ -4456,49 +4456,49 @@ mod tests {
         [
           {
             "Plan": {
+              "Node Type": "Projection",
               "Expressions": [
                 "employee_csv.id"
               ],
-              "Node Type": "Projection",
-              "Output": [
-                "id"
-              ],
               "Plans": [
                 {
-                  "Condition": "employee_csv.state IN (<subquery>)",
                   "Node Type": "Filter",
-                  "Output": [
-                    "id",
-                    "state"
-                  ],
+                  "Condition": "employee_csv.state IN (<subquery>)",
                   "Plans": [
                     {
                       "Node Type": "Subquery",
-                      "Output": [
-                        "state"
-                      ],
                       "Plans": [
                         {
                           "Node Type": "TableScan",
+                          "Relation Name": "employee_csv",
+                          "Plans": [],
                           "Output": [
                             "state"
-                          ],
-                          "Plans": [],
-                          "Relation Name": "employee_csv"
+                          ]
                         }
+                      ],
+                      "Output": [
+                        "state"
                       ]
                     },
                     {
                       "Node Type": "TableScan",
+                      "Relation Name": "employee_csv",
+                      "Plans": [],
                       "Output": [
                         "id",
                         "state"
-                      ],
-                      "Plans": [],
-                      "Relation Name": "employee_csv"
+                      ]
                     }
+                  ],
+                  "Output": [
+                    "id",
+                    "state"
                   ]
                 }
+              ],
+              "Output": [
+                "id"
               ]
             }
           }

datafusion/sqllogictest/test_files/explain.slt

@@ -646,11 +646,11 @@ logical_plan
 02)--{
 03)----"Plan": {
 04)------"Node Type": "Values",
-05)------"Output": [
-06)--------"column1"
-07)------],
-08)------"Plans": [],
-09)------"Values": "(Int64(1))"
+05)------"Values": "(Int64(1))",
+06)------"Plans": [],
+07)------"Output": [
+08)--------"column1"
+09)------]
 10)----}
 11)--}
 12)]

datafusion/substrait/Cargo.toml

@@ -44,7 +44,7 @@ object_store = { workspace = true }
 # We need to match the version in substrait, so we don't use the workspace version here
 pbjson-types = { version = "0.8.0" }
 prost = { workspace = true }
-substrait = { version = "0.62", features = ["serde"] }
+substrait = { version = "0.63.0", features = ["serde"] }
 url = { workspace = true }
 tokio = { workspace = true, features = ["fs"] }
 

datafusion/substrait/src/extensions.rs

@@ -113,14 +113,10 @@ impl TryFrom<&Vec<SimpleExtensionDeclaration>> for Extensions {
 }
 
 impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
-    // Silence deprecation warnings for `extension_uri_reference` during the uri -> urn migration
-    // See: https://github.com/substrait-io/substrait/issues/856
-    #[expect(deprecated)]
     fn from(val: Extensions) -> Vec<SimpleExtensionDeclaration> {
         let mut extensions = vec![];
         for (f_anchor, f_name) in val.functions {
             let function_extension = ExtensionFunction {
-                extension_uri_reference: u32::MAX,
                 extension_urn_reference: u32::MAX,
                 function_anchor: f_anchor,
                 name: f_name,
@@ -133,7 +129,6 @@ impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
 
         for (t_anchor, t_name) in val.types {
             let type_extension = ExtensionType {
-                extension_uri_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545
                 extension_urn_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545
                 type_anchor: t_anchor,
                 name: t_name,
@@ -146,8 +141,7 @@ impl From<Extensions> for Vec<SimpleExtensionDeclaration> {
 
         for (tv_anchor, tv_name) in val.type_variations {
             let type_variation_extension = ExtensionTypeVariation {
-                extension_uri_reference: u32::MAX, // We don't register proper extension URIs yet
-                extension_urn_reference: u32::MAX, // We don't register proper extension URIs yet
+                extension_urn_reference: u32::MAX, // We don't register proper extension URNs yet
                 type_variation_anchor: tv_anchor,
                 name: tv_name,
             };

datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs

@@ -56,6 +56,9 @@ pub(crate) fn from_substrait_field_reference(
                     Some(RootType::Expression(_)) => not_impl_err!(
                         "Expression root type in field reference is not supported"
                     ),
+                    Some(RootType::LambdaParameterReference(_)) => not_impl_err!(
+                        "Lambda parameter reference in field reference is not yet supported"
+                    ),
                 }
             }
             _ => not_impl_err!(

datafusion/substrait/src/logical_plan/consumer/expr/literal.rs

@@ -43,7 +43,7 @@ use prost::Message;
 use std::sync::Arc;
 use substrait::proto;
 use substrait::proto::expression::Literal;
-use substrait::proto::expression::literal::user_defined::Val;
+use substrait::proto::expression::literal::user_defined::{TypeAnchorType, Val};
 use substrait::proto::expression::literal::{
     IntervalCompound, IntervalDayToSecond, IntervalYearToMonth, LiteralType,
     interval_day_to_second,
@@ -474,11 +474,17 @@ pub(crate) fn from_substrait_literal(
                     )))
                 };
 
-            if let Some(name) = consumer
-                .get_extensions()
-                .types
-                .get(&user_defined.type_reference)
-            {
+            let type_ref = match user_defined.type_anchor_type {
+                Some(TypeAnchorType::TypeReference(ref_val)) => ref_val,
+                Some(TypeAnchorType::TypeAliasReference(_)) => {
+                    return not_impl_err!(
+                        "Type alias references in user-defined literals are not yet supported"
+                    );
+                }
+                None => 0,
+            };
+
+            if let Some(name) = consumer.get_extensions().types.get(&type_ref) {
                 match name.as_ref() {
                     FLOAT_16_TYPE_NAME => {
                         // Rules for encoding fp16 Substrait literals are defined as part of Arrow here:
@@ -518,14 +524,14 @@ pub(crate) fn from_substrait_literal(
                     _ => {
                         return not_impl_err!(
                             "Unsupported Substrait user defined type with ref {} and name {}",
-                            user_defined.type_reference,
+                            type_ref,
                             name
                         );
                     }
                 }
             } else {
                 #[expect(deprecated)]
-                match user_defined.type_reference {
+                match type_ref {
                     // Kept for backwards compatibility, producers should useIntervalYearToMonth instead
                     INTERVAL_YEAR_MONTH_TYPE_REF => {
                         let Some(Val::Value(raw_val)) = user_defined.val.as_ref() else {
@@ -568,7 +574,7 @@ pub(crate) fn from_substrait_literal(
                     _ => {
                         return not_impl_err!(
                             "Unsupported Substrait user defined type literal with ref {}",
-                            user_defined.type_reference
+                            type_ref
                         );
                     }
                 }

datafusion/substrait/src/logical_plan/consumer/expr/mod.rs

@@ -93,6 +93,9 @@ pub async fn from_substrait_rex(
             RexType::DynamicParameter(expr) => {
                 consumer.consume_dynamic_parameter(expr, input_schema).await
             }
+            RexType::Lambda(_) | RexType::LambdaInvocation(_) => {
+                not_impl_err!("Lambda expressions are not yet supported")
+            }
         },
         None => substrait_err!("Expression must set rex_type: {expression:?}"),
     }

datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs

@@ -141,7 +141,15 @@ use substrait::proto::{
 ///
 ///     // and user-defined literals
 ///     fn consume_user_defined_literal(&self, literal: &proto::expression::literal::UserDefined) -> Result<ScalarValue> {
-///         let type_string = self.extensions.types.get(&literal.type_reference).unwrap();
+///         // extract type_reference from the new TypeAnchorType oneof
+///         let type_ref = match literal.type_anchor_type {
+///             Some(proto::expression::literal::user_defined::TypeAnchorType::TypeReference(r)) => r,
+///             Some(proto::expression::literal::user_defined::TypeAnchorType::TypeAliasReference(_)) => {
+///                 return not_impl_err!("Type alias references are not yet supported")
+///             }
+///             None => 0,
+///         };
+///         let type_string = self.extensions.types.get(&type_ref).unwrap();
 ///         match type_string.as_str() {
 ///             "u!foo" => not_impl_err!("handle foo conversion"),
 ///             "u!bar" => not_impl_err!("handle bar conversion"),
@@ -444,10 +452,22 @@ pub trait SubstraitConsumer: Send + Sync + Sized {
         &self,
         user_defined_literal: &proto::expression::literal::UserDefined,
     ) -> datafusion::common::Result<ScalarValue> {
-        substrait_err!(
-            "Missing handler for user-defined literals {}",
-            user_defined_literal.type_reference
-        )
+        let type_ref = match user_defined_literal.type_anchor_type {
+            Some(
+                proto::expression::literal::user_defined::TypeAnchorType::TypeReference(
+                    ref_val,
+                ),
+            ) => ref_val,
+            Some(
+                proto::expression::literal::user_defined::TypeAnchorType::TypeAliasReference(_),
+            ) => {
+                return not_impl_err!(
+                    "Type alias references in user-defined literals are not yet supported"
+                )
+            }
+            None => 0,
+        };
+        substrait_err!("Missing handler for user-defined literals {}", type_ref)
     }
 }
 

datafusion/substrait/src/logical_plan/producer/expr/literal.rs

@@ -117,7 +117,7 @@ pub(crate) fn to_substrait_literal(
             (
                 LiteralType::UserDefined(
                     substrait::proto::expression::literal::UserDefined {
-                        type_reference: type_anchor,
+                        type_anchor_type: Some(substrait::proto::expression::literal::user_defined::TypeAnchorType::TypeReference(type_anchor)),
                         type_parameters: vec![],
                         val: Some(substrait::proto::expression::literal::user_defined::Val::Value(
                             pbjson_types::Any {

datafusion/substrait/src/logical_plan/producer/expr/mod.rs

@@ -60,9 +60,6 @@ use substrait::version;
 ///
 /// Substrait also requires the input schema of the expressions to be included in the
 /// message.  The field names of the input schema will be serialized.
-// Silence deprecation warnings for `extension_uris` during the uri -> urn migration
-// See: https://github.com/substrait-io/substrait/issues/856
-#[expect(deprecated)]
 pub fn to_substrait_extended_expr(
     exprs: &[(&Expr, &Field)],
     schema: &DFSchemaRef,
@@ -87,7 +84,6 @@ pub fn to_substrait_extended_expr(
     Ok(Box::new(ExtendedExpression {
         advanced_extensions: None,
         expected_type_urls: vec![],
-        extension_uris: vec![],
         extension_urns: vec![],
         extensions: extensions.into(),
         version: Some(version::version_with_producer("datafusion")),

datafusion/substrait/src/logical_plan/producer/plan.rs

@@ -24,9 +24,6 @@ use substrait::proto::{Plan, PlanRel, Rel, RelRoot, plan_rel};
 use substrait::version;
 
 /// Convert DataFusion LogicalPlan to Substrait Plan
-// Silence deprecation warnings for `extension_uris` during the uri -> urn migration
-// See: https://github.com/substrait-io/substrait/issues/856
-#[expect(deprecated)]
 pub fn to_substrait_plan(
     plan: &LogicalPlan,
     state: &SessionState,
@@ -47,7 +44,6 @@ pub fn to_substrait_plan(
     let extensions = producer.get_extensions();
     Ok(Box::new(Plan {
         version: Some(version::version_with_producer("datafusion")),
-        extension_uris: vec![],
         extension_urns: vec![],
         extensions: extensions.into(),
         relations: plan_rels,