From 36b549094d62d07d4190796873b9680a4ff903cc Mon Sep 17 00:00:00 2001 From: Gene Bordegaray Date: Mon, 3 Aug 2026 14:20:12 +0200 Subject: [PATCH] deps: support Arrow 59 --- Cargo.lock | 316 ++++++++---------- Cargo.toml | 28 +- .../python/src/datafusion_table_provider.rs | 8 +- crates/iceberg/Cargo.toml | 2 +- .../src/table/table_provider_factory.rs | 22 +- 5 files changed, 181 insertions(+), 195 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7660914a3..de04a59409 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,9 +227,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" dependencies = [ "arrow-arith", "arrow-array", @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -263,9 +263,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" dependencies = [ "ahash", "arrow-buffer", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84" dependencies = [ "arrow-array", "arrow-buffer", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" dependencies = [ "bytes", "half", @@ -318,9 +318,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" dependencies = [ "arrow-array", "arrow-buffer", @@ -340,9 +340,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" +checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" dependencies = [ "arrow-array", "arrow-cast", @@ -355,9 +355,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" dependencies = [ "arrow-buffer", "arrow-schema", @@ -368,9 +368,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" dependencies = [ "arrow-array", "arrow-buffer", @@ -384,9 +384,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" +checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" dependencies = [ "arrow-array", "arrow-buffer", @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" dependencies = [ "arrow-array", "arrow-buffer", @@ -422,9 +422,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" +checksum = "bf8d967bdece4fa5a0199706730175b3df3448b87e350250a86eb6c22639e445" dependencies = [ "arrow-array", "arrow-data", @@ -434,9 +434,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" dependencies = [ "arrow-array", "arrow-buffer", @@ -447,9 +447,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -458,9 +458,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" dependencies = [ "ahash", "arrow-array", @@ -472,9 +472,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" dependencies = [ "arrow-array", "arrow-buffer", @@ -1483,7 +1483,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -1897,8 +1897,7 @@ dependencies = [ [[package]] name = "datafusion" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-schema", @@ -1934,7 +1933,7 @@ dependencies = [ "flate2", "futures", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1951,8 +1950,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "async-trait", @@ -1966,7 +1964,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1976,8 +1974,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "async-trait", @@ -1991,16 +1988,16 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-cli" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f652cbc0854d1e24c8038e2c7be53ef395768b8713fceaaa00789ae6134a5ee" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "async-trait", @@ -2027,8 +2024,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-ipc", @@ -2039,9 +2035,10 @@ dependencies = [ "hashbrown 0.17.1", "hex", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -2054,8 +2051,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "futures", "log", @@ -2065,8 +2061,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "async-compression", @@ -2086,7 +2081,7 @@ dependencies = [ "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -2101,8 +2096,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-ipc", @@ -2117,7 +2111,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] @@ -2125,8 +2119,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-avro", @@ -2144,8 +2137,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "async-trait", @@ -2167,8 +2159,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "async-trait", @@ -2190,10 +2181,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -2210,7 +2201,7 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2221,18 +2212,17 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" [[package]] name = "datafusion-execution" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -2242,16 +2232,18 @@ dependencies = [ "object_store", "parking_lot", "parquet", + "pin-project-lite", "rand 0.9.5", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-schema", @@ -2264,7 +2256,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -2273,20 +2265,18 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-ffi" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a4a09f24f1387408810acca070207a9cf6c7fc2368122982a8e7671f643d1e" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-schema", @@ -2318,8 +2308,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-buffer", @@ -2336,7 +2325,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -2350,8 +2339,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", @@ -2362,7 +2350,6 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", "log", "num-traits", @@ -2371,8 +2358,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", @@ -2383,8 +2369,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-ord", @@ -2399,7 +2384,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2408,8 +2393,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "async-trait", @@ -2424,8 +2408,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", @@ -2441,8 +2424,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2451,8 +2433,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "datafusion-doc", "quote", @@ -2462,8 +2443,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "chrono", @@ -2472,7 +2452,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "recursive", "regex", @@ -2482,8 +2462,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", @@ -2491,10 +2470,11 @@ dependencies = [ "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -2504,8 +2484,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", @@ -2513,22 +2492,22 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", + "datafusion-proto-models", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] @@ -2536,8 +2515,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", @@ -2548,15 +2526,14 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "arrow-data", @@ -2564,6 +2541,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2573,23 +2551,25 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "chrono", @@ -2608,6 +2588,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost", ] @@ -2615,19 +2596,26 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-proto-models" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" +dependencies = [ + "datafusion-proto-common", + "prost", +] + [[package]] name = "datafusion-pruning" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "datafusion-common", @@ -2642,8 +2630,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "async-trait", "datafusion-common", @@ -2656,8 +2643,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "bigdecimal", @@ -2686,8 +2672,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "bigdecimal", @@ -2700,13 +2685,13 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] name = "datafusion-sqllogictest" version = "54.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d11faf3e95b017819a10800dc6d612c1f003696dea201700796b9268704f591" +source = "git+https://github.com/apache/datafusion?rev=eef101769d650b2b06ac2bcf298f71113a047c9d#eef101769d650b2b06ac2bcf298f71113a047c9d" dependencies = [ "arrow", "async-trait", @@ -2717,7 +2702,7 @@ dependencies = [ "futures", "half", "indicatif", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "sqllogictest", @@ -3025,7 +3010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3718,7 +3703,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -4206,12 +4191,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "integer-encoding" version = "4.1.0" @@ -4261,6 +4240,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -5436,9 +5424,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.4.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" dependencies = [ "ahash", "arrow-array", @@ -5465,7 +5453,6 @@ dependencies = [ "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -5553,7 +5540,7 @@ dependencies = [ "async-recursion", "bytes", "faststr", - "integer-encoding 4.1.0", + "integer-encoding", "lazy_static", "linkedbytes", "ordered-float 4.6.0", @@ -5829,9 +5816,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "fc153f5fd745cc038b5eed86622125969f8a39834a57bc96beaaf2512b1da729" dependencies = [ "libc", "once_cell", @@ -5843,18 +5830,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "bb77d9aa6d647507b55c69ee714d266d84c526c78ff0bc6dd8757f58591e64d1" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "3160087aa5733bce7d9a729f8c55f85a2a53b74742a09613178eeebff0722253" dependencies = [ "libc", "pyo3-build-config", @@ -5862,9 +5849,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "91f9d455db760a9a0b0ddeaac25f1390b8a36ba73dfbda9f127cac6fc340d4d5" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -5874,13 +5861,12 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "e343bcec300ff262f5806a33a4e51b6d097a8a46435f512fcb83e95592581625" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn", ] @@ -5924,7 +5910,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -5963,9 +5949,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6571,7 +6557,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6629,7 +6615,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6862,6 +6848,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -7631,10 +7618,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7692,17 +7679,6 @@ dependencies = [ "cfg-if 1.0.4", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding 3.0.4", - "ordered-float 2.10.1", -] - [[package]] name = "time" version = "0.3.53" @@ -8542,7 +8518,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 693c9ffc91..278bdaa8e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,15 +49,15 @@ aes-gcm = "0.10" anyhow = "1.0.72" apache-avro = { version = "0.21", features = ["snappy", "zstandard"] } array-init = "2" -arrow = "58.4" -arrow-arith = "58.4" -arrow-array = "58.4" -arrow-buffer = "58.4" -arrow-cast = "58.4" -arrow-ord = "58.4" -arrow-schema = "58.4" -arrow-select = "58.4" -arrow-string = "58.4" +arrow = "59.1" +arrow-arith = "59.1" +arrow-array = "59.1" +arrow-buffer = "59.1" +arrow-cast = "59.1" +arrow-ord = "59.1" +arrow-schema = "59.1" +arrow-select = "59.1" +arrow-string = "59.1" as-any = "0.3.2" async-trait = "0.1.89" aws-config = "1.8.7" @@ -122,10 +122,10 @@ murmur3 = "0.5.2" once_cell = "1.20" opendal = "0.57" ordered-float = "4" -parquet = "58.4" +parquet = "59.1" pilota = "0.11.10" pretty_assertions = "1.4" -pyo3 = "0.28" +pyo3 = "0.29" rand = "0.9.3" regex = "1.11.3" reqwest = { version = "0.12.12", default-features = false, features = ["json"] } @@ -158,3 +158,9 @@ volo = "0.10.6" volo-thrift = "0.10.8" zeroize = "1.7" zstd = "0.13.3" + +[patch.crates-io] +datafusion = { git = "https://github.com/apache/datafusion", rev = "eef101769d650b2b06ac2bcf298f71113a047c9d" } +datafusion-cli = { git = "https://github.com/apache/datafusion", rev = "eef101769d650b2b06ac2bcf298f71113a047c9d" } +datafusion-ffi = { git = "https://github.com/apache/datafusion", rev = "eef101769d650b2b06ac2bcf298f71113a047c9d" } +datafusion-sqllogictest = { git = "https://github.com/apache/datafusion", rev = "eef101769d650b2b06ac2bcf298f71113a047c9d" } diff --git a/bindings/python/src/datafusion_table_provider.rs b/bindings/python/src/datafusion_table_provider.rs index 2f4745fa7a..153db63815 100644 --- a/bindings/python/src/datafusion_table_provider.rs +++ b/bindings/python/src/datafusion_table_provider.rs @@ -16,7 +16,7 @@ // under the License. use std::collections::HashMap; -use std::ffi::{CStr, CString}; +use std::ffi::CStr; use std::sync::Arc; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; @@ -32,7 +32,7 @@ use pyo3::types::{PyAny, PyCapsule}; use crate::runtime::runtime; -// pyo3 0.28's CapsuleName only exposes `unsafe fn as_cstr() -> &CStr`, +// CapsuleName only exposes `unsafe fn as_cstr() -> &CStr`, // so we accept &CStr to allow direct comparison without UTF-8 validation. pub(crate) fn validate_pycapsule(capsule: &Bound, name: &CStr) -> PyResult<()> { let capsule_name = capsule.name()?; @@ -121,7 +121,7 @@ impl PyIcebergDataFusionTable { py: Python<'py>, session: Bound<'py, PyAny>, ) -> PyResult> { - let capsule_name = CString::new("datafusion_table_provider").unwrap(); + let capsule_name = c"datafusion_table_provider"; let logical_codec = ffi_logical_codec_from_pycapsule(session)?; @@ -132,7 +132,7 @@ impl PyIcebergDataFusionTable { logical_codec, ); - PyCapsule::new(py, ffi_provider, Some(capsule_name)) + PyCapsule::new_with_value(py, ffi_provider, capsule_name) } } diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 66eadfb7ec..71859d751a 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -92,7 +92,7 @@ mockall = { workspace = true } pretty_assertions = { workspace = true } rand = { workspace = true } regex = { workspace = true } -serde_arrow = { version = "0.14", features = ["arrow-58"] } +serde_arrow = { version = "0.14", features = ["arrow-59"] } tempfile = { workspace = true } [package.metadata.cargo-machete] diff --git a/crates/integrations/datafusion/src/table/table_provider_factory.rs b/crates/integrations/datafusion/src/table/table_provider_factory.rs index 9692340cac..a3b293f33a 100644 --- a/crates/integrations/datafusion/src/table/table_provider_factory.rs +++ b/crates/integrations/datafusion/src/table/table_provider_factory.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::catalog::{Session, TableProvider, TableProviderFactory}; +use datafusion::common::TableReference; use datafusion::error::Result as DFResult; use datafusion::logical_expr::CreateExternalTable; -use datafusion::sql::TableReference; use iceberg::io::{FileIOBuilder, LocalFsStorageFactory, StorageFactory}; use iceberg::table::StaticTable; use iceberg::{Error, ErrorKind, Result, TableIdent}; @@ -41,9 +41,9 @@ use crate::to_datafusion_error; /// ``` /// use std::sync::Arc; /// +/// use datafusion::common::TableReference; /// use datafusion::execution::session_state::SessionStateBuilder; /// use datafusion::prelude::*; -/// use datafusion::sql::TableReference; /// use iceberg_datafusion::IcebergTableProviderFactory; /// /// #[tokio::main] @@ -123,10 +123,9 @@ impl TableProviderFactory for IcebergTableProviderFactory { _state: &dyn Session, cmd: &CreateExternalTable, ) -> DFResult> { - check_cmd(cmd).map_err(to_datafusion_error)?; + let metadata_file_path = check_cmd(cmd).map_err(to_datafusion_error)?; let table_name = &cmd.name; - let metadata_file_path = &cmd.location; let options = &cmd.options; let table_name_with_ns = complement_namespace_if_necessary(table_name); @@ -154,7 +153,7 @@ impl TableProviderFactory for IcebergTableProviderFactory { } } -fn check_cmd(cmd: &CreateExternalTable) -> Result<()> { +fn check_cmd(cmd: &CreateExternalTable) -> Result<&str> { let CreateExternalTable { schema, table_partition_cols, @@ -178,7 +177,13 @@ fn check_cmd(cmd: &CreateExternalTable) -> Result<()> { )); } - Ok(()) + match cmd.locations.as_slice() { + [location] => Ok(location), + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Iceberg external tables require exactly one metadata location.", + )), + } } /// Complements the namespace of a table name if necessary. @@ -218,12 +223,11 @@ mod tests { use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::catalog::TableProviderFactory; - use datafusion::common::{Constraints, DFSchema}; + use datafusion::common::{Constraints, DFSchema, TableReference}; use datafusion::execution::session_state::SessionStateBuilder; use datafusion::logical_expr::CreateExternalTable; use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; use datafusion::prelude::SessionContext; - use datafusion::sql::TableReference; use super::*; @@ -257,7 +261,7 @@ mod tests { CreateExternalTable { name: TableReference::partial("static_ns", "static_table"), - location: metadata_file_path, + locations: vec![metadata_file_path], schema: Arc::new(DFSchema::empty()), file_type: "iceberg".to_string(), options: Default::default(),