diff --git a/Cargo.lock b/Cargo.lock index 05d5196..4d9490b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,35 +143,6 @@ version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" -[[package]] -name = "apache-avro" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" -dependencies = [ - "bigdecimal", - "bon", - "bzip2", - "crc32fast", - "digest 0.10.7", - "liblzma", - "log", - "miniz_oxide", - "num-bigint", - "quad-rand", - "rand 0.9.2", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum", - "strum_macros", - "thiserror 2.0.17", - "uuid", - "zstd", -] - [[package]] name = "approx" version = "0.5.1" @@ -210,9 +181,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" dependencies = [ "arrow-arith", "arrow-array", @@ -231,9 +202,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" dependencies = [ "arrow-array", "arrow-buffer", @@ -245,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -256,17 +227,41 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", ] +[[package]] +name = "arrow-avro" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bytes", + "bzip2", + "crc", + "flate2", + "indexmap", + "liblzma", + "rand 0.9.2", + "serde", + "serde_json", + "snap", + "strum_macros", + "uuid", + "zstd", +] + [[package]] name = "arrow-buffer" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" dependencies = [ "bytes", "half", @@ -276,9 +271,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" dependencies = [ "arrow-array", "arrow-buffer", @@ -298,9 +293,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" dependencies = [ "arrow-array", "arrow-cast", @@ -313,9 +308,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -326,9 +321,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" dependencies = [ "arrow-array", "arrow-buffer", @@ -342,15 +337,16 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", "indexmap", @@ -366,9 +362,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -386,7 +382,7 @@ dependencies = [ "async-trait", "bytes", "chrono", - "datafusion", + "datafusion 54.0.0", "futures", "geo-postgis", "geo-traits", @@ -402,9 +398,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" dependencies = [ "arrow-array", "arrow-buffer", @@ -425,9 +421,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -439,9 +435,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" dependencies = [ "arrow-array", "arrow-buffer", @@ -525,7 +521,6 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "serde", ] [[package]] @@ -592,31 +587,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "bon" -version = "3.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234655ec178edd82b891e262ea7cf71f6584bcd09eff94db786be23f1821825c" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365" -dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - [[package]] name = "borsh" version = "1.6.0" @@ -747,7 +717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures", "rand_core 0.10.0", ] @@ -926,22 +896,28 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] -name = "cpufeatures" -version = "0.3.0" +name = "crc" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ - "libc", + "crc-catalog", ] +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1066,41 +1042,6 @@ dependencies = [ "cmov", ] -[[package]] -name = "darling" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", -] - -[[package]] -name = "darling_macro" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" -dependencies = [ - "darling_core", - "quote", - "syn 2.0.117", -] - [[package]] name = "dashmap" version = "6.1.0" @@ -1125,44 +1066,88 @@ dependencies = [ "arrow-schema", "async-trait", "bytes", + "chrono", + "datafusion-catalog 53.1.0", + "datafusion-catalog-listing 53.1.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-datasource-arrow 53.1.0", + "datafusion-datasource-csv 53.1.0", + "datafusion-datasource-json 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate 53.1.0", + "datafusion-functions-table 53.1.0", + "datafusion-functions-window 53.1.0", + "datafusion-optimizer 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-optimizer 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "regex", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", "bzip2", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", + "datafusion-catalog 54.0.0", + "datafusion-catalog-listing 54.0.0", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-datasource-arrow 54.0.0", "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", + "datafusion-datasource-csv 54.0.0", + "datafusion-datasource-json 54.0.0", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate 54.0.0", "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-functions-table 54.0.0", + "datafusion-functions-window 54.0.0", + "datafusion-optimizer 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-optimizer 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "datafusion-sql", "flate2", "futures", + "indexmap", "itertools 0.14.0", "liblzma", "log", "object_store", "parking_lot", "parquet", - "rand 0.9.2", - "regex", "sqlparser", "tempfile", "tokio", @@ -1180,14 +1165,39 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "log", @@ -1204,15 +1214,38 @@ checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 54.0.0", + "datafusion-common 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", "futures", "itertools 0.14.0", "log", @@ -1226,7 +1259,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash 0.8.12", - "apache-avro", "arrow", "arrow-ipc", "chrono", @@ -1237,11 +1269,34 @@ dependencies = [ "libc", "log", "object_store", - "parquet", "paste", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "libc", + "log", + "object_store", + "parquet", "recursive", "sqlparser", "tokio", + "uuid", "web-time", ] @@ -1256,11 +1311,51 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-common-runtime" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +dependencies = [ + "futures", + "log", + "tokio", +] + [[package]] name = "datafusion-datasource" version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "rand 0.9.2", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" dependencies = [ "arrow", "async-compression", @@ -1268,15 +1363,15 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "flate2", "futures", "glob", @@ -1284,6 +1379,7 @@ dependencies = [ "liblzma", "log", "object_store", + "parking_lot", "rand 0.9.2", "tokio", "tokio-util", @@ -1301,14 +1397,38 @@ dependencies = [ "arrow-ipc", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "object_store", @@ -1317,21 +1437,20 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" +checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" dependencies = [ - "apache-avro", "arrow", + "arrow-avro", "async-trait", "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", - "num-traits", "object_store", ] @@ -1344,14 +1463,37 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "object_store", "regex", @@ -1367,14 +1509,14 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "object_store", "serde_json", @@ -1382,27 +1524,51 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "datafusion-datasource-json" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-pruning 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "log", @@ -1418,6 +1584,12 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +[[package]] +name = "datafusion-doc" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" + [[package]] name = "datafusion-execution" version = "53.1.0" @@ -1429,9 +1601,9 @@ dependencies = [ "async-trait", "chrono", "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "futures", "log", "object_store", @@ -1441,24 +1613,67 @@ dependencies = [ "url", ] +[[package]] +name = "datafusion-execution" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "dashmap", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "indexmap", + "itertools 0.14.0", + "paste", + "serde_json", +] + [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", "indexmap", "itertools 0.14.0", - "paste", "recursive", "serde_json", "sqlparser", @@ -1471,17 +1686,57 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 53.1.0", "indexmap", "itertools 0.14.0", "paste", ] +[[package]] +name = "datafusion-expr-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "indexmap", + "itertools 0.14.0", +] + [[package]] name = "datafusion-functions" version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "chrono-tz", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-macros 53.1.0", + "hex", + "itertools 0.14.0", + "log", + "memchr", + "num-traits", + "rand 0.9.2", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" dependencies = [ "arrow", "arrow-buffer", @@ -1490,22 +1745,22 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr-common 54.0.0", "hex", "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5", "memchr", "num-traits", "rand 0.9.2", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2", "uuid", ] @@ -1517,20 +1772,41 @@ checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "half", "log", "num-traits", "paste", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" @@ -1539,34 +1815,46 @@ checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", ] [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] @@ -1577,14 +1865,30 @@ checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-plan 53.1.0", "parking_lot", "paste", ] +[[package]] +name = "datafusion-functions-table" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 54.0.0", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-plan 54.0.0", + "parking_lot", +] + [[package]] name = "datafusion-functions-window" version = "53.1.0" @@ -1592,25 +1896,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "log", "paste", ] +[[package]] +name = "datafusion-functions-window" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "log", +] + [[package]] name = "datafusion-functions-window-common" version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +dependencies = [ + "datafusion-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", ] [[package]] @@ -1619,7 +1950,18 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ - "datafusion-doc", + "datafusion-doc 53.1.0", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "datafusion-macros" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +dependencies = [ + "datafusion-doc 54.0.0", "quote", "syn 2.0.117", ] @@ -1632,10 +1974,29 @@ checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "indexmap", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-optimizer" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", "indexmap", "itertools 0.14.0", "log", @@ -1651,7 +2012,7 @@ dependencies = [ "arrow-pg", "async-trait", "criterion", - "datafusion", + "datafusion 54.0.0", "env_logger", "futures", "log", @@ -1667,11 +2028,11 @@ checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", "half", "hashbrown 0.16.1", "indexmap", @@ -1679,6 +2040,27 @@ dependencies = [ "parking_lot", "paste", "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "petgraph", "recursive", "tokio", ] @@ -1690,11 +2072,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", "itertools 0.14.0", ] @@ -1707,14 +2104,31 @@ dependencies = [ "ahash 0.8.12", "arrow", "chrono", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr-common 53.1.0", "hashbrown 0.16.1", "indexmap", "itertools 0.14.0", "parking_lot", ] +[[package]] +name = "datafusion-physical-expr-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 54.0.0", + "datafusion-expr-common 54.0.0", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "pin-project", +] + [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" @@ -1722,14 +2136,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-pruning 53.1.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-pruning 54.0.0", "itertools 0.14.0", "recursive", ] @@ -1745,15 +2177,15 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "futures", "half", "hashbrown 0.16.1", @@ -1766,6 +2198,39 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-physical-plan" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + [[package]] name = "datafusion-postgres" version = "0.17.0" @@ -1774,7 +2239,7 @@ dependencies = [ "async-trait", "bytes", "chrono", - "datafusion", + "datafusion 54.0.0", "datafusion-pg-catalog", "env_logger", "futures", @@ -1794,7 +2259,7 @@ dependencies = [ name = "datafusion-postgres-cli" version = "0.17.0" dependencies = [ - "datafusion", + "datafusion 54.0.0", "datafusion-postgres", "env_logger", "log", @@ -1809,16 +2274,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", "itertools 0.14.0", "log", ] +[[package]] +name = "datafusion-pruning" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "log", +] + [[package]] name = "datafusion-session" version = "53.1.0" @@ -1826,24 +2307,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-plan 53.1.0", + "parking_lot", +] + +[[package]] +name = "datafusion-session" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +dependencies = [ + "async-trait", + "datafusion-common 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-plan 54.0.0", "parking_lot", ] [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", "datafusion-functions-nested", "indexmap", "log", @@ -2013,12 +2508,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "foldhash" version = "0.1.5" @@ -2256,7 +2745,7 @@ dependencies = [ "arrow-arith", "arrow-array", "arrow-schema", - "datafusion", + "datafusion 53.1.0", "geo", "geo-traits", "geoarrow-array", @@ -2398,6 +2887,17 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "heapless" version = "0.8.0" @@ -2456,9 +2956,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hybrid-array" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ "typenum", ] @@ -2617,12 +3117,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "1.1.0" @@ -2646,12 +3140,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -2845,9 +3339,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "liblzma" @@ -2911,16 +3405,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest 0.10.7", -] - [[package]] name = "md-5" version = "0.11.0" @@ -2972,7 +3456,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -3122,9 +3605,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -3140,7 +3623,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "lz4_flex", "num-bigint", "num-integer", @@ -3253,6 +3736,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -3330,10 +3833,10 @@ dependencies = [ "bytes", "fallible-iterator", "hmac", - "md-5 0.11.0", + "md-5", "memchr", "rand 0.10.0", - "sha2 0.11.0", + "sha2", "stringprep", ] @@ -3453,12 +3956,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - [[package]] name = "quote" version = "1.0.45" @@ -3848,16 +4345,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -3891,17 +4378,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -3909,7 +4385,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures", "digest 0.11.3", ] @@ -4018,9 +4494,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", @@ -4068,12 +4544,6 @@ dependencies = [ "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "structopt" version = "0.3.26" @@ -4098,17 +4568,11 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "strum" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" - [[package]] name = "strum_macros" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -4410,9 +4874,9 @@ checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-bidi" @@ -4503,7 +4967,6 @@ checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "getrandom 0.4.1", "js-sys", - "serde_core", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index 4b095d4..7c8fbd9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ arrow = "58" arrow-schema = "58" bytes = "1.11.1" chrono = { version = "0.4", features = ["std"] } -datafusion = { version = "53" } +datafusion = { version = "54" } futures = "0.3" pgwire = { version = "0.40", default-features = false } postgres-types = "0.2" diff --git a/datafusion-pg-catalog/src/pg_catalog.rs b/datafusion-pg-catalog/src/pg_catalog.rs index 722b58c..2776bdb 100644 --- a/datafusion-pg-catalog/src/pg_catalog.rs +++ b/datafusion-pg-catalog/src/pg_catalog.rs @@ -26,6 +26,7 @@ use crate::pg_catalog::catalog_info::CatalogInfo; use crate::pg_catalog::context::PgCatalogContextProvider; use crate::pg_catalog::empty_table::EmptyTable; +pub mod array_bounds_udf; pub mod catalog_info; pub mod context; pub mod empty_table; @@ -33,6 +34,7 @@ pub mod format_type; pub mod has_privilege_udf; pub mod oid_coercion_rule; pub mod oid_field; +pub mod oid_type_planner; pub mod pg_attribute; pub mod pg_class; pub mod pg_database; @@ -208,10 +210,6 @@ pub struct PgCatalogSchemaProvider { #[async_trait] impl SchemaProvider for PgCatalogSchemaProvider { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn table_names(&self) -> Vec { PG_CATALOG_TABLES.iter().map(ToString::to_string).collect() } @@ -1384,10 +1382,6 @@ pub fn create_pg_get_constraintdef() -> ScalarUDF { } impl ScalarUDFImpl for GetConstraintDefUDF { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { "pg_get_constraintdef" } @@ -1499,6 +1493,11 @@ where session_context.register_udf(create_pg_get_partition_ancestors_udf()); session_context.register_udf(quote_ident_udf::create_quote_ident_udf()); session_context.register_udf(quote_ident_udf::create_parse_ident_udf()); + // Postgres array bounds. DataFusion ships array_length but not array_upper / + // array_lower; without these, client queries (psql/dbeaver/grafan) that use + // them were only "supported" via hardcoded blacklist token substitution. + session_context.register_udf(array_bounds_udf::create_array_upper_udf()); + session_context.register_udf(array_bounds_udf::create_array_lower_udf()); // Make oid / oid-alias columns (relnamespace, atttypid, ...) compare // against string literals the way Postgres does. Requires the schema @@ -1507,6 +1506,21 @@ where pg_catalog.clone() as Arc, ))); + // Teach the SQL planner to accept Postgres oid-alias type names + // (`regclass`, `regproc`, ...) -- DataFusion rejects them as unsupported + // otherwise. Each maps to int4 carrying `pg.oid_alias` metadata, which the + // analyzer rule above then reads to resolve name strings -> oids. This + // replaces the former SQL-layer cast-stripping / cast-rewriting rules. + { + use datafusion::execution::session_state::SessionStateBuilder; + let state = session_context.state_ref(); + let existing = state.read().clone(); + let new_state = SessionStateBuilder::new_from_existing(existing) + .with_type_planner(Arc::new(oid_type_planner::PgOidTypePlanner)) + .build(); + *state.write() = new_state; + } + Ok(()) } diff --git a/datafusion-pg-catalog/src/pg_catalog/array_bounds_udf.rs b/datafusion-pg-catalog/src/pg_catalog/array_bounds_udf.rs new file mode 100644 index 0000000..e9a5cdb --- /dev/null +++ b/datafusion-pg-catalog/src/pg_catalog/array_bounds_udf.rs @@ -0,0 +1,345 @@ +//! PostgreSQL `array_upper` / `array_lower` scalar functions. +//! +//! DataFusion does not ship these, so any client query containing them (psql, +//! dbeaver, grafana) was rejected with "Invalid function 'array_upper'". They +//! used to only "work" because the exact query fragments were on the parser's +//! token blacklist and got swapped out before execution. +//! +//! Arrow's list types (`List`/`LargeList`/`FixedSizeList`) are always 1-based +//! and contiguous, so for any present dimension: +//! * `array_lower(arr, dim)` = 1 +//! * `array_upper(arr, dim)` = `array_length(arr, dim)` +//! +//! Following Postgres, both return NULL when the array is NULL, when `dim` is +//! out of range (< 1 or beyond the array's dimensionality), or when the array +//! is empty on the requested dimension. + +use std::sync::Arc; + +use datafusion::arrow::array::{Array, ArrayRef, AsArray, Int32Builder}; +use datafusion::arrow::datatypes::{DataType, Int32Type, Int64Type}; +use datafusion::common::DataFusionError; +use datafusion::logical_expr::{ + ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, TypeSignature, Volatility, +}; + +/// Length of a list at a given 1-based dimension, with Postgres NULL semantics. +/// +/// Returns `Some(len)` only when `dim` is in range and that dimension is +/// non-empty; returns `None` for a null array, an out-of-range dimension, or an +/// empty dimension. This matches the behavior of Postgres `array_length` / +/// `array_upper` / `array_lower`, which all yield NULL in those cases. +fn length_at_dim(arr: Option, dim: i64) -> Option { + if dim < 1 { + return None; + } + let mut value = arr?; + let mut current = 1; + loop { + let len = value.len(); + if current == dim { + return if len == 0 { None } else { Some(len) }; + } + // Cannot descend further into an empty or non-list dimension. + if len == 0 { + return None; + } + value = match value.data_type() { + DataType::List(_) => value.as_list::().value(0), + DataType::LargeList(_) => value.as_list::().value(0), + DataType::FixedSizeList(_, _) => value.as_fixed_size_list().value(0), + _ => return None, + }; + current += 1; + } +} + +/// Read the per-row dimension argument as `Option`, accepting either +/// Postgres `integer` (Int32) or `bigint` (Int64). +fn extract_dims(arg: &ArrayRef) -> Result>, DataFusionError> { + let dims = match arg.data_type() { + DataType::Int32 => arg + .as_primitive::() + .iter() + .map(|v| v.map(|x| x as i64)) + .collect(), + DataType::Int64 => arg.as_primitive::().iter().collect(), + other => { + return Err(DataFusionError::Internal(format!( + "array dimension must be an integer, got {other}" + ))); + } + }; + Ok(dims) +} + +/// Core implementation shared by `array_upper` and `array_lower`. +/// +/// `upper` selects the upper bound (the length at the dimension); otherwise the +/// lower bound (always 1 for Arrow's 1-based lists) is produced. +fn array_bounds_impl( + args: &[ColumnarValue], + upper: bool, +) -> Result { + let args = ColumnarValue::values_to_arrays(args)?; + if args.len() != 2 { + return Err(DataFusionError::Internal(format!( + "array_upper/array_lower expect exactly 2 arguments, got {}", + args.len() + ))); + } + let list_arr = &args[0]; + let dims = extract_dims(&args[1])?; + + let rows = list_arr.len(); + let mut out = Int32Builder::with_capacity(rows); + + let build = |row: Option, dim: Option, out: &mut Int32Builder| match dim { + // Null dimension -> NULL result. + None => out.append_null(), + Some(d) => match length_at_dim(row, d) { + // Empty / out-of-range dimension -> NULL (Postgres semantics). + None => out.append_null(), + Some(len) if upper => out.append_value(len as i32), + // Lower bound of a present 1-based dimension is always 1. + Some(_) => out.append_value(1), + }, + }; + + match list_arr.data_type() { + DataType::List(_) => { + let l = list_arr.as_list::(); + for (row, dim) in l.iter().zip(dims.iter().copied()) { + build(row, dim, &mut out); + } + } + DataType::LargeList(_) => { + let l = list_arr.as_list::(); + for (row, dim) in l.iter().zip(dims.iter().copied()) { + build(row, dim, &mut out); + } + } + DataType::FixedSizeList(_, _) => { + let l = list_arr.as_fixed_size_list(); + for (row, dim) in l.iter().zip(dims.iter().copied()) { + build(row, dim, &mut out); + } + } + other => { + return Err(DataFusionError::Internal(format!( + "array_upper/array_lower expect a list argument, got {other}" + ))); + } + } + + Ok(ColumnarValue::Array(Arc::new(out.finish()))) +} + +fn two_arg_array_signature() -> Signature { + Signature::one_of( + vec![TypeSignature::ArraySignature( + ArrayFunctionSignature::Array { + arguments: vec![ArrayFunctionArgument::Array, ArrayFunctionArgument::Index], + array_coercion: None, + }, + )], + Volatility::Immutable, + ) +} + +#[derive(Debug, Hash, PartialEq, Eq)] +pub struct ArrayUpperUDF { + signature: Signature, +} + +impl Default for ArrayUpperUDF { + fn default() -> Self { + Self::new() + } +} + +impl ArrayUpperUDF { + pub fn new() -> Self { + Self { + signature: two_arg_array_signature(), + } + } + + pub fn into_scalar_udf(self) -> ScalarUDF { + ScalarUDF::new_from_impl(self) + } +} + +impl ScalarUDFImpl for ArrayUpperUDF { + fn name(&self) -> &str { + "array_upper" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int32) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + array_bounds_impl(&args.args, true) + } +} + +#[derive(Debug, Hash, PartialEq, Eq)] +pub struct ArrayLowerUDF { + signature: Signature, +} + +impl Default for ArrayLowerUDF { + fn default() -> Self { + Self::new() + } +} + +impl ArrayLowerUDF { + pub fn new() -> Self { + Self { + signature: two_arg_array_signature(), + } + } + + pub fn into_scalar_udf(self) -> ScalarUDF { + ScalarUDF::new_from_impl(self) + } +} + +impl ScalarUDFImpl for ArrayLowerUDF { + fn name(&self) -> &str { + "array_lower" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int32) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + array_bounds_impl(&args.args, false) + } +} + +/// Create the PostgreSQL `array_upper` UDF. +pub fn create_array_upper_udf() -> ScalarUDF { + ArrayUpperUDF::new().into_scalar_udf() +} + +/// Create the PostgreSQL `array_lower` UDF. +pub fn create_array_lower_udf() -> ScalarUDF { + ArrayLowerUDF::new().into_scalar_udf() +} + +#[cfg(test)] +mod tests { + use super::*; + + use datafusion::arrow::array::{Int32Builder, ListBuilder}; + + /// Build a list array where each entry is one row. `Some(vec![])` is an empty + /// (non-null) list, `Some(vec![1,2,3])` is `[1,2,3]`, and `None` is a NULL + /// list value. + fn int_list(rows: &[Option>]) -> ArrayRef { + let mut builder = ListBuilder::new(Int32Builder::new()); + for row in rows { + match row { + Some(elems) => { + for v in elems { + builder.values().append_value(*v); + } + builder.append(true); + } + None => builder.append_null(), + } + } + Arc::new(builder.finish()) + } + + fn run_bounds(list: ArrayRef, dim: i64, upper: bool) -> Vec> { + let dim_arr = Arc::new(datafusion::arrow::array::Int64Array::from(vec![ + dim; + list.len() + ])); + let args = vec![ColumnarValue::Array(list), ColumnarValue::Array(dim_arr)]; + let out = array_bounds_impl(&args, upper).unwrap(); + let arr = match out { + ColumnarValue::Array(a) => a, + _ => unreachable!(), + }; + arr.as_primitive::().iter().collect() + } + + #[test] + fn test_length_at_dim() { + let three = int_list(&[Some(vec![1, 2, 3])]); + // length_at_dim operates on a single extracted row's list value. + let row = three.as_list::().value(0); + assert_eq!(length_at_dim(Some(row.clone()), 1), Some(3)); + // dim 2 is out of range for a 1-D list + assert_eq!(length_at_dim(Some(row), 2), None); + // null array + assert_eq!(length_at_dim(None, 1), None); + // dim < 1 + let one = int_list(&[Some(vec![1])]); + assert_eq!(length_at_dim(Some(one.as_list::().value(0)), 0), None); + // empty list at dim 1 -> None (Postgres: empty array has no length) + let empty = int_list(&[Some(vec![])]); + assert_eq!( + length_at_dim(Some(empty.as_list::().value(0)), 1), + None + ); + } + + #[test] + fn test_array_upper() { + let arr = int_list(&[Some(vec![1, 2, 3])]); + assert_eq!(run_bounds(arr, 1, true), vec![Some(3)]); + + // empty (non-null) array -> NULL (Postgres semantics) + let arr = int_list(&[Some(vec![])]); + assert_eq!(run_bounds(arr, 1, true), vec![None]); + + // null array value -> NULL + let arr = int_list(&[None]); + assert_eq!(run_bounds(arr, 1, true), vec![None]); + } + + #[test] + fn test_array_lower() { + let arr = int_list(&[Some(vec![1, 2, 3])]); + assert_eq!(run_bounds(arr, 1, false), vec![Some(1)]); + + // empty (non-null) array -> NULL + let arr = int_list(&[Some(vec![])]); + assert_eq!(run_bounds(arr, 1, false), vec![None]); + + // null array value -> NULL + let arr = int_list(&[None]); + assert_eq!(run_bounds(arr, 1, false), vec![None]); + } + + #[test] + fn test_out_of_range_dim() { + let arr = int_list(&[Some(vec![1, 2, 3])]); + assert_eq!(run_bounds(arr.clone(), 2, true), vec![None]); + assert_eq!(run_bounds(arr.clone(), 2, false), vec![None]); + assert_eq!(run_bounds(arr, 0, true), vec![None]); + } + + #[test] + fn test_multiple_rows() { + // Per-row independence. + let arr = int_list(&[Some(vec![1]), Some(vec![1, 2]), Some(vec![]), None]); + assert_eq!(run_bounds(arr, 1, true), vec![Some(1), Some(2), None, None]); + } +} diff --git a/datafusion-pg-catalog/src/pg_catalog/format_type.rs b/datafusion-pg-catalog/src/pg_catalog/format_type.rs index 24da102..78319d2 100644 --- a/datafusion-pg-catalog/src/pg_catalog/format_type.rs +++ b/datafusion-pg-catalog/src/pg_catalog/format_type.rs @@ -267,10 +267,6 @@ impl ScalarUDFImpl for FormatTypeUDF { fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { format_type_impl(&args.args) } - - fn as_any(&self) -> &dyn std::any::Any { - self - } } pub fn create_format_type_udf() -> ScalarUDF { diff --git a/datafusion-pg-catalog/src/pg_catalog/has_privilege_udf.rs b/datafusion-pg-catalog/src/pg_catalog/has_privilege_udf.rs index 0e17c62..b505f47 100644 --- a/datafusion-pg-catalog/src/pg_catalog/has_privilege_udf.rs +++ b/datafusion-pg-catalog/src/pg_catalog/has_privilege_udf.rs @@ -60,10 +60,6 @@ impl ScalarUDFImpl for PgHasPrivilegeUDF { Ok(ColumnarValue::Array(array)) } - - fn as_any(&self) -> &dyn std::any::Any { - self - } } pub fn create_has_privilege_udf(name: &str) -> ScalarUDF { diff --git a/datafusion-pg-catalog/src/pg_catalog/oid_coercion_rule.rs b/datafusion-pg-catalog/src/pg_catalog/oid_coercion_rule.rs index 3ed5e79..e81a43f 100644 --- a/datafusion-pg-catalog/src/pg_catalog/oid_coercion_rule.rs +++ b/datafusion-pg-catalog/src/pg_catalog/oid_coercion_rule.rs @@ -37,7 +37,7 @@ use datafusion::common::{DFSchemaRef, ExprSchema, Result, ScalarValue, Spans}; use datafusion::datasource::{TableProvider, provider_as_source}; use datafusion::logical_expr::expr::InList; use datafusion::logical_expr::{ - BinaryExpr, Cast, Expr, Filter, LogicalPlan, LogicalPlanBuilder, Operator, Subquery, TryCast, + BinaryExpr, Cast, Expr, LogicalPlan, LogicalPlanBuilder, Operator, Subquery, TryCast, }; use datafusion::optimizer::analyzer::AnalyzerRule; @@ -90,17 +90,20 @@ impl<'a> TreeNodeRewriter for PlanRewriter<'a> { type Node = LogicalPlan; fn f_down(&mut self, plan: LogicalPlan) -> Result> { - let LogicalPlan::Filter(filter) = plan else { - return Ok(Transformed::no(plan)); - }; - let input = filter.input.clone(); - let mut expr_rewriter = ExprOidRewriter { - schema: input.schema().clone(), - provider: self.provider, - }; - let new_predicate = filter.predicate.rewrite(&mut expr_rewriter)?.data; - let new_filter = Filter::try_new(new_predicate, input)?; - Ok(Transformed::yes(LogicalPlan::Filter(new_filter))) + // Rewrite every expression in every plan node (filters, join ON + // filters, projections, ...) using that node's schema. The + // schema-free cast arm (which carries its own kind via the + // PgOidTypePlanner metadata) is what makes projections and + // oid-column casts like `classoid = 'pg_namespace'::regclass` work; + // the schema is only consulted by the bare `col = 'str'` path. + let schema = plan.schema().clone(); + plan.map_expressions(|expr| { + let mut rewriter = ExprOidRewriter { + schema: schema.clone(), + provider: self.provider, + }; + expr.rewrite(&mut rewriter) + }) } } @@ -192,6 +195,30 @@ fn rewrite_expr_shallow( Ok(None) } } + // An explicit oid-alias cast whose kind is stamped on its target Field + // by `PgOidTypePlanner` -- e.g. `'public'::regnamespace`, + // `typinput = 'array_in'::regproc`, or `classoid = 'pg_namespace'::regclass` + // (where `classoid` is a plain `oid`, so only the CAST declares the kind). + // Resolve the cast operand to an oid (literal for numeric strings, + // name-lookup subquery otherwise), in ANY plan position (this arm is + // reached via `map_expressions`, so projections / filters / joins all + // benefit). Returning `None` here for a non-string operand lets the + // rewriter descend into nested casts (e.g. `'x'::regclass::oid`). + Expr::Cast(Cast { + expr: operand, + field, + }) + | Expr::TryCast(TryCast { + expr: operand, + field, + }) if kind_from_field(field).is_some() => { + let kind = kind_from_field(field).unwrap(); + if let Some(s) = immediate_str_literal(operand) { + resolve_operand(&kind, s, provider) + } else { + Ok(None) + } + } _ => Ok(None), } } @@ -222,8 +249,16 @@ fn oid_kind(col: &Column, schema: &DFSchemaRef) -> Option { .cloned() } -/// If `e` is a string literal, return its value; otherwise `None`. -fn as_str_literal(e: &Expr) -> Option<&str> { +/// The oid-alias kind stamped on a cast's target [`Field`] by the +/// `PgOidTypePlanner`, if any (e.g. `'x'::regclass` -> `"regclass"`). +fn kind_from_field(field: &datafusion::arrow::datatypes::Field) -> Option { + field.metadata().get(OID_ALIAS_KEY).cloned() +} + +/// The string value of `e` if it is an *immediate* string literal (NOT peeking +/// through a cast), else `None`. Used to recognize the operand of an oid-alias +/// cast (`'name'::regclass`) without conflating it with a nested inner cast. +fn immediate_str_literal(e: &Expr) -> Option<&str> { if let Expr::Literal(s, _) = e { match s { ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => Some(s.as_str()), @@ -235,6 +270,30 @@ fn as_str_literal(e: &Expr) -> Option<&str> { } } +/// If `e` is a string literal -- possibly wrapped in a `CAST`/`TRY_CAST` to an +/// integer type that DataFusion's type-coercion analyzer produces when it sees +/// an `Int32` oid column compared against a string -- return its value; +/// otherwise `None`. +/// +/// Peeking through the cast keeps this rule order-independent: whether our +/// rule runs before type coercion (bare `'str'`) or after it +/// (`CAST('str' AS Int32)`), the same string is recovered. +fn as_str_literal(e: &Expr) -> Option<&str> { + let inner = match e { + Expr::Cast(Cast { expr, .. }) | Expr::TryCast(TryCast { expr, .. }) => expr.as_ref(), + other => other, + }; + if let Expr::Literal(s, _) = inner { + match s { + ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => Some(s.as_str()), + ScalarValue::Utf8View(Some(s)) => Some(s.as_str()), + _ => None, + } + } else { + None + } +} + /// The `(table, name_column)` used to resolve a name to an oid for a given /// oid-alias kind. Returns `None` for kinds without a backing catalog table /// (e.g. bare `oid`, or `regtype` where `pg_type` is unavailable) -- those only @@ -243,6 +302,7 @@ fn lookup_target(kind: &str) -> Option<(&'static str, &'static str)> { match kind { oid_field::kind::REGNAMESPACE => Some(("pg_namespace", "nspname")), oid_field::kind::REGCLASS => Some(("pg_class", "relname")), + oid_field::kind::REGTYPE => Some(("pg_type", "typname")), oid_field::kind::REGPROC => Some(("pg_proc", "proname")), _ => None, } @@ -315,9 +375,8 @@ mod tests { lookup_target(oid_field::kind::REGPROC), Some(("pg_proc", "proname")) ); - // bare oid and regtype have no backing table -> name resolution unsupported + // bare oid has no backing table -> name resolution unsupported assert_eq!(lookup_target(oid_field::kind::OID), None); - assert_eq!(lookup_target(oid_field::kind::REGTYPE), None); } #[test] @@ -330,7 +389,11 @@ mod tests { let cast = Expr::Cast(Cast { expr: Box::new(Expr::Column(Column::new_unqualified("relnamespace"))), - data_type: DataType::Utf8, + field: std::sync::Arc::new(datafusion::arrow::datatypes::Field::new( + "", + DataType::Utf8, + true, + )), }); assert_eq!( unwrap_column(&cast).map(|c| c.name().to_string()), @@ -339,7 +402,11 @@ mod tests { let trycast = Expr::TryCast(TryCast { expr: Box::new(Expr::Column(Column::new_unqualified("oid"))), - data_type: DataType::Utf8, + field: std::sync::Arc::new(datafusion::arrow::datatypes::Field::new( + "", + DataType::Utf8, + true, + )), }); assert_eq!( unwrap_column(&trycast).map(|c| c.name().to_string()), @@ -349,7 +416,11 @@ mod tests { // a cast around something that isn't a column is not unwrapped let nested = Expr::Cast(Cast { expr: Box::new(Expr::Literal(ScalarValue::Int32(Some(1)), None)), - data_type: DataType::Utf8, + field: std::sync::Arc::new(datafusion::arrow::datatypes::Field::new( + "", + DataType::Utf8, + true, + )), }); assert_eq!(unwrap_column(&nested), None); } @@ -364,6 +435,23 @@ mod tests { assert_eq!(as_str_literal(&lu), Some("public")); assert_eq!(as_str_literal(&uv), Some("public")); assert_eq!(as_str_literal(&n), None); + + // DataFusion's type-coercion analyzer wraps the literal in a cast to the + // column's Int32 type (`col = CAST('public' AS Int32)`) when it runs + // before this rule. The string must still be recoverable so the rule + // stays order-independent. + let cast = Expr::Cast(Cast { + expr: Box::new(Expr::Literal( + ScalarValue::Utf8(Some("public".to_string())), + None, + )), + field: std::sync::Arc::new(datafusion::arrow::datatypes::Field::new( + "", + DataType::Int32, + true, + )), + }); + assert_eq!(as_str_literal(&cast), Some("public")); } // --- end-to-end: a real pg_catalog with the analyzer rule installed --- diff --git a/datafusion-pg-catalog/src/pg_catalog/oid_field.rs b/datafusion-pg-catalog/src/pg_catalog/oid_field.rs index 9665858..ab1c102 100644 --- a/datafusion-pg-catalog/src/pg_catalog/oid_field.rs +++ b/datafusion-pg-catalog/src/pg_catalog/oid_field.rs @@ -40,12 +40,13 @@ pub mod kind { /// This is the **single source of truth** for: /// * the `pg.oid_alias` metadata value stamped on Int32 catalog columns (see /// [`oid_field`]) and embedded in the pg_catalog feather exports, and -/// * the set of oid-alias cast type names that the `RemoveOidTypeCast` -/// rewrite rule must strip before DataFusion's `sql_to_plan` (which rejects -/// them as unsupported SQL types). +/// * the set of oid-alias cast type names accepted by the `PgOidTypePlanner` +/// SQL type planner (so DataFusion's `sql_to_plan` maps them to int4 oids +/// carrying this metadata instead of rejecting them as unsupported types). /// -/// All of these are stored as `Int32` (the raw object identifier); the -/// oid-coercion analyzer rule resolves string comparisons against them. The +/// The oid-coercion analyzer rule then reads the metadata (on the column or +/// on the cast's target field) to resolve string comparisons the way Postgres +/// does. The /// [`kind`] named constants anchor the subset that the dynamic catalog tables /// use and that have name-lookup support (`regclass`/`regnamespace`/`regproc`); /// the remainder are numeric-resolution-only. diff --git a/datafusion-pg-catalog/src/pg_catalog/oid_type_planner.rs b/datafusion-pg-catalog/src/pg_catalog/oid_type_planner.rs new file mode 100644 index 0000000..07a1c48 --- /dev/null +++ b/datafusion-pg-catalog/src/pg_catalog/oid_type_planner.rs @@ -0,0 +1,303 @@ +//! A DataFusion [`TypePlanner`] that teaches the SQL planner to accept the +//! Postgres type names DataFusion otherwise rejects as "Unsupported SQL type": +//! +//! # oid-alias types +//! +//! `regclass`, `regproc`, `regtype`, `regnamespace`, `oid`, ... Each is mapped +//! to an `Int32` [`Field`] (oids are stored as int4) carrying the same +//! `pg.oid_alias` metadata the catalog columns use, so the kind survives into +//! the logical plan inside the resulting [`Expr::Cast`]'s `field`. The +//! oid-coercion analyzer rule then reads that metadata to resolve name strings +//! -> oids the way Postgres does -- no SQL/AST rewriting needed. +//! +//! This is the metadata-aware replacement for the former `RemoveOidTypeCast` +//! (SQL rewrite) + `RewriteRegCastToSubquery` (SQL rewrite) pair: instead of +//! stripping the cast at the AST layer (losing the type) and reconstructing it +//! later, the type is accepted up front and resolved at the analyzer layer. +//! +//! # `pg_catalog`-qualified builtins +//! +//! `pg_catalog.text`, `pg_catalog.int2`, `pg_catalog.int4`, ... sqlparser can +//! only represent a schema-qualified builtin as the catch-all `Custom` type, +//! which DataFusion rejects. This planner maps the canonical pg name to the +//! same Arrow type its unqualified builtin would produce, so casts like +//! `reloftype::pg_catalog.regtype::pg_catalog.text` plan cleanly. Arrays +//! (`pg_catalog.int2[]`) need no special handling: DataFusion recurses into +//! this planner for the array element type. This replaces the cast branch of +//! the former `RemoveQualifier` SQL rewrite. + +use std::sync::Arc; + +use datafusion::arrow::datatypes::{DataType, Field}; +use datafusion::common::Result; +use datafusion::logical_expr::planner::TypePlanner; +use datafusion::sql::sqlparser::ast::{DataType as SQLDataType, ObjectNamePart}; + +use crate::pg_catalog::oid_field::{self, OID_ALIAS_KEY, OID_ALIAS_TYPE_NAMES}; + +/// Recognize Postgres type names DataFusion rejects and map them to Arrow +/// types/metadata at planning time. +#[derive(Debug, Default)] +pub struct PgOidTypePlanner; + +impl PgOidTypePlanner { + /// Build the int4 oid field for a given kind (`regclass`, `oid`, ...). + fn oid_field(kind: &str) -> Arc { + Arc::new(Field::new("", DataType::Int32, true).with_metadata( + std::collections::HashMap::from([(OID_ALIAS_KEY.to_string(), kind.to_string())]), + )) + } + + /// Lowercase last identifier of a `Custom(...)` type name, if any. + /// + /// `regproc` -> `regproc`, `pg_catalog.regclass` -> `regclass`. + fn custom_type_name(sql_type: &SQLDataType) -> Option { + let SQLDataType::Custom(name, args) = sql_type else { + return None; + }; + if !args.is_empty() { + return None; + } + let last = name.0.last()?; + let ObjectNamePart::Identifier(ident) = last else { + return None; + }; + Some(ident.value.to_lowercase()) + } + + /// The oid-alias kind for a SQL type, if it is one we handle. + fn kind_for(sql_type: &SQLDataType) -> Option { + match sql_type { + SQLDataType::Regclass => Some(oid_field::kind::REGCLASS.to_string()), + SQLDataType::Custom(_, _) => Self::custom_type_name(sql_type) + .filter(|n| OID_ALIAS_TYPE_NAMES.contains(&n.as_str())), + _ => None, + } + } + + /// Map a `pg_catalog.` custom type to the Arrow [`DataType`] of its + /// unqualified builtin. + /// + /// Returns `None` unless the type is exactly a two-part + /// `pg_catalog.` with no type arguments, or `` is not a + /// builtin we know about (in which case it is left for DataFusion to error + /// on, or for another planner/rule to handle). The oid-alias arm in + /// [`TypePlanner::plan_type_field`] runs first, so oid-alias names + /// (`pg_catalog.regclass`, ...) never reach here. + fn pg_catalog_builtin(sql_type: &SQLDataType) -> Option { + let SQLDataType::Custom(name, args) = sql_type else { + return None; + }; + if !args.is_empty() || name.0.len() != 2 { + return None; + } + let mut parts = name.0.iter().filter_map(|p| match p { + ObjectNamePart::Identifier(i) => Some(i.value.as_str()), + _ => None, + }); + let (schema, type_name) = match (parts.next(), parts.next()) { + (Some(s), Some(t)) => (s, t), + _ => return None, + }; + if !schema.eq_ignore_ascii_case("pg_catalog") { + return None; + } + builtin_arrow_type(type_name) + } +} + +/// Arrow [`DataType`] for a Postgres pg_catalog builtin type name, or `None` +/// if it is not one we map (a user-defined type, or a builtin whose parameters +/// we don't model). +/// +/// String types map to `Utf8`, matching DataFusion's default conversion of the +/// unqualified `text`/`varchar`/`char` variants. The +/// `map_string_types_to_utf8view` session option is not consulted because the +/// [`TypePlanner`] trait does not expose session config; if a deployment needs +/// `Utf8View` it can register an additional planner. +fn builtin_arrow_type(name: &str) -> Option { + use DataType::*; + let dt = match name.to_ascii_lowercase().as_str() { + // booleans + "bool" | "boolean" => Boolean, + // integers + "int2" | "smallint" => Int16, + "int4" | "integer" | "int" => Int32, + "int8" | "bigint" => Int64, + // floats + "float4" | "real" => Float32, + "float8" => Float64, + // strings -> Utf8 (DataFusion's default for unqualified Text/Varchar) + "text" | "varchar" | "bpchar" | "char" | "name" => Utf8, + // bytes + "bytea" => Binary, + _ => return None, + }; + Some(dt) +} + +impl TypePlanner for PgOidTypePlanner { + fn plan_type_field(&self, sql_type: &SQLDataType) -> Result>> { + // 1. Scalar oid-alias types (regclass, oid, ...) -> int4 with kind + // metadata. Arrays of these (`regtype[]`) are handled by DataFusion + // recursing into this planner for the element type, so no Array arm + // is needed here. + if let Some(kind) = Self::kind_for(sql_type) { + return Ok(Some(Self::oid_field(&kind))); + } + // 2. pg_catalog-qualified builtins (pg_catalog.text, pg_catalog.int2, + // ...). sqlparser represents these as `Custom`, which DataFusion + // otherwise rejects; map them to their builtin Arrow type. As above, + // `pg_catalog.int2[]` works via element-type recursion. + if let Some(dt) = Self::pg_catalog_builtin(sql_type) { + return Ok(Some(Arc::new(Field::new("", dt, true)))); + } + Ok(None) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::sql::sqlparser::dialect::PostgreSqlDialect; + use datafusion::sql::sqlparser::parser::Parser; + + fn cast_target_type(sql: &str) -> SQLDataType { + // SELECT 'x':: AS c -> the cast's target DataType + let mut stmts = Parser::parse_sql(&PostgreSqlDialect {}, sql).unwrap(); + let stmt = stmts.remove(0); + use datafusion::sql::sqlparser::ast::{SetExpr, Statement}; + let Statement::Query(q) = stmt else { + panic!("not a query"); + }; + let SetExpr::Select(sel) = *q.body else { + panic!("not a select"); + }; + let cast = match &sel.projection[0] { + datafusion::sql::sqlparser::ast::SelectItem::UnnamedExpr(e) => e, + datafusion::sql::sqlparser::ast::SelectItem::ExprWithAlias { expr, .. } => expr, + other => panic!("unexpected projection {other:?}"), + }; + let datafusion::sql::sqlparser::ast::Expr::Cast { data_type, .. } = cast else { + panic!("not a cast: {cast:?}"); + }; + data_type.clone() + } + + #[test] + fn recognizes_all_oid_alias_type_names() { + let planner = PgOidTypePlanner; + for t in [ + "regclass", + "regproc", + "regtype", + "regnamespace", + "oid", + "pg_catalog.regclass", + "pg_catalog.regtype", + "regrole", + ] { + let dt = cast_target_type(&format!("SELECT 'x'::{t} AS c")); + let field = planner.plan_type_field(&dt).unwrap().unwrap(); + assert_eq!(field.data_type(), &DataType::Int32, "{t}"); + let kind = field.metadata().get(OID_ALIAS_KEY).cloned(); + assert!(kind.is_some(), "{t} should carry oid-alias metadata"); + assert!( + OID_ALIAS_TYPE_NAMES.contains(&kind.unwrap().as_str()), + "{t} kind must be canonical" + ); + } + } + + #[test] + fn leaves_non_oid_types_alone() { + let planner = PgOidTypePlanner; + for t in ["int4", "text", "varchar", "timestamp"] { + let dt = cast_target_type(&format!("SELECT 'x'::{t} AS c")); + assert_eq!(planner.plan_type_field(&dt).unwrap(), None, "{t}"); + } + } + + #[test] + fn pg_catalog_prefix_normalizes_to_kind() { + let planner = PgOidTypePlanner; + let dt = cast_target_type("SELECT 'x'::pg_catalog.regproc AS c"); + let field = planner.plan_type_field(&dt).unwrap().unwrap(); + assert_eq!( + field.metadata().get(OID_ALIAS_KEY).map(String::as_str), + Some(oid_field::kind::REGPROC) + ); + } + + #[test] + fn pg_catalog_builtins_map_to_arrow_types() { + let planner = PgOidTypePlanner; + for (sql, expected) in [ + ("pg_catalog.text", DataType::Utf8), + ("pg_catalog.varchar", DataType::Utf8), + ("pg_catalog.bpchar", DataType::Utf8), + ("pg_catalog.name", DataType::Utf8), + ("pg_catalog.bool", DataType::Boolean), + ("pg_catalog.boolean", DataType::Boolean), + ("pg_catalog.int2", DataType::Int16), + ("pg_catalog.smallint", DataType::Int16), + ("pg_catalog.int4", DataType::Int32), + ("pg_catalog.int8", DataType::Int64), + ("pg_catalog.bigint", DataType::Int64), + ("pg_catalog.float4", DataType::Float32), + ("pg_catalog.real", DataType::Float32), + ("pg_catalog.float8", DataType::Float64), + ("pg_catalog.bytea", DataType::Binary), + ] { + let dt = cast_target_type(&format!("SELECT 'x'::{sql} AS c")); + let field = planner + .plan_type_field(&dt) + .unwrap() + .unwrap_or_else(|| panic!("{sql} should be handled by the planner")); + assert_eq!(field.data_type(), &expected, "{sql}"); + // No oid-alias metadata for plain builtins. + assert!( + field.metadata().get(OID_ALIAS_KEY).is_none(), + "{sql} must not carry oid-alias metadata" + ); + } + } + + #[test] + fn pg_catalog_builtin_rejects_non_builtins_and_unknown_schemas() { + let planner = PgOidTypePlanner; + + // User-defined-looking custom type under pg_catalog is not mapped (left + // for DataFusion to error on / another rule to handle). + let dt = cast_target_type("SELECT 'x'::pg_catalog.some_udt AS c"); + assert_eq!(planner.plan_type_field(&dt).unwrap(), None); + + // A builtin name qualified by a different schema is not mapped either. + let dt = cast_target_type("SELECT 'x'::public.text AS c"); + assert_eq!(planner.plan_type_field(&dt).unwrap(), None); + + // `pg_catalog.regclass` is an oid-alias: handled by the oid arm (Int32 + // + metadata), NOT by the builtin arm. + let dt = cast_target_type("SELECT 'x'::pg_catalog.regclass AS c"); + let field = planner.plan_type_field(&dt).unwrap().unwrap(); + assert_eq!(field.data_type(), &DataType::Int32); + assert!(field.metadata().get(OID_ALIAS_KEY).is_some()); + } + + #[test] + fn pg_catalog_builtin_does_not_need_array_arm() { + // `pg_catalog.int2[]` parses as Array(Custom(pg_catalog.int2)). The + // planner only handles the scalar Custom element; DataFusion is the one + // that recurses into the planner for the element and wraps it into a + // list. So at the planner level, the Array itself returns None and only + // its element resolves -- proving no Array arm is required here. + let planner = PgOidTypePlanner; + let dt = cast_target_type("SELECT 'x'::pg_catalog.int2[] AS c"); + assert_eq!(planner.plan_type_field(&dt).unwrap(), None); + + // ...but the element type alone resolves to Int16. + let dt = cast_target_type("SELECT 'x'::pg_catalog.int2 AS c"); + let field = planner.plan_type_field(&dt).unwrap().unwrap(); + assert_eq!(field.data_type(), &DataType::Int16); + } +} diff --git a/datafusion-pg-catalog/src/pg_catalog/pg_get_expr_udf.rs b/datafusion-pg-catalog/src/pg_catalog/pg_get_expr_udf.rs index 7750536..510d38f 100644 --- a/datafusion-pg-catalog/src/pg_catalog/pg_get_expr_udf.rs +++ b/datafusion-pg-catalog/src/pg_catalog/pg_get_expr_udf.rs @@ -65,10 +65,6 @@ impl ScalarUDFImpl for PgGetExprUDF { Ok(ColumnarValue::Array(array)) } - - fn as_any(&self) -> &dyn std::any::Any { - self - } } pub fn create_pg_get_expr_udf() -> ScalarUDF { diff --git a/datafusion-pg-catalog/src/pg_catalog/quote_ident_udf.rs b/datafusion-pg-catalog/src/pg_catalog/quote_ident_udf.rs index f189e1b..1eeefde 100644 --- a/datafusion-pg-catalog/src/pg_catalog/quote_ident_udf.rs +++ b/datafusion-pg-catalog/src/pg_catalog/quote_ident_udf.rs @@ -81,10 +81,6 @@ impl Default for ParseIdentUDF { } impl ScalarUDFImpl for ParseIdentUDF { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn signature(&self) -> &Signature { &self.signature } diff --git a/datafusion-pg-catalog/src/sql/parser.rs b/datafusion-pg-catalog/src/sql/parser.rs index 57db57c..df40bb6 100644 --- a/datafusion-pg-catalog/src/sql/parser.rs +++ b/datafusion-pg-catalog/src/sql/parser.rs @@ -9,12 +9,12 @@ use datafusion::sql::sqlparser::tokenizer::Token; use datafusion::sql::sqlparser::tokenizer::TokenWithSpan; use super::rules::AliasDuplicatedProjectionRewrite; +use super::rules::CastArrayBoundsForGenerateSeries; use super::rules::CurrentUserVariableToSessionUserFunctionCall; use super::rules::FixArrayLiteral; use super::rules::FixCollate; use super::rules::FixVersionColumnName; use super::rules::PrependUnqualifiedPgTableName; -use super::rules::RemoveOidTypeCast; use super::rules::RemoveQualifier; use super::rules::RemoveSubqueryFromProjection; use super::rules::ResolveUnqualifiedIdentifer; @@ -229,7 +229,7 @@ impl PostgresCompatibilityParser { Arc::new(RewriteArrayAnyAllOperation), Arc::new(PrependUnqualifiedPgTableName), Arc::new(RemoveQualifier), - Arc::new(RemoveOidTypeCast::new()), + Arc::new(CastArrayBoundsForGenerateSeries), Arc::new(FixArrayLiteral), Arc::new(CurrentUserVariableToSessionUserFunctionCall), Arc::new(FixCollate), diff --git a/datafusion-pg-catalog/src/sql/rules.rs b/datafusion-pg-catalog/src/sql/rules.rs index 9aa18aa..f7c1295 100644 --- a/datafusion-pg-catalog/src/sql/rules.rs +++ b/datafusion-pg-catalog/src/sql/rules.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use std::fmt::Debug; use std::ops::ControlFlow; -use crate::pg_catalog::{PG_CATALOG_TABLES, oid_field}; +use crate::pg_catalog::PG_CATALOG_TABLES; use datafusion::sql::sqlparser::ast::Array; use datafusion::sql::sqlparser::ast::ArrayElemTypeDef; @@ -27,8 +27,8 @@ use datafusion::sql::sqlparser::ast::SelectItemQualifiedWildcardKind; use datafusion::sql::sqlparser::ast::SetExpr; use datafusion::sql::sqlparser::ast::Statement; use datafusion::sql::sqlparser::ast::TableFactor; +use datafusion::sql::sqlparser::ast::TableFunctionArgs; use datafusion::sql::sqlparser::ast::TableWithJoins; -use datafusion::sql::sqlparser::ast::TypedString; use datafusion::sql::sqlparser::ast::UnaryOperator; use datafusion::sql::sqlparser::ast::Value; use datafusion::sql::sqlparser::ast::ValueWithSpan; @@ -299,106 +299,76 @@ impl SqlStatementRewriteRule for ResolveUnqualifiedIdentifer { } } -/// Strip oid-alias cast type names (`::regclass`, `::regtype`, `::oid`, ...) -/// that DataFusion rejects as unsupported SQL types during `sql_to_plan`. +/// Cast `array_upper`/`array_lower` results to `bigint` when used as +/// `generate_series` arguments. /// -/// This runs at the SQL/AST layer -- *before* `sql_to_plan` -- because the -/// oid-coercion analyzer rule lives at the logical-plan layer and never gets a -/// chance to see these casts (planning fails first). The set of stripped type -/// names is derived from [`oid_field::OID_ALIAS_TYPE_NAMES`] so there is a -/// single source of truth shared with the oid-alias column metadata. -/// -/// Casts are peeled to a fixed point so nested forms like -/// `'x'::regclass::oid` fully unwind to `'x'` (the bare value, which the -/// oid-coercion rule then resolves against any oid-alias column it is compared -/// with) rather than stopping at the intermediate `'x'::regclass` that would -/// itself be rejected. +/// DataFusion's `generate_series` requires Int64 bounds, but our +/// `array_upper`/`array_lower` UDFs return Int32 (correct Postgres semantics -- +/// Postgres `array_upper` returns `int4`). Postgres implicitly coerces int4 to +/// int8 for `generate_series`; DataFusion does not, so without this rule +/// `generate_series(1, array_upper(current_schemas(false), 1))` fails with +/// "Argument #2 must be an INTEGER". This rule performs the coercion the way +/// Postgres would, removing the need to blacklist such client queries. #[derive(Debug)] -pub struct RemoveOidTypeCast { - unsupported_types: HashSet, -} +pub struct CastArrayBoundsForGenerateSeries; -impl Default for RemoveOidTypeCast { - fn default() -> Self { - Self::new() - } -} +struct CastArrayBoundsForGenerateSeriesVisitor; -impl RemoveOidTypeCast { - pub fn new() -> Self { - let mut unsupported_types = HashSet::new(); - // Single source of truth: every oid-alias type name, in four spellings - // (bare, `pg_catalog.`-qualified, array, and qualified-array) to match - // however sqlparser stringifies the cast's target type. - for name in oid_field::OID_ALIAS_TYPE_NAMES { - unsupported_types.insert((*name).to_string()); - unsupported_types.insert(format!("pg_catalog.{name}")); - unsupported_types.insert(format!("{name}[]")); - unsupported_types.insert(format!("pg_catalog.{name}[]")); +impl CastArrayBoundsForGenerateSeriesVisitor { + /// True when `expr` is a call to `array_upper`/`array_lower` (optionally + /// `pg_catalog.`-qualified) -- the Int32-returning UDFs that collide with + /// `generate_series`'s Int64 requirement. + fn is_array_bounds_call(expr: &Expr) -> bool { + if let Expr::Function(f) = expr + && let Some(ObjectNamePart::Identifier(ident)) = f.name.0.last() + { + let v = ident.value.to_lowercase(); + return v == "array_upper" || v == "array_lower"; } - Self { unsupported_types } + false } -} -struct RemoveOidTypeCastVisitor<'a> { - unsupported_types: &'a HashSet, + /// Wrap `expr` in `::bigint` if it's an array_bounds call. + fn maybe_cast(expr: &mut Expr) { + if Self::is_array_bounds_call(expr) { + let inner = std::mem::replace(expr, Expr::Value(Value::Null.with_empty_span())); + *expr = Expr::Cast { + expr: Box::new(inner), + kind: CastKind::DoubleColon, + data_type: DataType::BigInt(None), + array: false, + format: None, + }; + } + } } -impl VisitorMut for RemoveOidTypeCastVisitor<'_> { +impl VisitorMut for CastArrayBoundsForGenerateSeriesVisitor { type Break = (); - fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow { - match expr { - // This is the key part: identify constants with type annotations. - Expr::TypedString(TypedString { - data_type, - value, - uses_odbc_syntax: _, - }) if self - .unsupported_types - .contains(data_type.to_string().to_lowercase().as_str()) => - { - *expr = Expr::Value(Value::SingleQuotedString(value.to_string()).with_empty_span()); - } - Expr::Cast { - data_type, - expr: value, - .. - } if self - .unsupported_types - .contains(data_type.to_string().to_lowercase().as_str()) => - { - // Replace with the inner expression. The single-pass visitor - // won't re-visit this replacement, so the rule driver loops to - // a fixed point to fully peel nested casts like - // `'x'::regclass::oid` -> `'x'`. - *expr = *value.clone(); + fn pre_visit_table_factor(&mut self, tf: &mut TableFactor) -> ControlFlow { + if let TableFactor::Table { name, args, .. } = tf + && let Some(ObjectNamePart::Identifier(ident)) = name.0.last() + && ident.value.to_lowercase() == "generate_series" + && let Some(TableFunctionArgs { + args: func_args, .. + }) = args + { + for fa in func_args { + if let FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) = fa { + Self::maybe_cast(e); + } } - - // Add more match arms for other expression types (e.g., `Function`, `InList`) as needed. - _ => {} } - ControlFlow::Continue(()) } } -impl SqlStatementRewriteRule for RemoveOidTypeCast { - fn rewrite(&self, mut statement: Statement) -> Statement { - // The cast-peeling visitor may need multiple passes to fully unwind - // nested casts (`'x'::regclass::oid`); loop until a fixed point. - loop { - let mut visitor = RemoveOidTypeCastVisitor { - unsupported_types: &self.unsupported_types, - }; - let before = statement.to_string(); - let _ = statement.visit(&mut visitor); - let after = statement.to_string(); - if before == after { - break; - } - } - statement +impl SqlStatementRewriteRule for CastArrayBoundsForGenerateSeries { + fn rewrite(&self, mut s: Statement) -> Statement { + let mut visitor = CastArrayBoundsForGenerateSeriesVisitor; + let _ = s.visit(&mut visitor); + s } } @@ -635,10 +605,16 @@ impl SqlStatementRewriteRule for FixArrayLiteral { /// Remove qualifier from unsupported items /// -/// This rewriter removes qualifier from following items: -/// 1. type cast: for example: `pg_catalog.text` -/// 2. function name: for example: `pg_catalog.array_to_string`, -/// 3. table function name +/// This rewriter removes the `pg_catalog.` qualifier from: +/// 1. function names, for example `pg_catalog.array_to_string` +/// 2. table function names +/// +/// Type-cast qualifiers such as `pg_catalog.text` / `pg_catalog.int2[]` used +/// to be handled here too, but are now resolved by the [`PgOidTypePlanner`] +/// type planner (which sees the cast target type at planning time and maps the +/// pg name to its Arrow type). +/// +/// [`PgOidTypePlanner`]: crate::pg_catalog::oid_type_planner::PgOidTypePlanner #[derive(Debug)] pub struct RemoveQualifier; @@ -666,34 +642,14 @@ impl VisitorMut for RemoveQualifierVisitor { } fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow { - match expr { - Expr::Cast { data_type, .. } => { - // rewrite custom pg_catalog. qualified types - let data_type_str = data_type.to_string(); - match data_type_str.as_str() { - "pg_catalog.text" => { - *data_type = DataType::Text; - } - "pg_catalog.int2[]" => { - *data_type = DataType::Array(ArrayElemTypeDef::SquareBracket( - Box::new(DataType::Int16), - None, - )); - } - _ => {} - } - } - Expr::Function(function) => { - // remove qualifier from pg_catalog.function - let name = &mut function.name; - if name.0.len() > 1 - && let Some(last_ident) = name.0.pop() - { - *name = ObjectName(vec![last_ident]); - } + // remove qualifier from pg_catalog.function + if let Expr::Function(function) = expr { + let name = &mut function.name; + if name.0.len() > 1 + && let Some(last_ident) = name.0.pop() + { + *name = ObjectName(vec![last_ident]); } - - _ => {} } ControlFlow::Continue(()) } @@ -972,8 +928,7 @@ impl SqlStatementRewriteRule for FixVersionColumnName { mod tests { use super::*; use datafusion::sql::sqlparser::dialect::PostgreSqlDialect; - use datafusion::sql::sqlparser::parser::Parser; - use datafusion::sql::sqlparser::parser::ParserError; + use datafusion::sql::sqlparser::parser::{Parser, ParserError}; use std::sync::Arc; fn parse(sql: &str) -> Result, ParserError> { @@ -1064,70 +1019,34 @@ mod tests { } #[test] - fn test_remove_oid_type_cast() { - let rules: Vec> = vec![ - Arc::new(RemoveQualifier), - Arc::new(RemoveOidTypeCast::new()), - ]; - assert_rewrite!( - &rules, - "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog'::regclass ORDER BY n.nspname", - "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname" - ); - - assert_rewrite!( - &rules, - "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.oid = 1 AND n.nspname = 'pg_catalog'::regclass ORDER BY n.nspname", - "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.oid = 1 AND n.nspname = 'pg_catalog' ORDER BY n.nspname" - ); - - assert_rewrite!( - &rules, - "SELECT n.oid, n.*, d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace'::regclass ORDER BY nspname", - "SELECT n.oid, n.*, d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace' ORDER BY nspname" - ); + fn test_cast_array_bounds_for_generate_series() { + // array_upper/array_lower return Int32; generate_series wants Int64. + // The rule wraps those calls in ::bigint when used as generate_series args. + let rules: Vec> = + vec![Arc::new(CastArrayBoundsForGenerateSeries)]; assert_rewrite!( &rules, - "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname", - "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname" + "SELECT s.r FROM generate_series(1, array_upper(current_schemas(false), 1)) as s(r)", + "SELECT s.r FROM generate_series(1, array_upper(current_schemas(false), 1)::BIGINT) AS s (r)" ); - assert_rewrite!( &rules, - "SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, false AS relhasoids, c.relispartition, '', c.reltablespace, CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::pg_catalog.regtype::pg_catalog.text END, c.relpersistence, c.relreplident, am.amname - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) - LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) - WHERE c.oid = '16386'", - "SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, false AS relhasoids, c.relispartition, '', c.reltablespace, CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::TEXT END, c.relpersistence, c.relreplident, am.amname FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) WHERE c.oid = '16386'" + "SELECT s.r FROM generate_series(array_lower(x, 1), array_upper(x, 1)) as s(r)", + "SELECT s.r FROM generate_series(array_lower(x, 1)::BIGINT, array_upper(x, 1)::BIGINT) AS s (r)" ); - // Regression: nested casts must peel to a fixed point so that - // `'x'::regclass::oid` -> `'x'` (not the partial `'x'::regclass`, - // which DataFusion rejects as an unsupported SQL type at plan time). - // The oid-coercion analyzer rule then resolves the bare name against - // any oid-alias column it is compared with. + // Non-array-bounds args are left alone (literal ints already work). assert_rewrite!( &rules, - "SELECT * FROM pg_catalog.pg_class WHERE oid = 't'::pg_catalog.regclass::pg_catalog.oid", - "SELECT * FROM pg_catalog.pg_class WHERE oid = 't'" + "SELECT s.r FROM generate_series(1, 10) as s(r)", + "SELECT s.r FROM generate_series(1, 10) AS s (r)" ); + // array_upper used outside generate_series is NOT touched. assert_rewrite!( &rules, - "SELECT 'pg_namespace'::regclass::oid", - "SELECT 'pg_namespace'" - ); - - // The stripped set is derived from oid_field::OID_ALIAS_TYPE_NAMES, so - // every oid-alias type -- not just the original hardcoded six -- is - // peeled. Verify a representative newly-covered one (regrole) plus the - // array variant (regtype[], used by pgcli's proallargtypes). - assert_rewrite!(&rules, "SELECT 'postgres'::regrole", "SELECT 'postgres'"); - assert_rewrite!( - &rules, - "SELECT proallargtypes::regtype[] FROM pg_proc", - "SELECT proallargtypes FROM pg_proc" + "SELECT array_upper(x, 1)", + "SELECT array_upper(x, 1)" ); } diff --git a/datafusion-postgres/src/hooks/permissions.rs b/datafusion-postgres/src/hooks/permissions.rs index 42f7594..8d5172d 100644 --- a/datafusion-postgres/src/hooks/permissions.rs +++ b/datafusion-postgres/src/hooks/permissions.rs @@ -82,6 +82,7 @@ impl PermissionsHook { Statement::Set { .. } | Statement::ShowVariable { .. } | Statement::ShowStatus { .. } + | Statement::ShowCatalogs { .. } | Statement::StartTransaction { .. } | Statement::Commit { .. } | Statement::Rollback { .. } diff --git a/datafusion-postgres/src/hooks/set_show.rs b/datafusion-postgres/src/hooks/set_show.rs index 91fd36d..82abe2d 100644 --- a/datafusion-postgres/src/hooks/set_show.rs +++ b/datafusion-postgres/src/hooks/set_show.rs @@ -37,7 +37,9 @@ impl QueryHook for SetShowHook { Statement::Set { .. } => { try_respond_set_statements(client, statement, session_context).await } - Statement::ShowVariable { .. } | Statement::ShowStatus { .. } => { + Statement::ShowVariable { .. } + | Statement::ShowStatus { .. } + | Statement::ShowCatalogs { .. } => { try_respond_show_statements(client, statement, session_context).await } _ => None, @@ -64,7 +66,9 @@ impl QueryHook for SetShowHook { .map_err(|e| PgWireError::ApiError(Box::new(e))); Some(result) } - Statement::ShowVariable { .. } | Statement::ShowStatus { .. } => { + Statement::ShowVariable { .. } + | Statement::ShowStatus { .. } + | Statement::ShowCatalogs { .. } => { let show_schema = Arc::new(Schema::new(vec![Field::new("show", DataType::Utf8, false)])); let result = show_schema @@ -94,7 +98,9 @@ impl QueryHook for SetShowHook { Statement::Set { .. } => { try_respond_set_statements(client, statement, session_context).await } - Statement::ShowVariable { .. } | Statement::ShowStatus { .. } => { + Statement::ShowVariable { .. } + | Statement::ShowStatus { .. } + | Statement::ShowCatalogs { .. } => { try_respond_show_statements(client, statement, session_context).await } _ => None, @@ -279,6 +285,13 @@ async fn try_respond_show_statements( statement: &Statement, session_context: &SessionContext, ) -> Option> { + // Handle SHOW CATALOGS separately since it's its own variant in sqlparser 0.62+ + if let Statement::ShowCatalogs { .. } = statement { + let catalogs = session_context.catalog_names(); + let value = catalogs.join(", "); + return Some(mock_show_response("Catalogs", &value).map(Response::Query)); + } + let Statement::ShowVariable { variable } = statement else { return None; }; @@ -596,6 +609,8 @@ mod tests { let show_response = try_respond_show_statements(&client, &statement, &session_context).await; + dbg!(query); + dbg!(&show_response); let Some(Ok(Response::Query(show_response))) = show_response else { panic!("unexpected show response"); }; diff --git a/datafusion-postgres/src/planner.rs b/datafusion-postgres/src/planner.rs index 290015e..002e872 100644 --- a/datafusion-postgres/src/planner.rs +++ b/datafusion-postgres/src/planner.rs @@ -16,7 +16,7 @@ fn extract_placeholder_cast_types(plan: &LogicalPlan) -> Result rewrite -> plan -> execute path and prove the +//! UDFs are registered and resolvable. Before these UDFs existed, such queries +//! failed at planning with "Invalid function 'array_upper'". + +use pgwire::api::query::SimpleQueryHandler; + +use datafusion_postgres::testing::*; + +const QUERIES: &[&str] = &[ + // Bare UDFs over an array literal. + "SELECT array_upper(ARRAY[1, 2, 3], 1), array_lower(ARRAY[1, 2, 3], 1)", + // pg_catalog-qualified form (qualifier stripped by RemoveQualifier rule). + "SELECT pg_catalog.array_upper(ARRAY['a', 'b'], 1)", + "SELECT pg_catalog.array_lower(ARRAY['a', 'b'], 1)", + // Typical usage pattern: generate_series over array bounds, mirroring the + // shape grafana/psql clients send (but without matching the blacklisted + // fragment verbatim). + "SELECT array_upper(string_to_array('public,utils', ','), 1)", + "SELECT array_lower(string_to_array('public,utils', ','), 1)", +]; + +#[tokio::test] +pub async fn test_array_bounds_udfs() { + env_logger::init(); + let service = setup_handlers(); + let mut client = MockClient::new(); + + for query in QUERIES { + SimpleQueryHandler::do_query(&service, &mut client, query) + .await + .unwrap_or_else(|e| { + panic!("failed to run sql:\n--------------\n {query}\n--------------\n{e}") + }); + } +}