From 051f7c283fe3ebc53f00eb54a9e8f17e26fae939 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Sun, 2 Aug 2026 21:34:01 -0400 Subject: [PATCH 1/4] Perf: remove extra allocations for hex operations --- datafusion/functions/src/crypto/md5.rs | 21 ++++--- datafusion/spark/src/function/hash/sha2.rs | 62 +++++++++++-------- datafusion/spark/src/function/string/quote.rs | 6 +- 3 files changed, 53 insertions(+), 36 deletions(-) diff --git a/datafusion/functions/src/crypto/md5.rs b/datafusion/functions/src/crypto/md5.rs index b1206d2e423cc..b34155c403b0a 100644 --- a/datafusion/functions/src/crypto/md5.rs +++ b/datafusion/functions/src/crypto/md5.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -use arrow::{array::StringViewArray, datatypes::DataType}; +use arrow::{ + array::{Array, StringViewArray}, + datatypes::DataType, +}; use datafusion_common::{ Result, ScalarValue, cast::as_binary_array, @@ -32,7 +35,10 @@ use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; use datafusion_macros::user_doc; use std::sync::Arc; -use crate::crypto::basic::{DigestAlgorithm, digest_process}; +use crate::{ + crypto::basic::{DigestAlgorithm, digest_process}, + strings::StringViewArrayBuilder, +}; #[user_doc( doc_section(label = "Hashing Functions"), @@ -107,11 +113,12 @@ fn md5(args: &[ColumnarValue]) -> Result { Ok(match value { ColumnarValue::Array(array) => { let binary_array = as_binary_array(&array)?; - let string_array: StringViewArray = binary_array - .iter() - .map(|opt| opt.map(|b| encode_bytes(b, HexCase::Lower))) - .collect(); - ColumnarValue::Array(Arc::new(string_array)) + let mut string_builder = + StringViewArrayBuilder::with_capacity(binary_array.len()); + binary_array.iter().flatten().for_each(|b| { + string_builder.append_value(&encode_bytes(b, HexCase::Lower)); + }); + ColumnarValue::Array(Arc::new(string_builder.finish(None)?)) } ColumnarValue::Scalar(ScalarValue::Binary(opt)) => ColumnarValue::Scalar( ScalarValue::Utf8View(opt.map(|b| encode_bytes(&b, HexCase::Lower))), diff --git a/datafusion/spark/src/function/hash/sha2.rs b/datafusion/spark/src/function/hash/sha2.rs index 541df2957669e..be921ac80cc36 100644 --- a/datafusion/spark/src/function/hash/sha2.rs +++ b/datafusion/spark/src/function/hash/sha2.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{ArrayRef, AsArray, BinaryArrayType, Int32Array, StringArray}; +use arrow::array::{ArrayRef, AsArray, BinaryArrayType, Int32Array, StringBuilder}; use arrow::datatypes::{DataType, Int32Type}; use datafusion_common::types::{ NativeType, logical_binary, logical_int32, logical_string, @@ -216,33 +216,41 @@ where BinaryArrType: BinaryArrayType<'a>, I: Iterator>, { - let array = values + let mut builder = StringBuilder::with_capacity(values.len(), values.len() * 128); + + values .iter() .zip(bit_lengths) - .map(|(value, bit_length)| match (value, bit_length) { - (Some(value), Some(224)) => { - let mut digest = sha2::Sha224::default(); - digest.update(value); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) - } - (Some(value), Some(0 | 256)) => { - let mut digest = sha2::Sha256::default(); - digest.update(value); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) - } - (Some(value), Some(384)) => { - let mut digest = sha2::Sha384::default(); - digest.update(value); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) - } - (Some(value), Some(512)) => { - let mut digest = sha2::Sha512::default(); - digest.update(value); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + .for_each(|(value, bit_length)| { + match (value, bit_length) { + (Some(value), Some(224)) => { + let mut digest = sha2::Sha224::default(); + digest.update(value); + builder + .append_value(&encode_bytes(&digest.finalize(), HexCase::Lower)); + } + (Some(value), Some(0 | 256)) => { + let mut digest = sha2::Sha256::default(); + digest.update(value); + builder + .append_value(&encode_bytes(&digest.finalize(), HexCase::Lower)); + } + (Some(value), Some(384)) => { + let mut digest = sha2::Sha384::default(); + digest.update(value); + builder + .append_value(&encode_bytes(&digest.finalize(), HexCase::Lower)); + } + (Some(value), Some(512)) => { + let mut digest = sha2::Sha512::default(); + digest.update(value); + builder + .append_value(&encode_bytes(&digest.finalize(), HexCase::Lower)); + } + // Unknown bit-lengths go to null, same as in Spark + _ => builder.append_null(), } - // Unknown bit-lengths go to null, same as in Spark - _ => None, - }) - .collect::(); - Arc::new(array) + }); + + Arc::new(builder.finish()) } diff --git a/datafusion/spark/src/function/string/quote.rs b/datafusion/spark/src/function/string/quote.rs index 39ad8bf841764..55d827fb7a280 100644 --- a/datafusion/spark/src/function/string/quote.rs +++ b/datafusion/spark/src/function/string/quote.rs @@ -17,12 +17,14 @@ use arrow::array::{ArrayRef, OffsetSizeTrait, StringArray}; use arrow::datatypes::DataType; -use datafusion::logical_expr::{Coercion, ColumnarValue, Signature, TypeSignatureClass}; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; use datafusion_common::types::{NativeType, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err}; -use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Volatility}; +use datafusion_expr::{ + Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignatureClass, Volatility, +}; use datafusion_functions::utils::make_scalar_function; use std::sync::Arc; From 9074fa84dbb2b6211532eb6daa1c81a9291d4206 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Mon, 3 Aug 2026 00:53:15 -0400 Subject: [PATCH 2/4] re-use vectors --- datafusion/functions/src/crypto/md5.rs | 46 +++++++++---- datafusion/spark/src/function/hash/sha2.rs | 79 +++++++++++++++++----- 2 files changed, 93 insertions(+), 32 deletions(-) diff --git a/datafusion/functions/src/crypto/md5.rs b/datafusion/functions/src/crypto/md5.rs index b34155c403b0a..c9aba15e6ef22 100644 --- a/datafusion/functions/src/crypto/md5.rs +++ b/datafusion/functions/src/crypto/md5.rs @@ -16,7 +16,7 @@ // under the License. use arrow::{ - array::{Array, StringViewArray}, + array::{Array, BinaryViewBuilder}, datatypes::DataType, }; use datafusion_common::{ @@ -24,7 +24,7 @@ use datafusion_common::{ cast::as_binary_array, internal_err, types::{logical_binary, logical_string}, - utils::hex::{HexCase, encode_bytes}, + utils::hex::{HexCase, encode_bytes_into}, utils::take_function_args, }; use datafusion_expr::{ @@ -35,10 +35,7 @@ use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; use datafusion_macros::user_doc; use std::sync::Arc; -use crate::{ - crypto::basic::{DigestAlgorithm, digest_process}, - strings::StringViewArrayBuilder, -}; +use crate::crypto::basic::{DigestAlgorithm, digest_process}; #[user_doc( doc_section(label = "Hashing Functions"), @@ -113,16 +110,35 @@ fn md5(args: &[ColumnarValue]) -> Result { Ok(match value { ColumnarValue::Array(array) => { let binary_array = as_binary_array(&array)?; - let mut string_builder = - StringViewArrayBuilder::with_capacity(binary_array.len()); - binary_array.iter().flatten().for_each(|b| { - string_builder.append_value(&encode_bytes(b, HexCase::Lower)); - }); - ColumnarValue::Array(Arc::new(string_builder.finish(None)?)) + let mut byte_builder = BinaryViewBuilder::with_capacity(binary_array.len()); + let mut hex_bytes = Vec::with_capacity(32); + + for i in 0..binary_array.len() { + if binary_array.is_null(i) { + byte_builder.append_null(); + continue; + } + + hex_bytes.clear(); + let digest = binary_array.value(i); + encode_bytes_into(digest, HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); + } + + let str_array = unsafe { + // Safe: `encode_bytes_into` only writes ASCII hex digits, so the bytes are valid UTF-8. + byte_builder.finish().to_string_view_unchecked() + }; + ColumnarValue::Array(Arc::new(str_array)) + } + ColumnarValue::Scalar(ScalarValue::Binary(opt)) => { + ColumnarValue::Scalar(ScalarValue::Utf8View(opt.map(|b| { + let mut hex_bytes = Vec::with_capacity(b.len() * 2); + encode_bytes_into(&b, HexCase::Lower, &mut hex_bytes); + // Safe: `encode_bytes_into` only writes ASCII hex digits, so the bytes are valid UTF-8. + unsafe { String::from_utf8_unchecked(hex_bytes) } + }))) } - ColumnarValue::Scalar(ScalarValue::Binary(opt)) => ColumnarValue::Scalar( - ScalarValue::Utf8View(opt.map(|b| encode_bytes(&b, HexCase::Lower))), - ), _ => return internal_err!("Impossibly got invalid results from digest"), }) } diff --git a/datafusion/spark/src/function/hash/sha2.rs b/datafusion/spark/src/function/hash/sha2.rs index be921ac80cc36..96eb29800a802 100644 --- a/datafusion/spark/src/function/hash/sha2.rs +++ b/datafusion/spark/src/function/hash/sha2.rs @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{ArrayRef, AsArray, BinaryArrayType, Int32Array, StringBuilder}; +use arrow::array::{ArrayRef, AsArray, BinaryArrayType, BinaryViewBuilder, Int32Array}; use arrow::datatypes::{DataType, Int32Type}; use datafusion_common::types::{ NativeType, logical_binary, logical_int32, logical_string, }; -use datafusion_common::utils::hex::{HexCase, encode_bytes}; +use datafusion_common::utils::hex::{HexCase, encode_bytes_into}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ @@ -113,22 +113,58 @@ impl ScalarUDFImpl for SparkSha2 { 224 => { let mut digest = sha2::Sha224::default(); digest.update(bytes); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + let mut hex_bytes = Vec::with_capacity(56); + encode_bytes_into( + &digest.finalize(), + HexCase::Lower, + &mut hex_bytes, + ); + Some( + String::from_utf8(hex_bytes) + .expect("ASCII hex is valid UTF-8"), + ) } 0 | 256 => { let mut digest = sha2::Sha256::default(); digest.update(bytes); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + let mut hex_bytes = Vec::with_capacity(64); + encode_bytes_into( + &digest.finalize(), + HexCase::Lower, + &mut hex_bytes, + ); + Some( + String::from_utf8(hex_bytes) + .expect("ASCII hex is valid UTF-8"), + ) } 384 => { let mut digest = sha2::Sha384::default(); digest.update(bytes); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + let mut hex_bytes = Vec::with_capacity(96); + encode_bytes_into( + &digest.finalize(), + HexCase::Lower, + &mut hex_bytes, + ); + Some( + String::from_utf8(hex_bytes) + .expect("ASCII hex is valid UTF-8"), + ) } 512 => { let mut digest = sha2::Sha512::default(); digest.update(bytes); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + let mut hex_bytes = Vec::with_capacity(128); + encode_bytes_into( + &digest.finalize(), + HexCase::Lower, + &mut hex_bytes, + ); + Some( + String::from_utf8(hex_bytes) + .expect("ASCII hex is valid UTF-8"), + ) } _ => None, }; @@ -216,7 +252,8 @@ where BinaryArrType: BinaryArrayType<'a>, I: Iterator>, { - let mut builder = StringBuilder::with_capacity(values.len(), values.len() * 128); + let mut byte_builder = BinaryViewBuilder::with_capacity(values.len()); + let mut hex_bytes = Vec::with_capacity(128); values .iter() @@ -226,31 +263,39 @@ where (Some(value), Some(224)) => { let mut digest = sha2::Sha224::default(); digest.update(value); - builder - .append_value(&encode_bytes(&digest.finalize(), HexCase::Lower)); + hex_bytes.clear(); + encode_bytes_into(&digest.finalize(), HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); } (Some(value), Some(0 | 256)) => { let mut digest = sha2::Sha256::default(); digest.update(value); - builder - .append_value(&encode_bytes(&digest.finalize(), HexCase::Lower)); + hex_bytes.clear(); + encode_bytes_into(&digest.finalize(), HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); } (Some(value), Some(384)) => { let mut digest = sha2::Sha384::default(); digest.update(value); - builder - .append_value(&encode_bytes(&digest.finalize(), HexCase::Lower)); + hex_bytes.clear(); + encode_bytes_into(&digest.finalize(), HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); } (Some(value), Some(512)) => { let mut digest = sha2::Sha512::default(); digest.update(value); - builder - .append_value(&encode_bytes(&digest.finalize(), HexCase::Lower)); + hex_bytes.clear(); + encode_bytes_into(&digest.finalize(), HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); } // Unknown bit-lengths go to null, same as in Spark - _ => builder.append_null(), + _ => byte_builder.append_null(), } }); - Arc::new(builder.finish()) + let str_array = unsafe { + // Safe: `encode_bytes_into` only writes ASCII hex digits, so the bytes are valid UTF-8. + byte_builder.finish().to_string_view_unchecked() + }; + Arc::new(str_array) } From 4b880e1451f91032847fe1e08bd9ff2d95af968c Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Mon, 3 Aug 2026 01:19:47 -0400 Subject: [PATCH 3/4] deprecate encode_bytes --- datafusion/common/src/utils/hex.rs | 4 ++++ datafusion/functions/src/encoding/inner.rs | 8 ++++++-- datafusion/spark/src/function/hash/sha1.rs | 7 +++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/datafusion/common/src/utils/hex.rs b/datafusion/common/src/utils/hex.rs index 872d54f40c6f7..3b08697469077 100644 --- a/datafusion/common/src/utils/hex.rs +++ b/datafusion/common/src/utils/hex.rs @@ -161,6 +161,9 @@ pub fn encode_bytes_to_slice(bytes: &[u8], case: HexCase, out: &mut [u8]) -> Res /// Returns the hex encoding of `bytes` as an owned `String`. /// +/// Prefer [`encode_bytes_into`] when you already have a reusable output buffer. +#[deprecated(note = "use encode_bytes_into or encode_bytes_to_slice instead")] +/// /// # Example /// /// ``` @@ -237,6 +240,7 @@ fn write_digits(v: u64, case: HexCase, buf: &mut [u8; 16]) -> usize { } #[cfg(test)] +#[expect(deprecated)] mod tests { use super::*; diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs index 850e312abdb40..1f809b4e5da40 100644 --- a/datafusion/functions/src/encoding/inner.rs +++ b/datafusion/functions/src/encoding/inner.rs @@ -34,7 +34,7 @@ use datafusion_common::{ not_impl_err, plan_err, types::{NativeType, logical_string}, utils::{ - hex::{HexCase, encode_bytes as encode_hex, encode_bytes_to_slice}, + hex::{HexCase, encode_bytes_into, encode_bytes_to_slice}, take_function_args, }, }; @@ -373,7 +373,11 @@ impl Encoding { match self { Self::Base64 => BASE64_ENGINE.encode(value), Self::Base64Padded => BASE64_ENGINE_PADDED.encode(value), - Self::Hex => encode_hex(value, HexCase::Lower), + Self::Hex => { + let mut out = Vec::with_capacity(value.len() * 2); + encode_bytes_into(value, HexCase::Lower, &mut out); + unsafe { String::from_utf8_unchecked(out) } + } } } diff --git a/datafusion/spark/src/function/hash/sha1.rs b/datafusion/spark/src/function/hash/sha1.rs index 05a224f33f25a..64c52a7108283 100644 --- a/datafusion/spark/src/function/hash/sha1.rs +++ b/datafusion/spark/src/function/hash/sha1.rs @@ -24,7 +24,7 @@ use datafusion_common::cast::{ as_large_binary_array, }; use datafusion_common::types::{NativeType, logical_string}; -use datafusion_common::utils::hex::{HexCase, encode_bytes}; +use datafusion_common::utils::hex::{HexCase, encode_bytes_into}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -92,7 +92,10 @@ impl ScalarUDFImpl for SparkSha1 { #[inline] fn spark_sha1_digest(value: &[u8]) -> String { - encode_bytes(&Sha1::digest(value), HexCase::Lower) + let mut out = Vec::with_capacity(40); + // Safe: `encode_bytes_into` only writes ASCII hex digits, which are valid UTF-8. + encode_bytes_into(&Sha1::digest(value), HexCase::Lower, &mut out); + unsafe { String::from_utf8_unchecked(out) } } fn spark_sha1_impl<'a>(input: impl Iterator>) -> ArrayRef { From cd5492e46579f8ecc1548e1105110f9d0bff20a9 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Mon, 3 Aug 2026 02:03:28 -0400 Subject: [PATCH 4/4] fix bug --- datafusion/spark/src/function/hash/sha2.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datafusion/spark/src/function/hash/sha2.rs b/datafusion/spark/src/function/hash/sha2.rs index 96eb29800a802..53c1ff9465781 100644 --- a/datafusion/spark/src/function/hash/sha2.rs +++ b/datafusion/spark/src/function/hash/sha2.rs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{ArrayRef, AsArray, BinaryArrayType, BinaryViewBuilder, Int32Array}; +use arrow::array::{ + ArrayRef, AsArray, BinaryArrayType, BinaryBuilder, Int32Array, StringArray, +}; use arrow::datatypes::{DataType, Int32Type}; use datafusion_common::types::{ NativeType, logical_binary, logical_int32, logical_string, @@ -252,7 +254,7 @@ where BinaryArrType: BinaryArrayType<'a>, I: Iterator>, { - let mut byte_builder = BinaryViewBuilder::with_capacity(values.len()); + let mut byte_builder = BinaryBuilder::with_capacity(values.len(), values.len() * 2); let mut hex_bytes = Vec::with_capacity(128); values @@ -294,8 +296,10 @@ where }); let str_array = unsafe { + let binary_array = byte_builder.finish(); + let (offsets, values, nulls) = binary_array.into_parts(); // Safe: `encode_bytes_into` only writes ASCII hex digits, so the bytes are valid UTF-8. - byte_builder.finish().to_string_view_unchecked() + StringArray::new_unchecked(offsets, values, nulls) }; Arc::new(str_array) }