From b6e40bbd4507068e66e10578f139b51549f650db Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 11:37:58 -0500 Subject: [PATCH 01/12] feat: add with_child convenience method to dyn Array Adds a method to replace a single child of an array by index, building on the existing with_children infrastructure. This is needed by the upcoming iterative execution scheduler which replaces children one at a time as they are executed. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- vortex-array/src/array/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index d5159edbc9f..1f57b177bca 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -352,6 +352,19 @@ impl dyn Array + '_ { pub fn is_canonical(&self) -> bool { self.is::() } + + /// Returns a new array with the child at `child_idx` replaced by `replacement`. + pub fn with_child(&self, child_idx: usize, replacement: ArrayRef) -> VortexResult { + let mut children: Vec = self.children(); + vortex_ensure!( + child_idx < children.len(), + "child index {} out of bounds for array with {} children", + child_idx, + children.len() + ); + children[child_idx] = replacement; + self.with_children(children) + } } /// Trait for converting a type into a Vortex [`ArrayRef`]. From 0516da20353913a8ff567fd1f3314c9742401180 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 11:39:53 -0500 Subject: [PATCH 02/12] feat: add ExecutionStep enum for iterative array execution Adds the ExecutionStep enum (ExecuteChild, ColumnarizeChild, Done) that encodings will return from VTable::execute instead of ArrayRef. This is infrastructure for the upcoming iterative execution scheduler. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- vortex-array/src/executor.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 58e128d253f..8dfe549dd1a 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -195,6 +195,33 @@ impl Executable for ArrayRef { } } +/// The result of a single execution step on an array encoding. +/// +/// Instead of recursively executing children, encodings return an `ExecutionStep` that tells the +/// scheduler what to do next. This enables the scheduler to manage execution iteratively using +/// an explicit work stack, run cross-step optimizations, and cache shared sub-expressions. +#[derive(Debug)] +pub enum ExecutionStep { + /// Request that the scheduler execute child at index `i` to columnar form, replace it in + /// this array, then re-enter execution on the updated array. + /// + /// Between steps, the scheduler runs reduce/reduce_parent rules to fixpoint, enabling + /// cross-step optimization (e.g., pushing scalar functions through newly-decoded children). + ExecuteChild(usize), + + /// Request that the scheduler execute child at index `i` to columnar form, replace it in + /// this array, then re-enter execution on the updated array. + /// + /// Unlike [`ExecuteChild`](Self::ExecuteChild), the scheduler does **not** run cross-step + /// optimizations when popping back up. Use this when the parent knows it will consume the + /// child directly (e.g., Dict taking from its values). + ColumnarizeChild(usize), + + /// Execution is complete. The result may be in any encoding — not necessarily canonical. + /// The scheduler will continue executing the result if it is not yet columnar. + Done(ArrayRef), +} + /// Extension trait for creating an execution context from a session. pub trait VortexSessionExecute { /// Create a new execution context from this session. From a675c0439435aa4d9a7a3885952e3731608d4445 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 12:27:38 -0500 Subject: [PATCH 03/12] feat: change VTable::execute to return ExecutionStep + iterative scheduler Changes the VTable::execute signature to return ExecutionStep instead of ArrayRef, and replaces the Executable for Columnar implementation with an iterative work-stack scheduler. The ExecutionStep enum has three variants: - ExecuteChild(i): ask the scheduler to execute child i to columnar - ColumnarizeChild(i): same but skip cross-step optimization - Done(result): execution complete The new scheduler in Executable for Columnar uses an explicit stack instead of recursion, and runs reduce/reduce_parent rules between steps via the existing optimizer infrastructure. All encoding implementations are mechanically wrapped in ExecutionStep::Done(...) to preserve existing behavior. Individual encodings will be migrated to use ExecuteChild/ColumnarizeChild in follow-up PRs. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/alp/src/alp/array.rs | 7 +- encodings/alp/src/alp_rd/array.rs | 5 +- encodings/bytebool/src/array.rs | 7 +- encodings/datetime-parts/src/array.rs | 7 +- .../src/decimal_byte_parts/mod.rs | 5 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 5 +- encodings/fastlanes/src/delta/vtable/mod.rs | 7 +- encodings/fastlanes/src/for/vtable/mod.rs | 5 +- encodings/fastlanes/src/rle/vtable/mod.rs | 7 +- encodings/fsst/src/array.rs | 5 +- encodings/pco/src/array.rs | 5 +- encodings/runend/src/array.rs | 5 +- encodings/sequence/src/array.rs | 5 +- encodings/sparse/src/lib.rs | 5 +- encodings/zigzag/src/array.rs | 7 +- encodings/zstd/src/array.rs | 8 +- encodings/zstd/src/zstd_buffers.rs | 7 +- vortex-array/src/arrays/bool/vtable/mod.rs | 5 +- vortex-array/src/arrays/chunked/vtable/mod.rs | 5 +- .../src/arrays/constant/vtable/mod.rs | 7 +- vortex-array/src/arrays/decimal/vtable/mod.rs | 5 +- vortex-array/src/arrays/dict/vtable/mod.rs | 9 +- .../src/arrays/extension/vtable/mod.rs | 5 +- vortex-array/src/arrays/filter/vtable.rs | 9 +- .../src/arrays/fixed_size_list/vtable/mod.rs | 5 +- vortex-array/src/arrays/list/vtable/mod.rs | 7 +- .../src/arrays/listview/vtable/mod.rs | 5 +- vortex-array/src/arrays/masked/vtable/mod.rs | 11 +- vortex-array/src/arrays/null/mod.rs | 5 +- .../src/arrays/primitive/vtable/mod.rs | 5 +- .../src/arrays/scalar_fn/vtable/mod.rs | 5 +- vortex-array/src/arrays/shared/vtable.rs | 7 +- vortex-array/src/arrays/slice/vtable.rs | 7 +- vortex-array/src/arrays/struct_/vtable/mod.rs | 5 +- vortex-array/src/arrays/varbin/vtable/mod.rs | 7 +- .../src/arrays/varbinview/vtable/mod.rs | 5 +- vortex-array/src/columnar.rs | 110 +++++++++++++++--- vortex-array/src/executor.rs | 29 +++-- vortex-array/src/vtable/dyn_.rs | 47 ++++---- vortex-array/src/vtable/mod.rs | 27 ++--- vortex-python/src/arrays/py/vtable.rs | 3 +- 41 files changed, 297 insertions(+), 140 deletions(-) diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 055bc8583db..da337d97f91 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -234,9 +235,11 @@ impl VTable for ALPVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { // TODO(joe): take by value - Ok(execute_decompress(array.clone(), ctx)?.into_array()) + Ok(ExecutionStep::Done( + execute_decompress(array.clone(), ctx)?.into_array(), + )) } fn reduce_parent( diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 3f351fec007..3bd7cdc6812 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -11,6 +11,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -295,7 +296,7 @@ impl VTable for ALPRDVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let left_parts = array.left_parts().clone().execute::(ctx)?; let right_parts = array.right_parts().clone().execute::(ctx)?; @@ -334,7 +335,7 @@ impl VTable for ALPRDVTable { ) }; - Ok(decoded_array.into_array()) + Ok(ExecutionStep::Done(decoded_array.into_array())) } fn reduce_parent( diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index c7e21795a12..8c5c7c9fa31 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -9,6 +9,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::arrays::BoolArray; @@ -182,10 +183,12 @@ impl VTable for ByteBoolVTable { crate::rules::RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { let boolean_buffer = BitBuffer::from(array.as_slice()); let validity = array.validity().clone(); - Ok(BoolArray::new(boolean_buffer, validity).into_array()) + Ok(ExecutionStep::Done( + BoolArray::new(boolean_buffer, validity).into_array(), + )) } fn execute_parent( diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 0290ec9d0b9..c9e292d5498 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -221,8 +222,10 @@ impl VTable for DateTimePartsVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(decode_to_temporal(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + decode_to_temporal(array, ctx)?.into_array(), + )) } fn reduce_parent( diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 51a5e62c10e..9bfbd9c21de 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -13,6 +13,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -189,8 +190,8 @@ impl VTable for DecimalBytePartsVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - to_canonical_decimal(array, ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + to_canonical_decimal(array, ctx).map(ExecutionStep::Done) } fn execute_parent( diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index d213fb9f1ed..facb2aa5a4f 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -8,6 +8,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -353,8 +354,8 @@ impl VTable for BitPackedVTable { }) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(unpack_array(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(unpack_array(array, ctx)?.into_array())) } fn execute_parent( diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 96c0e0dd248..5bf67a561c5 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -9,6 +9,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -189,8 +190,10 @@ impl VTable for DeltaVTable { DeltaArray::try_new(bases, deltas, metadata.0.offset as usize, len) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(delta_decompress(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + delta_decompress(array, ctx)?.into_array(), + )) } } diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 37373456746..79740ca3061 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -8,6 +8,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::buffer::BufferHandle; @@ -165,8 +166,8 @@ impl VTable for FoRVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(decompress(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(decompress(array, ctx)?.into_array())) } fn execute_parent( diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 0a682d3abf7..ab12a22294d 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -8,6 +8,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -230,8 +231,10 @@ impl VTable for RLEVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(rle_decompress(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + rle_decompress(array, ctx)?.into_array(), + )) } } diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 0831f25002a..066d1b9fbe3 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -17,6 +17,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -332,8 +333,8 @@ impl VTable for FSSTVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - canonicalize_fsst(array, ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + canonicalize_fsst(array, ctx).map(ExecutionStep::Done) } fn execute_parent( diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index ba94651b360..eead34ee852 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -19,6 +19,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -262,8 +263,8 @@ impl VTable for PcoVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.decompress()?.into_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.decompress()?.into_array())) } fn reduce_parent( diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 65d2c57cb5d..ed2ba204bf4 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -200,8 +201,8 @@ impl VTable for RunEndVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - run_end_canonicalize(array, ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + run_end_canonicalize(array, ctx).map(ExecutionStep::Done) } } diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 207cff5c5a6..f5681841f95 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -7,6 +7,7 @@ use num_traits::cast::FromPrimitive; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -363,7 +364,7 @@ impl VTable for SequenceVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { let prim = match_each_native_ptype!(array.ptype(), |P| { let base = array.base().cast::

()?; let multiplier = array.multiplier().cast::

()?; @@ -374,7 +375,7 @@ impl VTable for SequenceVTable { PrimitiveArray::new(values, array.dtype.nullability().into()) }); - Ok(prim.into_array()) + Ok(ExecutionStep::Done(prim.into_array())) } fn execute_parent( diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index df41d26a49f..b1cc667a33b 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -11,6 +11,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -229,8 +230,8 @@ impl VTable for SparseVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - execute_sparse(array, ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + execute_sparse(array, ctx).map(ExecutionStep::Done) } } diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 38155892d24..b7229031151 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -9,6 +9,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::buffer::BufferHandle; @@ -148,8 +149,10 @@ impl VTable for ZigZagVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(zigzag_decode(array.encoded().clone().execute(ctx)?).into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + zigzag_decode(array.encoded().clone().execute(ctx)?).into_array(), + )) } fn reduce_parent( diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 5d0cd8f6503..f7263939e4d 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -12,6 +12,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -271,8 +272,11 @@ impl VTable for ZstdVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - array.decompress()?.execute(ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + array + .decompress()? + .execute::(ctx) + .map(ExecutionStep::Done) } fn reduce_parent( diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 30c26bbf30a..ff733475a95 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::buffer::BufferHandle; @@ -466,10 +467,12 @@ impl VTable for ZstdBuffersVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let session = ctx.session(); let inner_array = array.decompress_and_build_inner(session)?; - inner_array.execute::(ctx) + inner_array + .execute::(ctx) + .map(ExecutionStep::Done) } } diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index ae82d592c72..567d017c627 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -12,6 +12,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::DeserializeMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::BoolArray; @@ -183,8 +184,8 @@ impl VTable for BoolVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 073282c769a..e30eaf3cd50 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -15,6 +15,7 @@ use crate::ArrayRef; use crate::Canonical; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::ToCanonical; @@ -239,8 +240,8 @@ impl VTable for ChunkedVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(_canonicalize(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(_canonicalize(array, ctx)?.into_array())) } fn reduce(array: &Self::Array) -> VortexResult> { diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index 3e69fdfd218..9c7a0ea4f6d 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::arrays::ConstantArray; @@ -165,7 +166,9 @@ impl VTable for ConstantVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(constant_canonicalize(array)?.into_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + constant_canonicalize(array)?.into_array(), + )) } } diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 3dd71efc924..f7b8fb8dbdf 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::DeserializeMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::DecimalArray; @@ -205,8 +206,8 @@ impl VTable for DecimalVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index 8c2c9197019..37d88aaf7ee 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -29,6 +29,7 @@ use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; @@ -190,9 +191,9 @@ impl VTable for DictVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { if let Some(canonical) = execute_fast_path(array, ctx)? { - return Ok(canonical); + return Ok(ExecutionStep::Done(canonical)); } // TODO(joe): if the values are constant return a constant @@ -208,7 +209,9 @@ impl VTable for DictVTable { // TODO(ngates): if indices min is quite high, we could slice self and offset the indices // such that canonicalize does less work. - Ok(take_canonical(values, &codes, ctx)?.into_array()) + Ok(ExecutionStep::Done( + take_canonical(values, &codes, ctx)?.into_array(), + )) } fn reduce_parent( diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 3fe7bbb4cd2..c0404859555 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -18,6 +18,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::extension::ExtensionArray; use crate::arrays::extension::compute::rules::PARENT_RULES; @@ -148,8 +149,8 @@ impl VTable for ExtensionVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index 7deae7af8a0..fff8c90b992 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -27,6 +27,7 @@ use crate::arrays::filter::rules::RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::scalar::Scalar; use crate::serde::ArrayChildren; use crate::stats::StatsSetRef; @@ -154,9 +155,9 @@ impl VTable for FilterVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { if let Some(canonical) = execute_filter_fast_paths(array, ctx)? { - return Ok(canonical); + return Ok(ExecutionStep::Done(canonical)); } let Mask::Values(mask_values) = &array.mask else { unreachable!("`execute_filter_fast_paths` handles AllTrue and AllFalse") @@ -164,7 +165,9 @@ impl VTable for FilterVTable { // We rely on the optimization pass that runs prior to this execution for filter pushdown, // so now we can just execute the filter without worrying. - Ok(execute_filter(array.child.clone().execute(ctx)?, mask_values).into_array()) + Ok(ExecutionStep::Done( + execute_filter(array.child.clone().execute(ctx)?, mask_values).into_array(), + )) } fn reduce_parent( diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index fd377e41d7f..324c20db74d 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::compute::rules::PARENT_RULES; @@ -217,7 +218,7 @@ impl VTable for FixedSizeListVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } } diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index d574757b008..2857736af2c 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::Array; use crate::ArrayRef; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::ProstMetadata; @@ -210,8 +211,10 @@ impl VTable for ListVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(list_view_from_list(array.clone(), ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + list_view_from_list(array.clone(), ctx)?.into_array(), + )) } fn execute_parent( diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 5ee338c781f..e7844b183a0 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::DeserializeMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::ProstMetadata; use crate::SerializeMetadata; @@ -237,8 +238,8 @@ impl VTable for ListViewVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index faafac1d95c..c63a0687196 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -25,6 +25,7 @@ use crate::arrays::masked::mask_validity_canonical; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; @@ -160,15 +161,15 @@ impl VTable for MaskedVTable { MaskedArray::try_new(child, validity) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let validity_mask = array.validity_mask()?; // Fast path: all masked means result is all nulls. if validity_mask.all_false() { - return Ok( + return Ok(ExecutionStep::Done( ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len()) .into_array(), - ); + )); } // NB: We intentionally do NOT have a fast path for `validity_mask.all_true()`. @@ -178,7 +179,9 @@ impl VTable for MaskedVTable { // `AllTrue` masks (no data copying), so there's no benefit. let child = array.child().clone().execute::(ctx)?; - Ok(mask_validity_canonical(child, &validity_mask, ctx)?.into_array()) + Ok(ExecutionStep::Done( + mask_validity_canonical(child, &validity_mask, ctx)?.into_array(), + )) } fn reduce_parent( diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 95770dbbac8..dd08cc712be 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -11,6 +11,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::null::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; @@ -130,8 +131,8 @@ impl VTable for NullVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } } diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index 2a6529613aa..6637a51ffb8 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -11,6 +11,7 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::arrays::PrimitiveArray; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -197,8 +198,8 @@ impl VTable for PrimitiveVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 974923a23a1..e255f74ea11 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -30,6 +30,7 @@ use crate::arrays::scalar_fn::rules::RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::expr::Expression; use crate::matcher::Matcher; use crate::scalar_fn; @@ -194,10 +195,10 @@ impl VTable for ScalarFnVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { ctx.log(format_args!("scalar_fn({}): executing", array.scalar_fn)); let args = VecExecutionArgs::new(array.children.clone(), array.len); - array.scalar_fn.execute(&args, ctx) + array.scalar_fn.execute(&args, ctx).map(ExecutionStep::Done) } fn reduce(array: &Self::Array) -> VortexResult> { diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index d79ca3de4d6..9a7ca38b263 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -12,6 +12,7 @@ use crate::ArrayRef; use crate::Canonical; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::shared::SharedArray; use crate::buffer::BufferHandle; @@ -144,8 +145,10 @@ impl VTable for SharedVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - array.get_or_compute(|source| source.clone().execute::(ctx)) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + array + .get_or_compute(|source| source.clone().execute::(ctx)) + .map(ExecutionStep::Done) } } impl OperationsVTable for SharedVTable { diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index daa445e8b89..26cc932a8ea 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -26,6 +26,7 @@ use crate::arrays::slice::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::scalar::Scalar; use crate::serde::ArrayChildren; use crate::stats::StatsSetRef; @@ -154,7 +155,7 @@ impl VTable for SliceVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { // Execute the child to get canonical form, then slice it let Some(canonical) = array.child.as_opt::() else { // If the child is not canonical, recurse. @@ -162,13 +163,15 @@ impl VTable for SliceVTable { .child .clone() .execute::(ctx)? - .slice(array.slice_range().clone()); + .slice(array.slice_range().clone()) + .map(ExecutionStep::Done); }; // TODO(ngates): we should inline canonical slice logic here. Canonical::from(canonical) .as_ref() .slice(array.range.clone()) + .map(ExecutionStep::Done) } fn reduce_parent( diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index c81286ef782..b8aa7d01a36 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -15,6 +15,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::arrays::struct_::StructArray; use crate::arrays::struct_::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; @@ -205,8 +206,8 @@ impl VTable for StructVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index 9b3d3a1ec71..369f72b9c10 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -10,6 +10,7 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::DeserializeMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::ProstMetadata; use crate::SerializeMetadata; @@ -218,8 +219,10 @@ impl VTable for VarBinVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(varbin_to_canonical(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + varbin_to_canonical(array, ctx)?.into_array(), + )) } } diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index bf52a8d6e47..95ae0422f22 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -17,6 +17,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::BinaryView; use crate::arrays::varbinview::VarBinViewArray; @@ -241,7 +242,7 @@ impl VTable for VarBinViewVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } } diff --git a/vortex-array/src/columnar.rs b/vortex-array/src/columnar.rs index 09746a67032..3633028c205 100644 --- a/vortex-array/src/columnar.rs +++ b/vortex-array/src/columnar.rs @@ -4,6 +4,7 @@ use std::env::VarError; use std::sync::LazyLock; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; @@ -15,11 +16,13 @@ use crate::Canonical; use crate::CanonicalView; use crate::Executable; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::arrays::ConstantArray; use crate::arrays::ConstantVTable; use crate::dtype::DType; use crate::matcher::Matcher; +use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; /// Represents a columnnar array of data, either in canonical form or as a constant array. @@ -71,14 +74,20 @@ impl IntoArray for Columnar { } } -/// Executing into a [`Columnar`] is implemented by repeatedly executing the array until we -/// converge on either a constant or canonical. +/// Executing into a [`Columnar`] is implemented using an iterative scheduler with an explicit +/// work stack. +/// +/// The scheduler repeatedly: +/// 1. Checks if the current array is columnar (constant or canonical) — if so, pops the stack. +/// 2. Runs reduce/reduce_parent rules to fixpoint. +/// 3. Tries execute_parent on each child. +/// 4. Calls `execute` which returns an [`ExecutionStep`]. /// /// For safety, we will error when the number of execution iterations reaches 128. We may want this /// to be configurable in the future in case of highly complex array trees, but in practice we /// don't expect to ever reach this limit. impl Executable for Columnar { - fn execute(mut array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(root: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { static MAX_ITERATIONS: LazyLock = LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { Ok(val) => val.parse::().unwrap_or_else(|e| { @@ -90,26 +99,101 @@ impl Executable for Columnar { } }); + let mut current = root.optimize()?; + let mut stack: Vec<(ArrayRef, usize)> = Vec::new(); + for _ in 0..*MAX_ITERATIONS { - // Check for termination conditions - if let Some(constant) = array.as_opt::() { - ctx.log(format_args!("-> constant({})", constant.scalar())); - return Ok(Columnar::Constant(constant.clone())); + // Check for columnar termination (constant or canonical) + if let Some(columnar) = try_as_columnar(¤t) { + match stack.pop() { + None => { + // Stack empty — we're done + ctx.log(format_args!("-> columnar {}", current)); + return Ok(columnar); + } + Some((parent, child_idx)) => { + // Replace the child in the parent and continue + current = parent.with_child(child_idx, current)?; + current = current.optimize()?; + continue; + } + } } - if let Some(canonical) = array.as_opt::() { - ctx.log(format_args!("-> canonical {}", array)); - return Ok(Columnar::Canonical(canonical.into())); + + // Try execute_parent (child-driven optimized execution) + if let Some(rewritten) = try_execute_parent(¤t, ctx)? { + ctx.log(format_args!( + "execute_parent rewrote {} -> {}", + current, rewritten + )); + current = rewritten.optimize()?; + continue; } - // Otherwise execute the array one step - array = array.execute(ctx)?; + // Execute the array itself + match current.vtable().execute(¤t, ctx)? { + ExecutionStep::ExecuteChild(i) => { + let child = current + .nth_child(i) + .vortex_expect("ExecuteChild index in bounds"); + ctx.log(format_args!( + "ExecuteChild({i}): pushing {}, focusing on {}", + current, child + )); + stack.push((current, i)); + current = child.optimize()?; + } + ExecutionStep::ColumnarizeChild(i) => { + let child = current + .nth_child(i) + .vortex_expect("ColumnarizeChild index in bounds"); + ctx.log(format_args!( + "ColumnarizeChild({i}): pushing {}, focusing on {}", + current, child + )); + stack.push((current, i)); + // No cross-step optimization for ColumnarizeChild + current = child; + } + ExecutionStep::Done(result) => { + ctx.log(format_args!("Done: {} -> {}", current, result)); + current = result; + } + } } - // If we reach here, we exceeded the maximum number of iterations, so error. vortex_bail!("Exceeded maximum execution iterations while executing to Columnar") } } +/// Try to interpret an array as columnar (constant or canonical). +fn try_as_columnar(array: &ArrayRef) -> Option { + if let Some(constant) = array.as_opt::() { + Some(Columnar::Constant(constant.clone())) + } else { + array + .as_opt::() + .map(|c| Columnar::Canonical(c.into())) + } +} + +/// Try execute_parent on each child of the array. +fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult> { + for child_idx in 0..array.nchildren() { + let child = array + .nth_child(child_idx) + .vortex_expect("checked nchildren"); + if let Some(result) = child + .vtable() + .execute_parent(&child, array, child_idx, ctx)? + { + result.statistics().inherit_from(array.statistics()); + return Ok(Some(result)); + } + } + Ok(None) +} + pub enum ColumnarView<'a> { Canonical(CanonicalView<'a>), Constant(&'a ConstantArray), diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 8dfe549dd1a..122d7e4b065 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -122,10 +122,10 @@ impl Drop for ExecutionCtx { /// /// The execution steps are as follows: /// 0. Check for canonical. -/// 1. Attempt to call `reduce_parent` on each child. -/// 2. Attempt to `reduce` the array with metadata-only optimizations. +/// 1. Attempt to `reduce` the array with metadata-only optimizations. +/// 2. Attempt to call `reduce_parent` on each child. /// 3. Attempt to call `execute_parent` on each child. -/// 4. Call `execute` on the array itself. +/// 4. Call `execute` on the array itself (which returns an [`ExecutionStep`]). /// /// Most users will not call this method directly, instead preferring to specify an executable /// target such as [`crate::Columnar`], [`Canonical`], or any of the canonical array types (such as @@ -182,16 +182,21 @@ impl Executable for ArrayRef { } } - // 4. execute (optimized execution) + // 4. execute (returns an ExecutionStep) ctx.log(format_args!("executing {}", array)); - let array = array - .vtable() - .execute(&array, ctx) - .map(|c| c.into_array())?; - array.statistics().inherit_from(array.statistics()); - ctx.log(format_args!("-> {}", array.as_ref())); - - Ok(array) + match array.vtable().execute(&array, ctx)? { + ExecutionStep::Done(result) => { + ctx.log(format_args!("-> {}", result.as_ref())); + Ok(result) + } + ExecutionStep::ExecuteChild(i) | ExecutionStep::ColumnarizeChild(i) => { + // For single-step execution, handle ExecuteChild by executing the child, + // replacing it, and returning the updated array. + let child = array.nth_child(i).vortex_expect("valid child index"); + let executed_child = child.execute::(ctx)?; + array.with_child(i, executed_child) + } + } } } diff --git a/vortex-array/src/vtable/dyn_.rs b/vortex-array/src/vtable/dyn_.rs index 242bce3d532..37be712f07b 100644 --- a/vortex-array/src/vtable/dyn_.rs +++ b/vortex-array/src/vtable/dyn_.rs @@ -16,6 +16,7 @@ use vortex_session::VortexSession; use crate::Array; use crate::ArrayAdapter; use crate::ArrayRef; +use crate::ExecutionStep; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; @@ -59,7 +60,7 @@ pub trait DynVTable: 'static + private::Sealed + Send + Sync + Debug { ) -> VortexResult>; /// See [`VTable::execute`] - fn execute(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult; + fn execute(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult; /// See [`VTable::execute_parent`] fn execute_parent( @@ -144,29 +145,31 @@ impl DynVTable for ArrayVTableAdapter { Ok(Some(reduced)) } - fn execute(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - let result = V::execute(downcast::(array), ctx)?; - - if cfg!(debug_assertions) { - vortex_ensure!( - result.as_ref().len() == array.len(), - "Result length mismatch for {:?}", - self - ); - vortex_ensure!( - result.as_ref().dtype() == array.dtype(), - "Executed canonical dtype mismatch for {:?}", - self - ); + fn execute(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + let step = V::execute(downcast::(array), ctx)?; + + if let ExecutionStep::Done(ref result) = step { + if cfg!(debug_assertions) { + vortex_ensure!( + result.as_ref().len() == array.len(), + "Result length mismatch for {:?}", + self + ); + vortex_ensure!( + result.as_ref().dtype() == array.dtype(), + "Executed canonical dtype mismatch for {:?}", + self + ); + } + + // TODO(ngates): do we want to do this on every execution? We used to in to_canonical. + result + .as_ref() + .statistics() + .inherit_from(array.statistics()); } - // TODO(ngates): do we want to do this on every execution? We used to in to_canonical. - result - .as_ref() - .statistics() - .inherit_from(array.statistics()); - - Ok(result) + Ok(step) } fn execute_parent( diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index 08d6868bb4e..2a600f07015 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -22,6 +22,7 @@ use vortex_session::VortexSession; use crate::Array; use crate::ArrayRef; use crate::Canonical; +use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::arrays::ConstantArray; @@ -180,30 +181,24 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { /// of children must be expected. fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()>; - /// Execute this array to produce an [`ArrayRef`]. + /// Execute this array by returning an [`ExecutionStep`] that tells the scheduler what to + /// do next. + /// + /// Instead of recursively executing children, implementations should return + /// [`ExecutionStep::ExecuteChild(i)`] or [`ExecutionStep::ColumnarizeChild(i)`] to request + /// that the scheduler execute a child first, or [`ExecutionStep::Done(result)`] when the + /// encoding can produce a result directly. /// /// Array execution is designed such that repeated execution of an array will eventually /// converge to a canonical representation. Implementations of this function should therefore /// ensure they make progress towards that goal. /// - /// This includes fully evaluating the array, such us decoding run-end encoding, or executing - /// one of the array's children and re-building the array with the executed child. - /// - /// It is recommended to only perform a single step of execution per call to this function, - /// such that surrounding arrays have an opportunity to perform their own parent reduction - /// or execution logic. - /// - /// The returned array must be logically equivalent to the input array. In other words, the - /// recursively canonicalized forms of both arrays must be equal. + /// The returned array (in `Done`) must be logically equivalent to the input array. In other + /// words, the recursively canonicalized forms of both arrays must be equal. /// /// Debug builds will panic if the returned array is of the wrong type, wrong length, or /// incorrectly contains null values. - /// - // TODO(ngates): in the future, we may pass a "target encoding hint" such that this array - // can produce a more optimal representation for the parent. This could be used to preserve - // varbin vs varbinview or list vs listview encodings when the parent knows it prefers - // one representation over another, such as when exporting to a specific Arrow array. - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult; + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult; /// Attempt to execute the parent of this array. /// diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index 83ebfa226cf..be3b2ad26a3 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -10,6 +10,7 @@ use pyo3::prelude::*; use pyo3::types::PyBytes; use vortex::array::ArrayRef; use vortex::array::ExecutionCtx; +use vortex::array::ExecutionStep; use vortex::array::Precision; use vortex::array::RawMetadata; use vortex::array::SerializeMetadata; @@ -155,7 +156,7 @@ impl VTable for PythonVTable { Ok(()) } - fn execute(_array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(_array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { todo!() } } From 2b9697c6d665fcbef5392dfe4616116f8a99c1a3 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 13:59:21 -0500 Subject: [PATCH 04/12] chore: regenerate public-api.lock files Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/alp/public-api.lock | 4 +- encodings/bytebool/public-api.lock | 2 +- encodings/datetime-parts/public-api.lock | 2 +- encodings/decimal-byte-parts/public-api.lock | 2 +- encodings/fastlanes/public-api.lock | 8 +- encodings/fsst/public-api.lock | 2 +- encodings/pco/public-api.lock | 2 +- encodings/runend/public-api.lock | 2 +- encodings/sequence/public-api.lock | 2 +- encodings/sparse/public-api.lock | 2 +- encodings/zigzag/public-api.lock | 2 +- encodings/zstd/public-api.lock | 2 +- vortex-array/public-api.lock | 96 +++++++++++--------- vortex-array/src/columnar.rs | 12 --- vortex-array/src/executor.rs | 10 +- vortex-array/src/vtable/mod.rs | 4 +- 16 files changed, 72 insertions(+), 82 deletions(-) diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index f2a1ae9089d..fe227544ded 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -196,7 +196,7 @@ pub fn vortex_alp::ALPRDVTable::deserialize(bytes: &[u8], _dtype: &vortex_array: pub fn vortex_alp::ALPRDVTable::dtype(array: &vortex_alp::ALPRDArray) -> &vortex_array::dtype::DType -pub fn vortex_alp::ALPRDVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_alp::ALPRDVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_alp::ALPRDVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> @@ -300,7 +300,7 @@ pub fn vortex_alp::ALPVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::d pub fn vortex_alp::ALPVTable::dtype(array: &vortex_alp::ALPArray) -> &vortex_array::dtype::DType -pub fn vortex_alp::ALPVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_alp::ALPVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_alp::ALPVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 0a1a01e7ae2..47108e9378a 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -104,7 +104,7 @@ pub fn vortex_bytebool::ByteBoolVTable::deserialize(_bytes: &[u8], _dtype: &vort pub fn vortex_bytebool::ByteBoolVTable::dtype(array: &vortex_bytebool::ByteBoolArray) -> &vortex_array::dtype::DType -pub fn vortex_bytebool::ByteBoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_bytebool::ByteBoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_bytebool::ByteBoolVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index 04626e436e8..14d850ade0a 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -178,7 +178,7 @@ pub fn vortex_datetime_parts::DateTimePartsVTable::deserialize(bytes: &[u8], _dt pub fn vortex_datetime_parts::DateTimePartsVTable::dtype(array: &vortex_datetime_parts::DateTimePartsArray) -> &vortex_array::dtype::DType -pub fn vortex_datetime_parts::DateTimePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::DateTimePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_datetime_parts::DateTimePartsVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 310a47c8c9a..41a2a23a086 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -108,7 +108,7 @@ pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::deserialize(bytes: &[u pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::dtype(array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> &vortex_array::dtype::DType -pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index a6797c89530..80b77f6bdaa 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -238,7 +238,7 @@ pub fn vortex_fastlanes::BitPackedVTable::deserialize(bytes: &[u8], _dtype: &vor pub fn vortex_fastlanes::BitPackedVTable::dtype(array: &vortex_fastlanes::BitPackedArray) -> &vortex_array::dtype::DType -pub fn vortex_fastlanes::BitPackedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPackedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPackedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> @@ -364,7 +364,7 @@ pub fn vortex_fastlanes::DeltaVTable::deserialize(bytes: &[u8], _dtype: &vortex_ pub fn vortex_fastlanes::DeltaVTable::dtype(array: &vortex_fastlanes::DeltaArray) -> &vortex_array::dtype::DType -pub fn vortex_fastlanes::DeltaVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::DeltaVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::DeltaVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId @@ -498,7 +498,7 @@ pub fn vortex_fastlanes::FoRVTable::deserialize(bytes: &[u8], dtype: &vortex_arr pub fn vortex_fastlanes::FoRVTable::dtype(array: &vortex_fastlanes::FoRArray) -> &vortex_array::dtype::DType -pub fn vortex_fastlanes::FoRVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::FoRVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::FoRVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> @@ -630,7 +630,7 @@ pub fn vortex_fastlanes::RLEVTable::deserialize(bytes: &[u8], _dtype: &vortex_ar pub fn vortex_fastlanes::RLEVTable::dtype(array: &vortex_fastlanes::RLEArray) -> &vortex_array::dtype::DType -pub fn vortex_fastlanes::RLEVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLEVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::RLEVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 103abc8a38c..57f1098d357 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -142,7 +142,7 @@ pub fn vortex_fsst::FSSTVTable::deserialize(bytes: &[u8], _dtype: &vortex_array: pub fn vortex_fsst::FSSTVTable::dtype(array: &vortex_fsst::FSSTArray) -> &vortex_array::dtype::DType -pub fn vortex_fsst::FSSTVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fsst::FSSTVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fsst::FSSTVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 2cbb6087556..f97ab79123b 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -156,7 +156,7 @@ pub fn vortex_pco::PcoVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::d pub fn vortex_pco::PcoVTable::dtype(array: &vortex_pco::PcoArray) -> &vortex_array::dtype::DType -pub fn vortex_pco::PcoVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_pco::PcoVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_pco::PcoVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index b6dce876336..4318cc7255e 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -176,7 +176,7 @@ pub fn vortex_runend::RunEndVTable::deserialize(bytes: &[u8], _dtype: &vortex_ar pub fn vortex_runend::RunEndVTable::dtype(array: &vortex_runend::RunEndArray) -> &vortex_array::dtype::DType -pub fn vortex_runend::RunEndVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_runend::RunEndVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_runend::RunEndVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index 6d4095649d0..520418d2a3a 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -128,7 +128,7 @@ pub fn vortex_sequence::SequenceVTable::deserialize(bytes: &[u8], _dtype: &vorte pub fn vortex_sequence::SequenceVTable::dtype(array: &vortex_sequence::SequenceArray) -> &vortex_array::dtype::DType -pub fn vortex_sequence::SequenceVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_sequence::SequenceVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_sequence::SequenceVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index f4456984bb1..24a0fb1e6c1 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -120,7 +120,7 @@ pub fn vortex_sparse::SparseVTable::deserialize(bytes: &[u8], _dtype: &vortex_ar pub fn vortex_sparse::SparseVTable::dtype(array: &vortex_sparse::SparseArray) -> &vortex_array::dtype::DType -pub fn vortex_sparse::SparseVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_sparse::SparseVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_sparse::SparseVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index 60dc4b073f9..1f9fca6c735 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -96,7 +96,7 @@ pub fn vortex_zigzag::ZigZagVTable::deserialize(_bytes: &[u8], _dtype: &vortex_a pub fn vortex_zigzag::ZigZagVTable::dtype(array: &vortex_zigzag::ZigZagArray) -> &vortex_array::dtype::DType -pub fn vortex_zigzag::ZigZagVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zigzag::ZigZagVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zigzag::ZigZagVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 04d8faf7109..9f351777a4c 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -196,7 +196,7 @@ pub fn vortex_zstd::ZstdVTable::deserialize(bytes: &[u8], _dtype: &vortex_array: pub fn vortex_zstd::ZstdVTable::dtype(array: &vortex_zstd::ZstdArray) -> &vortex_array::dtype::DType -pub fn vortex_zstd::ZstdVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::ZstdVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 5cbd6cfbfed..0942e30fa63 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -212,7 +212,7 @@ pub fn vortex_array::arrays::DictVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::DictVTable::dtype(array: &vortex_array::arrays::DictArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DictVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -520,7 +520,7 @@ pub fn vortex_array::arrays::BoolVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::BoolVTable::dtype(array: &vortex_array::arrays::BoolArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::BoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::BoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::BoolVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -690,7 +690,7 @@ pub fn vortex_array::arrays::ChunkedVTable::deserialize(_bytes: &[u8], _dtype: & pub fn vortex_array::arrays::ChunkedVTable::dtype(array: &vortex_array::arrays::ChunkedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ChunkedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ChunkedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ChunkedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -838,7 +838,7 @@ pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ConstantVTable::dtype(array: &vortex_array::arrays::ConstantArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ConstantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ConstantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ConstantVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1056,7 +1056,7 @@ pub fn vortex_array::arrays::DecimalVTable::deserialize(bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::DecimalVTable::dtype(array: &vortex_array::arrays::DecimalArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DecimalVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DecimalVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DecimalVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1262,7 +1262,7 @@ pub fn vortex_array::arrays::DictVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::DictVTable::dtype(array: &vortex_array::arrays::DictArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DictVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1450,7 +1450,7 @@ pub fn vortex_array::arrays::ExtensionVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ExtensionVTable::dtype(array: &vortex_array::arrays::ExtensionArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ExtensionVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ExtensionVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ExtensionVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1600,7 +1600,7 @@ pub fn vortex_array::arrays::FilterVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::FilterVTable::dtype(array: &vortex_array::arrays::FilterArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::FilterVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FilterVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FilterVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1756,7 +1756,7 @@ pub fn vortex_array::arrays::FixedSizeListVTable::deserialize(_bytes: &[u8], _dt pub fn vortex_array::arrays::FixedSizeListVTable::dtype(array: &vortex_array::arrays::FixedSizeListArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::FixedSizeListVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeListVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeListVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1954,7 +1954,7 @@ pub fn vortex_array::arrays::ListVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::ListVTable::dtype(array: &vortex_array::arrays::ListArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ListVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2134,7 +2134,7 @@ pub fn vortex_array::arrays::ListViewVTable::deserialize(bytes: &[u8], _dtype: & pub fn vortex_array::arrays::ListViewVTable::dtype(array: &vortex_array::arrays::ListViewArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ListViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListViewVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2256,7 +2256,7 @@ pub fn vortex_array::arrays::MaskedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::MaskedVTable::dtype(array: &vortex_array::arrays::MaskedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::MaskedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::MaskedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::MaskedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2454,7 +2454,7 @@ pub fn vortex_array::arrays::NullVTable::deserialize(_bytes: &[u8], _dtype: &vor pub fn vortex_array::arrays::NullVTable::dtype(_array: &vortex_array::arrays::NullArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::NullVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::NullVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::NullVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2708,7 +2708,7 @@ pub fn vortex_array::arrays::PrimitiveVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::PrimitiveVTable::dtype(array: &vortex_array::arrays::PrimitiveArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::PrimitiveVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::PrimitiveVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::PrimitiveVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2870,7 +2870,7 @@ pub fn vortex_array::arrays::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ScalarFnVTable::dtype(array: &vortex_array::arrays::ScalarFnArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ScalarFnVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ScalarFnVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ScalarFnVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2978,7 +2978,7 @@ pub fn vortex_array::arrays::SharedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::SharedVTable::dtype(array: &vortex_array::arrays::SharedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SharedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3138,7 +3138,7 @@ pub fn vortex_array::arrays::SliceVTable::deserialize(_bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::SliceVTable::dtype(array: &vortex_array::arrays::SliceArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SliceVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3326,7 +3326,7 @@ pub fn vortex_array::arrays::StructVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::StructVTable::dtype(array: &vortex_array::arrays::StructArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::StructVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::StructVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::StructVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3652,7 +3652,7 @@ pub fn vortex_array::arrays::VarBinVTable::deserialize(bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::VarBinVTable::dtype(array: &vortex_array::arrays::VarBinArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::VarBinVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3870,7 +3870,7 @@ pub fn vortex_array::arrays::VarBinViewVTable::deserialize(_bytes: &[u8], _dtype pub fn vortex_array::arrays::VarBinViewVTable::dtype(array: &vortex_array::arrays::VarBinViewArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::VarBinViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinViewVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16414,7 +16414,7 @@ pub trait vortex_array::vtable::DynVTable: 'static + vortex_array::vtable::dyn_: pub fn vortex_array::vtable::DynVTable::build(&self, id: vortex_array::vtable::ArrayId, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::vtable::DynVTable::execute(&self, array: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::vtable::DynVTable::execute(&self, array: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::vtable::DynVTable::execute_parent(&self, array: &vortex_array::ArrayRef, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16538,7 +16538,7 @@ pub fn vortex_array::vtable::VTable::deserialize(bytes: &[u8], _dtype: &vortex_a pub fn vortex_array::vtable::VTable::dtype(array: &Self::Array) -> &vortex_array::dtype::DType -pub fn vortex_array::vtable::VTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::vtable::VTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::vtable::VTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16592,7 +16592,7 @@ pub fn vortex_array::arrays::BoolVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::BoolVTable::dtype(array: &vortex_array::arrays::BoolArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::BoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::BoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::BoolVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16646,7 +16646,7 @@ pub fn vortex_array::arrays::ChunkedVTable::deserialize(_bytes: &[u8], _dtype: & pub fn vortex_array::arrays::ChunkedVTable::dtype(array: &vortex_array::arrays::ChunkedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ChunkedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ChunkedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ChunkedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16700,7 +16700,7 @@ pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ConstantVTable::dtype(array: &vortex_array::arrays::ConstantArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ConstantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ConstantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ConstantVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16754,7 +16754,7 @@ pub fn vortex_array::arrays::DecimalVTable::deserialize(bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::DecimalVTable::dtype(array: &vortex_array::arrays::DecimalArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DecimalVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DecimalVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DecimalVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16808,7 +16808,7 @@ pub fn vortex_array::arrays::DictVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::DictVTable::dtype(array: &vortex_array::arrays::DictArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DictVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16862,7 +16862,7 @@ pub fn vortex_array::arrays::ExtensionVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ExtensionVTable::dtype(array: &vortex_array::arrays::ExtensionArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ExtensionVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ExtensionVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ExtensionVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16916,7 +16916,7 @@ pub fn vortex_array::arrays::FilterVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::FilterVTable::dtype(array: &vortex_array::arrays::FilterArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::FilterVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FilterVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FilterVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16970,7 +16970,7 @@ pub fn vortex_array::arrays::FixedSizeListVTable::deserialize(_bytes: &[u8], _dt pub fn vortex_array::arrays::FixedSizeListVTable::dtype(array: &vortex_array::arrays::FixedSizeListArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::FixedSizeListVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeListVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeListVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17024,7 +17024,7 @@ pub fn vortex_array::arrays::ListVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::ListVTable::dtype(array: &vortex_array::arrays::ListArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ListVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17078,7 +17078,7 @@ pub fn vortex_array::arrays::ListViewVTable::deserialize(bytes: &[u8], _dtype: & pub fn vortex_array::arrays::ListViewVTable::dtype(array: &vortex_array::arrays::ListViewArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ListViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListViewVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17132,7 +17132,7 @@ pub fn vortex_array::arrays::MaskedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::MaskedVTable::dtype(array: &vortex_array::arrays::MaskedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::MaskedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::MaskedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::MaskedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17186,7 +17186,7 @@ pub fn vortex_array::arrays::NullVTable::deserialize(_bytes: &[u8], _dtype: &vor pub fn vortex_array::arrays::NullVTable::dtype(_array: &vortex_array::arrays::NullArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::NullVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::NullVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::NullVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17240,7 +17240,7 @@ pub fn vortex_array::arrays::PrimitiveVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::PrimitiveVTable::dtype(array: &vortex_array::arrays::PrimitiveArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::PrimitiveVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::PrimitiveVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::PrimitiveVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17294,7 +17294,7 @@ pub fn vortex_array::arrays::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ScalarFnVTable::dtype(array: &vortex_array::arrays::ScalarFnArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ScalarFnVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ScalarFnVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ScalarFnVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17348,7 +17348,7 @@ pub fn vortex_array::arrays::SharedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::SharedVTable::dtype(array: &vortex_array::arrays::SharedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SharedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17402,7 +17402,7 @@ pub fn vortex_array::arrays::SliceVTable::deserialize(_bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::SliceVTable::dtype(array: &vortex_array::arrays::SliceArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SliceVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17456,7 +17456,7 @@ pub fn vortex_array::arrays::StructVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::StructVTable::dtype(array: &vortex_array::arrays::StructArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::StructVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::StructVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::StructVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17510,7 +17510,7 @@ pub fn vortex_array::arrays::VarBinVTable::deserialize(bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::VarBinVTable::dtype(array: &vortex_array::arrays::VarBinArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::VarBinVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17564,7 +17564,7 @@ pub fn vortex_array::arrays::VarBinViewVTable::deserialize(_bytes: &[u8], _dtype pub fn vortex_array::arrays::VarBinViewVTable::dtype(array: &vortex_array::arrays::VarBinViewArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::VarBinViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinViewVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17896,7 +17896,7 @@ pub fn vortex_array::Columnar::len(&self) -> usize impl vortex_array::Executable for vortex_array::Columnar -pub fn vortex_array::Columnar::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::Columnar::execute(root: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::IntoArray for vortex_array::Columnar @@ -17912,6 +17912,16 @@ impl<'a> core::convert::AsRef for vortex_array::Columna pub fn vortex_array::ColumnarView<'a>::as_ref(&self) -> &dyn vortex_array::Array +pub enum vortex_array::ExecutionStep + +pub vortex_array::ExecutionStep::Done(vortex_array::ArrayRef) + +pub vortex_array::ExecutionStep::ExecuteChild(usize) + +impl core::fmt::Debug for vortex_array::ExecutionStep + +pub fn vortex_array::ExecutionStep::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub enum vortex_array::Precision pub vortex_array::Precision::Ptr @@ -18562,7 +18572,7 @@ pub fn vortex_array::CanonicalValidity::execute(array: vortex_array::ArrayRef, c impl vortex_array::Executable for vortex_array::Columnar -pub fn vortex_array::Columnar::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::Columnar::execute(root: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::Executable for vortex_array::RecursiveCanonical diff --git a/vortex-array/src/columnar.rs b/vortex-array/src/columnar.rs index 3633028c205..7cb60b5f5f5 100644 --- a/vortex-array/src/columnar.rs +++ b/vortex-array/src/columnar.rs @@ -143,18 +143,6 @@ impl Executable for Columnar { stack.push((current, i)); current = child.optimize()?; } - ExecutionStep::ColumnarizeChild(i) => { - let child = current - .nth_child(i) - .vortex_expect("ColumnarizeChild index in bounds"); - ctx.log(format_args!( - "ColumnarizeChild({i}): pushing {}, focusing on {}", - current, child - )); - stack.push((current, i)); - // No cross-step optimization for ColumnarizeChild - current = child; - } ExecutionStep::Done(result) => { ctx.log(format_args!("Done: {} -> {}", current, result)); current = result; diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 122d7e4b065..f17780b31e2 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -189,7 +189,7 @@ impl Executable for ArrayRef { ctx.log(format_args!("-> {}", result.as_ref())); Ok(result) } - ExecutionStep::ExecuteChild(i) | ExecutionStep::ColumnarizeChild(i) => { + ExecutionStep::ExecuteChild(i) => { // For single-step execution, handle ExecuteChild by executing the child, // replacing it, and returning the updated array. let child = array.nth_child(i).vortex_expect("valid child index"); @@ -214,14 +214,6 @@ pub enum ExecutionStep { /// cross-step optimization (e.g., pushing scalar functions through newly-decoded children). ExecuteChild(usize), - /// Request that the scheduler execute child at index `i` to columnar form, replace it in - /// this array, then re-enter execution on the updated array. - /// - /// Unlike [`ExecuteChild`](Self::ExecuteChild), the scheduler does **not** run cross-step - /// optimizations when popping back up. Use this when the parent knows it will consume the - /// child directly (e.g., Dict taking from its values). - ColumnarizeChild(usize), - /// Execution is complete. The result may be in any encoding — not necessarily canonical. /// The scheduler will continue executing the result if it is not yet columnar. Done(ArrayRef), diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index 2a600f07015..35184f39b25 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -185,8 +185,8 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { /// do next. /// /// Instead of recursively executing children, implementations should return - /// [`ExecutionStep::ExecuteChild(i)`] or [`ExecutionStep::ColumnarizeChild(i)`] to request - /// that the scheduler execute a child first, or [`ExecutionStep::Done(result)`] when the + /// [`ExecutionStep::ExecuteChild(i)`] to request that the scheduler execute a child first, + /// or [`ExecutionStep::Done(result)`] when the /// encoding can produce a result directly. /// /// Array execution is designed such that repeated execution of an array will eventually From b5b165268c651e9113abd5b0256a0a828b66cc46 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 15:06:11 -0500 Subject: [PATCH 05/12] feat: extract execute_until scheduler with per-child done predicates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the iterative execution scheduler into a general-purpose `execute_until` method on `dyn Array`. The scheduler terminates when the root array matches `M`, while each child can specify its own termination condition via a `DonePredicate` carried in `ExecutionStep::ExecuteChild`. `ExecutionStep` now provides constructor methods: - `execute_child::(idx)` — request child execution until M matches - `done(result)` — signal completion Both `Executable for Columnar` and `Executable for Canonical` are simplified to thin wrappers over `execute_until` with `AnyColumnar` and `AnyCanonical` matchers respectively. Signed-off-by: Nick Gates Co-Authored-By: Claude Opus 4.6 --- vortex-array/public-api.lock | 14 +- vortex-array/src/arrays/constant/mod.rs | 1 - vortex-array/src/canonical.rs | 30 ++--- vortex-array/src/columnar.rs | 122 ++--------------- vortex-array/src/executor.rs | 167 +++++++++++++++++++++++- 5 files changed, 193 insertions(+), 141 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 0942e30fa63..b885c08284c 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -17896,7 +17896,7 @@ pub fn vortex_array::Columnar::len(&self) -> usize impl vortex_array::Executable for vortex_array::Columnar -pub fn vortex_array::Columnar::execute(root: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::Columnar::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::IntoArray for vortex_array::Columnar @@ -17916,7 +17916,13 @@ pub enum vortex_array::ExecutionStep pub vortex_array::ExecutionStep::Done(vortex_array::ArrayRef) -pub vortex_array::ExecutionStep::ExecuteChild(usize) +pub vortex_array::ExecutionStep::ExecuteChild(usize, vortex_array::DonePredicate) + +impl vortex_array::ExecutionStep + +pub fn vortex_array::ExecutionStep::done(result: vortex_array::ArrayRef) -> Self + +pub fn vortex_array::ExecutionStep::execute_child(child_idx: usize) -> Self impl core::fmt::Debug for vortex_array::ExecutionStep @@ -18572,7 +18578,7 @@ pub fn vortex_array::CanonicalValidity::execute(array: vortex_array::ArrayRef, c impl vortex_array::Executable for vortex_array::Columnar -pub fn vortex_array::Columnar::execute(root: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::Columnar::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::Executable for vortex_array::RecursiveCanonical @@ -18829,3 +18835,5 @@ pub fn vortex_session::VortexSession::create_execution_ctx(&self) -> vortex_arra pub type vortex_array::ArrayContext = vortex_session::registry::Context<&'static dyn vortex_array::vtable::DynVTable> pub type vortex_array::ArrayRef = alloc::sync::Arc + +pub type vortex_array::DonePredicate = fn(&dyn vortex_array::Array) -> bool diff --git a/vortex-array/src/arrays/constant/mod.rs b/vortex-array/src/arrays/constant/mod.rs index bb5bb519401..6aa94ab8565 100644 --- a/vortex-array/src/arrays/constant/mod.rs +++ b/vortex-array/src/arrays/constant/mod.rs @@ -8,7 +8,6 @@ pub use arbitrary::ArbitraryConstantArray; mod array; pub use array::ConstantArray; -pub(crate) use vtable::canonical::constant_canonicalize; pub(crate) mod compute; diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index bd90965cf43..21f83511f10 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -14,7 +14,6 @@ use vortex_error::vortex_panic; use crate::Array; use crate::ArrayRef; -use crate::Columnar; use crate::Executable; use crate::ExecutionCtx; use crate::IntoArray; @@ -43,7 +42,6 @@ use crate::arrays::StructVTable; use crate::arrays::VarBinViewArray; use crate::arrays::VarBinViewArrayParts; use crate::arrays::VarBinViewVTable; -use crate::arrays::constant_canonicalize; use crate::builders::builder_with_capacity; use crate::dtype::DType; use crate::dtype::NativePType; @@ -439,28 +437,18 @@ impl From for ArrayRef { } } -/// Recursively execute the array until it reaches canonical form. +/// Execute into [`Canonical`] by running `execute_until` with the [`AnyCanonical`] matcher. /// -/// Callers should prefer to execute into `Columnar` if they are able to optimize their use for -/// constant arrays. +/// Unlike executing into [`crate::Columnar`], this will fully expand constant arrays into their +/// canonical form. Callers should prefer to execute into `Columnar` if they are able to optimize +/// their use for constant arrays. impl Executable for Canonical { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - if let Some(canonical) = array.as_opt::() { - return Ok(canonical.into()); - } - - // Invoke execute directly to avoid logging the call in the execution context. - Ok(match Columnar::execute(array.clone(), ctx)? { - Columnar::Canonical(c) => c, - Columnar::Constant(s) => { - let canonical = constant_canonicalize(&s)?; - canonical - .as_ref() - .statistics() - .inherit_from(array.statistics()); - canonical - } - }) + let result = array.execute_until::(ctx)?; + Ok(result + .as_opt::() + .map(Canonical::from) + .vortex_expect("execute_until:: must return a canonical array")) } } diff --git a/vortex-array/src/columnar.rs b/vortex-array/src/columnar.rs index 7cb60b5f5f5..8d272f8a033 100644 --- a/vortex-array/src/columnar.rs +++ b/vortex-array/src/columnar.rs @@ -1,13 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::env::VarError; -use std::sync::LazyLock; - use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_panic; use crate::AnyCanonical; use crate::Array; @@ -16,13 +11,11 @@ use crate::Canonical; use crate::CanonicalView; use crate::Executable; use crate::ExecutionCtx; -use crate::ExecutionStep; use crate::IntoArray; use crate::arrays::ConstantArray; use crate::arrays::ConstantVTable; use crate::dtype::DType; use crate::matcher::Matcher; -use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; /// Represents a columnnar array of data, either in canonical form or as a constant array. @@ -74,112 +67,21 @@ impl IntoArray for Columnar { } } -/// Executing into a [`Columnar`] is implemented using an iterative scheduler with an explicit -/// work stack. -/// -/// The scheduler repeatedly: -/// 1. Checks if the current array is columnar (constant or canonical) — if so, pops the stack. -/// 2. Runs reduce/reduce_parent rules to fixpoint. -/// 3. Tries execute_parent on each child. -/// 4. Calls `execute` which returns an [`ExecutionStep`]. -/// -/// For safety, we will error when the number of execution iterations reaches 128. We may want this -/// to be configurable in the future in case of highly complex array trees, but in practice we -/// don't expect to ever reach this limit. +/// Execute into [`Columnar`] by running `execute_until` with the [`AnyColumnar`] matcher. impl Executable for Columnar { - fn execute(root: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - static MAX_ITERATIONS: LazyLock = - LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { - Ok(val) => val.parse::().unwrap_or_else(|e| { - vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid usize: {e}") - }), - Err(VarError::NotPresent) => 128, - Err(VarError::NotUnicode(_)) => { - vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid unicode string") - } - }); - - let mut current = root.optimize()?; - let mut stack: Vec<(ArrayRef, usize)> = Vec::new(); - - for _ in 0..*MAX_ITERATIONS { - // Check for columnar termination (constant or canonical) - if let Some(columnar) = try_as_columnar(¤t) { - match stack.pop() { - None => { - // Stack empty — we're done - ctx.log(format_args!("-> columnar {}", current)); - return Ok(columnar); - } - Some((parent, child_idx)) => { - // Replace the child in the parent and continue - current = parent.with_child(child_idx, current)?; - current = current.optimize()?; - continue; - } - } - } - - // Try execute_parent (child-driven optimized execution) - if let Some(rewritten) = try_execute_parent(¤t, ctx)? { - ctx.log(format_args!( - "execute_parent rewrote {} -> {}", - current, rewritten - )); - current = rewritten.optimize()?; - continue; - } - - // Execute the array itself - match current.vtable().execute(¤t, ctx)? { - ExecutionStep::ExecuteChild(i) => { - let child = current - .nth_child(i) - .vortex_expect("ExecuteChild index in bounds"); - ctx.log(format_args!( - "ExecuteChild({i}): pushing {}, focusing on {}", - current, child - )); - stack.push((current, i)); - current = child.optimize()?; - } - ExecutionStep::Done(result) => { - ctx.log(format_args!("Done: {} -> {}", current, result)); - current = result; - } - } - } - - vortex_bail!("Exceeded maximum execution iterations while executing to Columnar") - } -} - -/// Try to interpret an array as columnar (constant or canonical). -fn try_as_columnar(array: &ArrayRef) -> Option { - if let Some(constant) = array.as_opt::() { - Some(Columnar::Constant(constant.clone())) - } else { - array - .as_opt::() - .map(|c| Columnar::Canonical(c.into())) - } -} - -/// Try execute_parent on each child of the array. -fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult> { - for child_idx in 0..array.nchildren() { - let child = array - .nth_child(child_idx) - .vortex_expect("checked nchildren"); - if let Some(result) = child - .vtable() - .execute_parent(&child, array, child_idx, ctx)? - { - result.statistics().inherit_from(array.statistics()); - return Ok(Some(result)); + fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + let result = array.execute_until::(ctx)?; + if let Some(constant) = result.as_opt::() { + Ok(Columnar::Constant(constant.clone())) + } else { + Ok(Columnar::Canonical( + result + .as_opt::() + .map(Canonical::from) + .vortex_expect("execute_until:: must return a columnar array"), + )) } } - Ok(None) } pub enum ColumnarView<'a> { diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index f17780b31e2..0f5516f5785 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -1,13 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::env::VarError; use std::fmt; use std::fmt::Display; use std::sync::Arc; +use std::sync::LazyLock; use std::sync::atomic::AtomicUsize; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::AnyCanonical; @@ -15,6 +19,8 @@ use crate::Array; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; +use crate::matcher::Matcher; +use crate::optimizer::ArrayOptimizer; /// Marker trait for types that an [`ArrayRef`] can be executed into. /// @@ -43,6 +49,110 @@ impl dyn Array + '_ { ) -> VortexResult { E::execute(self, ctx) } + + /// Iteratively execute this array until the [`Matcher`] matches, using an explicit work + /// stack. + /// + /// The scheduler repeatedly: + /// 1. Checks if the current array matches `M` — if so, pops the stack or returns. + /// 2. Runs `execute_parent` on each child for child-driven optimizations. + /// 3. Calls `execute` which returns an [`ExecutionStep`]. + /// + /// Note: the returned array may not match `M`. If execution converges to a canonical form + /// that does not match `M`, the canonical array is returned since no further execution + /// progress is possible. + /// + /// For safety, we will error when the number of execution iterations reaches a configurable + /// maximum (default 128, override with `VORTEX_MAX_ITERATIONS`). + pub fn execute_until( + self: Arc, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + static MAX_ITERATIONS: LazyLock = + LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { + Ok(val) => val.parse::().unwrap_or_else(|e| { + vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid usize: {e}") + }), + Err(VarError::NotPresent) => 128, + Err(VarError::NotUnicode(_)) => { + vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid unicode string") + } + }); + + let mut current = self.optimize()?; + // Stack frames: (parent, child_idx, done_predicate_for_child) + let mut stack: Vec<(ArrayRef, usize, DonePredicate)> = Vec::new(); + + for _ in 0..*MAX_ITERATIONS { + // Check for termination: use the stack frame's done predicate, or the root matcher. + let is_done = stack + .last() + .map_or(M::matches as DonePredicate, |frame| frame.2); + if is_done(current.as_ref()) { + match stack.pop() { + None => { + ctx.log(format_args!("-> {}", current)); + return Ok(current); + } + Some((parent, child_idx, _)) => { + current = parent.with_child(child_idx, current)?; + current = current.optimize()?; + continue; + } + } + } + + // If we've reached canonical form, we can't execute any further regardless + // of whether the matcher matched. + if AnyCanonical::matches(current.as_ref()) { + match stack.pop() { + None => { + ctx.log(format_args!("-> canonical (unmatched) {}", current)); + return Ok(current); + } + Some((parent, child_idx, _)) => { + current = parent.with_child(child_idx, current)?; + current = current.optimize()?; + continue; + } + } + } + + // Try execute_parent (child-driven optimized execution) + if let Some(rewritten) = try_execute_parent(¤t, ctx)? { + ctx.log(format_args!( + "execute_parent rewrote {} -> {}", + current, rewritten + )); + current = rewritten.optimize()?; + continue; + } + + // Execute the array itself + match current.vtable().execute(¤t, ctx)? { + ExecutionStep::ExecuteChild(i, done) => { + let child = current + .nth_child(i) + .vortex_expect("ExecuteChild index in bounds"); + ctx.log(format_args!( + "ExecuteChild({i}): pushing {}, focusing on {}", + current, child + )); + stack.push((current, i, done)); + current = child.optimize()?; + } + ExecutionStep::Done(result) => { + ctx.log(format_args!("Done: {} -> {}", current, result)); + current = result; + } + } + } + + vortex_bail!( + "Exceeded maximum execution iterations ({}) while executing array", + *MAX_ITERATIONS, + ) + } } /// Execution context for batch CPU compute. @@ -189,7 +299,7 @@ impl Executable for ArrayRef { ctx.log(format_args!("-> {}", result.as_ref())); Ok(result) } - ExecutionStep::ExecuteChild(i) => { + ExecutionStep::ExecuteChild(i, _) => { // For single-step execution, handle ExecuteChild by executing the child, // replacing it, and returning the updated array. let child = array.nth_child(i).vortex_expect("valid child index"); @@ -200,25 +310,70 @@ impl Executable for ArrayRef { } } +/// Try execute_parent on each child of the array. +fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult> { + for child_idx in 0..array.nchildren() { + let child = array + .nth_child(child_idx) + .vortex_expect("checked nchildren"); + if let Some(result) = child + .vtable() + .execute_parent(&child, array, child_idx, ctx)? + { + result.statistics().inherit_from(array.statistics()); + return Ok(Some(result)); + } + } + Ok(None) +} + +/// A predicate that determines when an array has reached a desired form during execution. +pub type DonePredicate = fn(&dyn Array) -> bool; + /// The result of a single execution step on an array encoding. /// /// Instead of recursively executing children, encodings return an `ExecutionStep` that tells the /// scheduler what to do next. This enables the scheduler to manage execution iteratively using /// an explicit work stack, run cross-step optimizations, and cache shared sub-expressions. -#[derive(Debug)] pub enum ExecutionStep { - /// Request that the scheduler execute child at index `i` to columnar form, replace it in - /// this array, then re-enter execution on the updated array. + /// Request that the scheduler execute child at the given index, using the provided + /// [`DonePredicate`] to determine when the child is "done", then replace the child in this + /// array and re-enter execution. /// /// Between steps, the scheduler runs reduce/reduce_parent rules to fixpoint, enabling /// cross-step optimization (e.g., pushing scalar functions through newly-decoded children). - ExecuteChild(usize), + /// + /// Use [`ExecutionStep::execute_child`] instead of constructing this variant directly. + ExecuteChild(usize, DonePredicate), /// Execution is complete. The result may be in any encoding — not necessarily canonical. - /// The scheduler will continue executing the result if it is not yet columnar. + /// The scheduler will continue executing the result if it has not yet reached the target form. Done(ArrayRef), } +impl ExecutionStep { + /// Request execution of child at `child_idx` until it matches the given [`Matcher`]. + pub fn execute_child(child_idx: usize) -> Self { + ExecutionStep::ExecuteChild(child_idx, M::matches) + } + + /// Signal that execution is complete with the given result. + pub fn done(result: ArrayRef) -> Self { + ExecutionStep::Done(result) + } +} + +impl fmt::Debug for ExecutionStep { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ExecutionStep::ExecuteChild(idx, _) => { + f.debug_tuple("ExecuteChild").field(idx).finish() + } + ExecutionStep::Done(result) => f.debug_tuple("Done").field(result).finish(), + } + } +} + /// Extension trait for creating an execution context from a session. pub trait VortexSessionExecute { /// Create a new execution context from this session. From 0385273b9107b89b946e4b1b280675f4017675ba Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 13:13:37 -0500 Subject: [PATCH 06/12] feat: migrate single-child wrappers to iterative execute (phase 4b) Replace recursive child.execute() calls with ExecutionStep returns in Slice, Filter, Masked, FoR, and ZigZag vtables. Each now checks if its child is already in the needed form (canonical/primitive/constant) and returns Done directly, or returns ExecuteChild(0)/ColumnarizeChild(0) to let the scheduler handle child execution iteratively. Also handles ConstantArray children explicitly to prevent infinite loops in the scheduler (since constants are already columnar and won't be re-executed). FoR decompress is split into try_fused_decompress and apply_reference for reuse without recursive execution. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- .../fastlanes/src/for/array/for_compress.rs | 11 +++- .../fastlanes/src/for/array/for_decompress.rs | 23 +++++--- encodings/fastlanes/src/for/vtable/mod.rs | 57 ++++++++++++++++++- encodings/zigzag/src/array.rs | 40 +++++++++++-- vortex-array/src/arrays/filter/vtable.rs | 30 +++++++--- vortex-array/src/arrays/masked/vtable/mod.rs | 25 ++++++-- vortex-array/src/arrays/slice/vtable.rs | 35 +++++++----- 7 files changed, 178 insertions(+), 43 deletions(-) diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index 9990df5fb6a..161d2b04439 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -68,8 +68,9 @@ mod test { use super::*; use crate::BitPackedArray; - use crate::r#for::array::for_decompress::decompress; + use crate::r#for::array::for_decompress::apply_reference; use crate::r#for::array::for_decompress::fused_decompress; + use crate::r#for::array::for_decompress::try_fused_decompress; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); @@ -169,7 +170,13 @@ mod test { let expected_unsigned = PrimitiveArray::from_iter(unsigned); assert_arrays_eq!(encoded, expected_unsigned); - let decompressed = decompress(&compressed, &mut SESSION.create_execution_ctx())?; + let mut ctx = SESSION.create_execution_ctx(); + let decompressed = if let Some(result) = try_fused_decompress(&compressed, &mut ctx)? { + result + } else { + let encoded = compressed.encoded().to_primitive(); + apply_reference(&compressed, encoded) + }; array .as_slice::() .iter() diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index f7292a5a06e..b462fa0646d 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -45,23 +45,28 @@ impl + FoR> UnpackStrategy for FoRStrategy } } -pub fn decompress(array: &FoRArray, ctx: &mut ExecutionCtx) -> VortexResult { - let ptype = array.ptype(); - - // Try to do fused unpack. +/// Try the fused BitPacked decompression path. Returns `None` if the child is not BitPacked +/// or the reference type is not unsigned. +pub fn try_fused_decompress( + array: &FoRArray, + ctx: &mut ExecutionCtx, +) -> VortexResult> { if array.reference_scalar().dtype().is_unsigned_int() && let Some(bp) = array.encoded().as_opt::() { return match_each_unsigned_integer_ptype!(array.ptype(), |T| { - fused_decompress::(array, bp, ctx) + fused_decompress::(array, bp, ctx).map(Some) }); } + Ok(None) +} - // TODO(ngates): Do we need this to be into_encoded() somehow? - let encoded = array.encoded().clone().execute::(ctx)?; +/// Apply the FoR reference value to an already-decoded PrimitiveArray. +pub fn apply_reference(array: &FoRArray, encoded: PrimitiveArray) -> PrimitiveArray { + let ptype = array.ptype(); let validity = encoded.validity().clone(); - Ok(match_each_integer_ptype!(ptype, |T| { + match_each_integer_ptype!(ptype, |T| { let min = array .reference_scalar() .as_primitive() @@ -75,7 +80,7 @@ pub fn decompress(array: &FoRArray, ctx: &mut ExecutionCtx) -> VortexResult VortexResult { - Ok(ExecutionStep::Done(decompress(array, ctx)?.into_array())) + // Try fused decompress with BitPacked child (no child execution needed). + if let Some(result) = try_fused_decompress(array, ctx)? { + return Ok(ExecutionStep::Done(result.into_array())); + } + + // If child is already a PrimitiveArray, add the reference value. + if array.encoded().is::() { + let encoded = array.encoded().as_::().clone(); + return Ok(ExecutionStep::Done( + apply_reference(array, encoded).into_array(), + )); + } + + // If child is a constant, compute the result as a constant. + if let Some(constant) = array.encoded().as_opt::() { + let scalar = constant.scalar(); + if scalar.is_null() { + return Ok(ExecutionStep::Done( + ConstantArray::new(Scalar::null(array.dtype().clone()), array.len()) + .into_array(), + )); + } + return Ok(ExecutionStep::Done(match_each_integer_ptype!( + array.ptype(), + |T| { + let enc_val = scalar + .as_primitive() + .typed_value::() + .vortex_expect("constant must be non-null after check"); + let ref_val = array + .reference_scalar() + .as_primitive() + .typed_value::() + .vortex_expect("reference must be non-null"); + ConstantArray::new( + Scalar::primitive( + enc_val.wrapping_add(ref_val), + scalar.dtype().nullability(), + ), + array.len(), + ) + .into_array() + } + ))); + } + + // Otherwise, ask the scheduler to execute the child first. + Ok(ExecutionStep::ExecuteChild(0)) } fn execute_parent( diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index b7229031151..104ab79ca76 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -12,6 +12,9 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ConstantVTable; +use vortex_array::arrays::PrimitiveVTable; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::PType; @@ -149,10 +152,39 @@ impl VTable for ZigZagVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(ExecutionStep::Done( - zigzag_decode(array.encoded().clone().execute(ctx)?).into_array(), - )) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + // If child is already a PrimitiveArray, decode it directly. + if array.encoded().is::() { + let encoded = array.encoded().as_::().clone(); + return Ok(ExecutionStep::Done(zigzag_decode(encoded).into_array())); + } + + // If child is a constant, decode the scalar value. + if let Some(constant) = array.encoded().as_opt::() { + let scalar = constant.scalar(); + if scalar.is_null() { + return Ok(ExecutionStep::Done( + ConstantArray::new(Scalar::null(array.dtype().clone()), array.len()) + .into_array(), + )); + } + let result = match_each_unsigned_integer_ptype!(scalar.as_primitive().ptype(), |P| { + let val = scalar + .as_primitive() + .typed_value::

() + .vortex_expect("constant must be non-null after check"); + Scalar::primitive( + <

::Int>::decode(val), + array.dtype().nullability(), + ) + }); + return Ok(ExecutionStep::Done( + ConstantArray::new(result, array.len()).into_array(), + )); + } + + // Otherwise, ask the scheduler to execute the child first. + Ok(ExecutionStep::ExecuteChild(0)) } fn reduce_parent( diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index fff8c90b992..ef329007ecf 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -13,12 +13,16 @@ use vortex_error::vortex_panic; use vortex_mask::Mask; use vortex_session::VortexSession; +use crate::AnyCanonical; use crate::Array; use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; +use crate::Canonical; use crate::IntoArray; use crate::Precision; +use crate::arrays::ConstantArray; +use crate::arrays::ConstantVTable; use crate::arrays::filter::array::FilterArray; use crate::arrays::filter::execute::execute_filter; use crate::arrays::filter::execute::execute_filter_fast_paths; @@ -156,18 +160,30 @@ impl VTable for FilterVTable { } fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - if let Some(canonical) = execute_filter_fast_paths(array, ctx)? { - return Ok(ExecutionStep::Done(canonical)); + if let Some(result) = execute_filter_fast_paths(array, ctx)? { + return Ok(ExecutionStep::Done(result)); } let Mask::Values(mask_values) = &array.mask else { unreachable!("`execute_filter_fast_paths` handles AllTrue and AllFalse") }; - // We rely on the optimization pass that runs prior to this execution for filter pushdown, - // so now we can just execute the filter without worrying. - Ok(ExecutionStep::Done( - execute_filter(array.child.clone().execute(ctx)?, mask_values).into_array(), - )) + // If child is already canonical, filter it directly. + if let Some(canonical) = array.child.as_opt::() { + return Ok(ExecutionStep::Done( + execute_filter(Canonical::from(canonical), mask_values).into_array(), + )); + } + + // If child is a constant, filtering just changes the length. + if let Some(constant) = array.child.as_opt::() { + return Ok(ExecutionStep::Done( + ConstantArray::new(constant.scalar().clone(), mask_values.true_count()) + .into_array(), + )); + } + + // Otherwise, ask the scheduler to execute the child first. + Ok(ExecutionStep::ExecuteChild(0)) } fn reduce_parent( diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index c63a0687196..5c838363142 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -13,12 +13,15 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::AnyCanonical; use crate::ArrayRef; use crate::Canonical; use crate::EmptyMetadata; use crate::IntoArray; use crate::Precision; use crate::arrays::ConstantArray; +use crate::arrays::ConstantVTable; +use crate::arrays::constant::constant_canonicalize; use crate::arrays::masked::MaskedArray; use crate::arrays::masked::compute::rules::PARENT_RULES; use crate::arrays::masked::mask_validity_canonical; @@ -178,10 +181,24 @@ impl VTable for MaskedVTable { // While we could manually convert the dtype, `mask_validity_canonical` is already O(1) for // `AllTrue` masks (no data copying), so there's no benefit. - let child = array.child().clone().execute::(ctx)?; - Ok(ExecutionStep::Done( - mask_validity_canonical(child, &validity_mask, ctx)?.into_array(), - )) + // If child is already canonical, apply the validity mask directly. + if let Some(canonical) = array.child().as_opt::() { + return Ok(ExecutionStep::Done( + mask_validity_canonical(Canonical::from(canonical), &validity_mask, ctx)? + .into_array(), + )); + } + + // If child is a constant, expand to canonical then apply the validity mask. + if let Some(constant) = array.child().as_opt::() { + let canonical = constant_canonicalize(constant)?; + return Ok(ExecutionStep::Done( + mask_validity_canonical(canonical, &validity_mask, ctx)?.into_array(), + )); + } + + // Otherwise, ask the scheduler to execute the child first. + Ok(ExecutionStep::ColumnarizeChild(0)) } fn reduce_parent( diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index 26cc932a8ea..09fb50caf88 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -20,7 +20,10 @@ use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; +use crate::IntoArray; use crate::Precision; +use crate::arrays::ConstantArray; +use crate::arrays::ConstantVTable; use crate::arrays::slice::array::SliceArray; use crate::arrays::slice::rules::PARENT_RULES; use crate::buffer::BufferHandle; @@ -155,23 +158,25 @@ impl VTable for SliceVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - // Execute the child to get canonical form, then slice it - let Some(canonical) = array.child.as_opt::() else { - // If the child is not canonical, recurse. - return array - .child - .clone() - .execute::(ctx)? - .slice(array.slice_range().clone()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + // If child is already canonical, slice it directly. + if let Some(canonical) = array.child.as_opt::() { + // TODO(ngates): we should inline canonical slice logic here. + return Canonical::from(canonical) + .as_ref() + .slice(array.range.clone()) .map(ExecutionStep::Done); - }; + } + + // If child is a constant, slicing just changes the length. + if let Some(constant) = array.child.as_opt::() { + return Ok(ExecutionStep::Done( + ConstantArray::new(constant.scalar().clone(), array.range.len()).into_array(), + )); + } - // TODO(ngates): we should inline canonical slice logic here. - Canonical::from(canonical) - .as_ref() - .slice(array.range.clone()) - .map(ExecutionStep::Done) + // Otherwise, ask the scheduler to execute the child first. + Ok(ExecutionStep::ExecuteChild(0)) } fn reduce_parent( From 617dd1f3df00b44f45c7433b785d1cf489dc8e16 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 13:38:40 -0500 Subject: [PATCH 07/12] feat: migrate multi-child encodings to iterative execute (phase 4c) Convert Dict, ALP-RD, DecimalByteParts, and Zstd VTable::execute implementations to return ExecutionStep instead of recursively calling execute on children. Each encoding checks if children are already in the expected form (Primitive/Canonical/Constant) before proceeding, returning ExecuteChild(n) to let the scheduler handle child execution. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/alp/src/alp_rd/array.rs | 29 ++++++---- .../src/decimal_byte_parts/mod.rs | 54 +++++++++---------- encodings/zstd/src/array.rs | 7 +-- vortex-array/src/arrays/dict/vtable/mod.rs | 33 +++++++----- 4 files changed, 70 insertions(+), 53 deletions(-) diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 3bd7cdc6812..05364bbe006 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -16,7 +16,9 @@ use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; +use vortex_array::arrays::ConstantVTable; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::PrimitiveVTable; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; @@ -41,7 +43,6 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; -use vortex_mask::Mask; use vortex_session::VortexSession; use crate::alp_rd::kernel::PARENT_KERNELS; @@ -297,17 +298,27 @@ impl VTable for ALPRDVTable { } fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - let left_parts = array.left_parts().clone().execute::(ctx)?; - let right_parts = array.right_parts().clone().execute::(ctx)?; + // Ensure left_parts (child 0) is a PrimitiveArray. + let left_parts = if let Some(primitive) = array.left_parts().as_opt::() { + primitive.clone() + } else if array.left_parts().is::() { + array.left_parts().to_canonical()?.into_primitive() + } else { + return Ok(ExecutionStep::ExecuteChild(0)); + }; + + // Ensure right_parts (child 1) is a PrimitiveArray. + let right_parts = if let Some(primitive) = array.right_parts().as_opt::() { + primitive.clone() + } else if array.right_parts().is::() { + array.right_parts().to_canonical()?.into_primitive() + } else { + return Ok(ExecutionStep::ExecuteChild(1)); + }; // Decode the left_parts using our builtin dictionary. let left_parts_dict = array.left_parts_dictionary(); - - let validity = array - .left_parts() - .validity()? - .to_array(array.len()) - .execute::(ctx)?; + let validity = left_parts.validity_mask()?; let decoded_array = if array.is_f32() { PrimitiveArray::new( diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 9bfbd9c21de..edbd78f90de 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -18,8 +18,9 @@ use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; +use vortex_array::arrays::ConstantVTable; use vortex_array::arrays::DecimalArray; -use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::PrimitiveVTable; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::DecimalDType; @@ -190,8 +191,31 @@ impl VTable for DecimalBytePartsVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - to_canonical_decimal(array, ctx).map(ExecutionStep::Done) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + // Ensure msp (child 0) is a PrimitiveArray. + let prim = if let Some(primitive) = array.msp.as_opt::() { + primitive.clone() + } else if array.msp.is::() { + array.msp.to_canonical()?.into_primitive() + } else { + return Ok(ExecutionStep::ExecuteChild(0)); + }; + + Ok(ExecutionStep::Done(match_each_signed_integer_ptype!( + prim.ptype(), + |P| { + // SAFETY: The primitive array's buffer is already validated with correct type. + // The decimal dtype matches the array's dtype, and validity is preserved. + unsafe { + DecimalArray::new_unchecked( + prim.to_buffer::

(), + *array.decimal_dtype(), + prim.validity().clone(), + ) + } + .into_array() + } + ))) } fn execute_parent( @@ -277,30 +301,6 @@ impl DecimalBytePartsVTable { pub const ID: ArrayId = ArrayId::new_ref("vortex.decimal_byte_parts"); } -/// Converts a DecimalBytePartsArray to its canonical DecimalArray representation. -fn to_canonical_decimal( - array: &DecimalBytePartsArray, - ctx: &mut ExecutionCtx, -) -> VortexResult { - // TODO(joe): support parts len != 1 - let prim = array.msp.clone().execute::(ctx)?; - // Depending on the decimal type and the min/max of the primitive array we can choose - // the correct buffer size - - Ok(match_each_signed_integer_ptype!(prim.ptype(), |P| { - // SAFETY: The primitive array's buffer is already validated with correct type. - // The decimal dtype matches the array's dtype, and validity is preserved. - unsafe { - DecimalArray::new_unchecked( - prim.to_buffer::

(), - *array.decimal_dtype(), - prim.validity().clone(), - ) - } - .into_array() - })) -} - impl OperationsVTable for DecimalBytePartsVTable { fn scalar_at(array: &DecimalBytePartsArray, index: usize) -> VortexResult { // TODO(joe): support parts len != 1 diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index f7263939e4d..9578ebcecb8 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -272,11 +272,8 @@ impl VTable for ZstdVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - array - .decompress()? - .execute::(ctx) - .map(ExecutionStep::Done) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.decompress()?)) } fn reduce_parent( diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index 37d88aaf7ee..b8e0dc51c15 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -14,6 +14,7 @@ use vortex_session::VortexSession; use super::DictArray; use super::DictMetadata; use super::take_canonical; +use crate::AnyCanonical; use crate::Array; use crate::ArrayRef; use crate::Canonical; @@ -23,6 +24,8 @@ use crate::Precision; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::ConstantArray; +use crate::arrays::ConstantVTable; +use crate::arrays::constant::constant_canonicalize; use crate::arrays::dict::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -196,18 +199,24 @@ impl VTable for DictVTable { return Ok(ExecutionStep::Done(canonical)); } - // TODO(joe): if the values are constant return a constant - let values = array.values().clone().execute::(ctx)?; - let codes = array - .codes() - .clone() - .execute::(ctx)? - .into_primitive(); - - // TODO(ngates): if indices are sorted and unique (strict-sorted), then we should delegate to - // the filter function since they're typically optimised for this case. - // TODO(ngates): if indices min is quite high, we could slice self and offset the indices - // such that canonicalize does less work. + // Ensure codes (child 0) are in canonical form. + let codes = if let Some(canonical) = array.codes().as_opt::() { + Canonical::from(canonical) + } else if let Some(constant) = array.codes().as_opt::() { + constant_canonicalize(constant)? + } else { + return Ok(ExecutionStep::ExecuteChild(0)); + }; + let codes = codes.into_primitive(); + + // Ensure values (child 1) are in canonical form. + let values = if let Some(canonical) = array.values().as_opt::() { + Canonical::from(canonical) + } else if let Some(constant) = array.values().as_opt::() { + constant_canonicalize(constant)? + } else { + return Ok(ExecutionStep::ExecuteChild(1)); + }; Ok(ExecutionStep::Done( take_canonical(values, &codes, ctx)?.into_array(), From a3c14a7e3199f8fa9f348dcb12700523adc838cf Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 12:52:43 -0500 Subject: [PATCH 08/12] feat: migrate Zstd decompressors to iterative execute ZstdVTable and ZstdBuffersVTable no longer recursively call .execute() after decompression. Instead they return Done with the decompressed intermediate, letting the scheduler re-enter execution on the result naturally. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/zstd/src/array.rs | 7 ++----- encodings/zstd/src/zstd_buffers.rs | 4 +--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index f7263939e4d..9578ebcecb8 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -272,11 +272,8 @@ impl VTable for ZstdVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - array - .decompress()? - .execute::(ctx) - .map(ExecutionStep::Done) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.decompress()?)) } fn reduce_parent( diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index ff733475a95..3fd2e84ce3e 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -470,9 +470,7 @@ impl VTable for ZstdBuffersVTable { fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let session = ctx.session(); let inner_array = array.decompress_and_build_inner(session)?; - inner_array - .execute::(ctx) - .map(ExecutionStep::Done) + Ok(ExecutionStep::Done(inner_array)) } } From b24eb07417fac1acad7d76d1076e4ba3d9fdd783 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 13:44:27 -0500 Subject: [PATCH 09/12] feat: migrate SharedArray and ZstdBuffers to iterative execute (phase 4d) Convert the last two VTable::execute implementations that had recursive execute calls. SharedVTable now checks if current array (cached or source) is already columnar, otherwise returns ExecuteChild(0). ZstdBuffersVTable decompresses and returns Done(inner_array), letting the scheduler handle further execution of the decompressed result. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/zstd/src/zstd_buffers.rs | 4 +--- vortex-array/src/arrays/shared/vtable.rs | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index ff733475a95..3fd2e84ce3e 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -470,9 +470,7 @@ impl VTable for ZstdBuffersVTable { fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let session = ctx.session(); let inner_array = array.decompress_and_build_inner(session)?; - inner_array - .execute::(ctx) - .map(ExecutionStep::Done) + Ok(ExecutionStep::Done(inner_array)) } } diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index 9a7ca38b263..8e1829511b1 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -8,12 +8,15 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::AnyCanonical; use crate::ArrayRef; use crate::Canonical; use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionStep; +use crate::IntoArray; use crate::Precision; +use crate::arrays::ConstantVTable; use crate::arrays::shared::SharedArray; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -145,10 +148,15 @@ impl VTable for SharedVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - array - .get_or_compute(|source| source.clone().execute::(ctx)) - .map(ExecutionStep::Done) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + let current = array.current_array_ref(); + if let Some(canonical) = current.as_opt::() { + return Ok(ExecutionStep::Done(Canonical::from(canonical).into_array())); + } + if current.as_opt::().is_some() { + return Ok(ExecutionStep::Done(current.clone())); + } + Ok(ExecutionStep::ExecuteChild(0)) } } impl OperationsVTable for SharedVTable { From e7e9518d9fee564956ff86a10844d69700b79bc5 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 14:00:19 -0500 Subject: [PATCH 10/12] chore: regenerate public-api.lock files Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/zstd/public-api.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 9f351777a4c..c863b07ac90 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -196,7 +196,7 @@ pub fn vortex_zstd::ZstdVTable::deserialize(bytes: &[u8], _dtype: &vortex_array: pub fn vortex_zstd::ZstdVTable::dtype(array: &vortex_zstd::ZstdArray) -> &vortex_array::dtype::DType -pub fn vortex_zstd::ZstdVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::ZstdVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId From 2f987cf2f492b9fe97980a4f9084c04ccaa3c066 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 14:02:12 -0500 Subject: [PATCH 11/12] chore: regenerate public-api.lock files Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/decimal-byte-parts/public-api.lock | 2 +- encodings/zigzag/public-api.lock | 2 +- encodings/zstd/public-api.lock | 2 +- vortex-array/public-api.lock | 8 ++++---- vortex-array/src/arrays/masked/vtable/mod.rs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 41a2a23a086..aab5e297326 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -108,7 +108,7 @@ pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::deserialize(bytes: &[u pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::dtype(array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> &vortex_array::dtype::DType -pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index 1f9fca6c735..45307c66270 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -96,7 +96,7 @@ pub fn vortex_zigzag::ZigZagVTable::deserialize(_bytes: &[u8], _dtype: &vortex_a pub fn vortex_zigzag::ZigZagVTable::dtype(array: &vortex_zigzag::ZigZagArray) -> &vortex_array::dtype::DType -pub fn vortex_zigzag::ZigZagVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zigzag::ZigZagVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zigzag::ZigZagVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 9f351777a4c..c863b07ac90 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -196,7 +196,7 @@ pub fn vortex_zstd::ZstdVTable::deserialize(bytes: &[u8], _dtype: &vortex_array: pub fn vortex_zstd::ZstdVTable::dtype(array: &vortex_zstd::ZstdArray) -> &vortex_array::dtype::DType -pub fn vortex_zstd::ZstdVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::ZstdVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index b885c08284c..a0e6c5cbb50 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -2978,7 +2978,7 @@ pub fn vortex_array::arrays::SharedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::SharedVTable::dtype(array: &vortex_array::arrays::SharedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SharedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3138,7 +3138,7 @@ pub fn vortex_array::arrays::SliceVTable::deserialize(_bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::SliceVTable::dtype(array: &vortex_array::arrays::SliceArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SliceVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17348,7 +17348,7 @@ pub fn vortex_array::arrays::SharedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::SharedVTable::dtype(array: &vortex_array::arrays::SharedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SharedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17402,7 +17402,7 @@ pub fn vortex_array::arrays::SliceVTable::deserialize(_bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::SliceVTable::dtype(array: &vortex_array::arrays::SliceArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SliceVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 5c838363142..b2a8099bf1b 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -198,7 +198,7 @@ impl VTable for MaskedVTable { } // Otherwise, ask the scheduler to execute the child first. - Ok(ExecutionStep::ColumnarizeChild(0)) + Ok(ExecutionStep::ExecuteChild(0)) } fn reduce_parent( From a34d0d8bc02bf4fcbb95a7168defd35d221c0fe8 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 10 Mar 2026 13:48:17 +0000 Subject: [PATCH 12/12] chore[array]: iterative execute Signed-off-by: Joe Isaacs --- encodings/alp/src/alp_rd/array.rs | 22 +------ vortex-array/src/arrays/dict/vtable/mod.rs | 68 +++++++--------------- vortex-array/src/executor.rs | 17 ++++++ 3 files changed, 41 insertions(+), 66 deletions(-) diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 2fc4fc70d6e..583b20a5805 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -5,7 +5,6 @@ use std::fmt::Debug; use std::hash::Hash; use itertools::Itertools; -use vortex_array::AnyCanonical; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; @@ -17,7 +16,6 @@ use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; -use vortex_array::arrays::ConstantVTable; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::PrimitiveVTable; use vortex_array::buffer::BufferHandle; @@ -26,6 +24,7 @@ use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::patches::Patches; use vortex_array::patches::PatchesMetadata; +use vortex_array::require_child; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSetRef; @@ -299,23 +298,8 @@ impl VTable for ALPRDVTable { } fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - // Ensure left_parts (child 0) is a PrimitiveArray. - let left_parts = if let Some(primitive) = array.left_parts().as_opt::() { - primitive.clone() - } else if array.left_parts().is::() { - array.left_parts().to_canonical()?.into_primitive() - } else { - return Ok(ExecutionStep::execute_child::(0)); - }; - - // Ensure right_parts (child 1) is a PrimitiveArray. - let right_parts = if let Some(primitive) = array.right_parts().as_opt::() { - primitive.clone() - } else if array.right_parts().is::() { - array.right_parts().to_canonical()?.into_primitive() - } else { - return Ok(ExecutionStep::execute_child::(1)); - }; + let left_parts = require_child!(array.left_parts(), 0 => PrimitiveVTable).clone(); + let right_parts = require_child!(array.right_parts(), 1 => PrimitiveVTable).clone(); // Decode the left_parts using our builtin dictionary. let left_parts_dict = array.left_parts_dictionary(); diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index c0119cd566c..215335e10fd 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -24,8 +24,7 @@ use crate::Precision; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::ConstantArray; -use crate::arrays::ConstantVTable; -use crate::arrays::constant::constant_canonicalize; +use crate::arrays::PrimitiveVTable; use crate::arrays::dict::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -35,6 +34,7 @@ use crate::executor::ExecutionCtx; use crate::executor::ExecutionStep; use crate::hash::ArrayEq; use crate::hash::ArrayHash; +use crate::require_child; use crate::scalar::Scalar; use crate::serde::ArrayChildren; use crate::stats::StatsSetRef; @@ -195,31 +195,29 @@ impl VTable for DictVTable { } fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - if let Some(canonical) = execute_fast_path(array, ctx)? { - return Ok(ExecutionStep::Done(canonical)); + if array.is_empty() { + let result_dtype = array + .dtype() + .union_nullability(array.codes().dtype().nullability()); + return Ok(ExecutionStep::Done( + Canonical::empty(&result_dtype).into_array(), + )); } - // Ensure codes (child 0) are in canonical form. - let codes = if let Some(canonical) = array.codes().as_opt::() { - Canonical::from(canonical) - } else if let Some(constant) = array.codes().as_opt::() { - constant_canonicalize(constant)? - } else { - return Ok(ExecutionStep::execute_child::(0)); - }; - let codes = codes.into_primitive(); - - // Ensure values (child 1) are in canonical form. - let values = if let Some(canonical) = array.values().as_opt::() { - Canonical::from(canonical) - } else if let Some(constant) = array.values().as_opt::() { - constant_canonicalize(constant)? - } else { - return Ok(ExecutionStep::execute_child::(1)); - }; + let codes = require_child!(array.codes(), 0 => PrimitiveVTable); + + if codes.all_invalid()? { + return Ok(ExecutionStep::Done( + ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.codes.len()) + .into_array(), + )); + } + + let values = require_child!(array.values(), 1 => AnyCanonical); + let values = Canonical::from(values); Ok(ExecutionStep::Done( - take_canonical(values, &codes, ctx)?.into_array(), + take_canonical(values, codes, ctx)?.into_array(), )) } @@ -240,27 +238,3 @@ impl VTable for DictVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } } - -/// Check for fast-path execution conditions. -pub(super) fn execute_fast_path( - array: &DictArray, - _ctx: &mut ExecutionCtx, -) -> VortexResult> { - // Empty array - nothing to do - if array.is_empty() { - let result_dtype = array - .dtype() - .union_nullability(array.codes().dtype().nullability()); - return Ok(Some(Canonical::empty(&result_dtype).into_array())); - } - - // All codes are null - result is all nulls - if array.codes.all_invalid()? { - return Ok(Some( - ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.codes.len()) - .into_array(), - )); - } - - Ok(None) -} diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index da05450f8de..1e6ae528090 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -387,6 +387,23 @@ impl fmt::Debug for ExecutionStep { } } +/// Require that a child array matches `$M`. If it does, evaluates to the matched value. +/// Otherwise, early-returns `Ok(ExecutionStep::execute_child::<$M>($idx))`. +/// +/// ```ignore +/// let codes = require_child!(array.codes(), 0 => PrimitiveVTable); +/// let values = require_child!(array.values(), 1 => AnyCanonical); +/// ``` +#[macro_export] +macro_rules! require_child { + ($child:expr, $idx:expr => $M:ty) => { + match $child.as_opt::<$M>() { + Some(c) => c, + None => return Ok($crate::ExecutionStep::execute_child::<$M>($idx)), + } + }; +} + /// Extension trait for creating an execution context from a session. pub trait VortexSessionExecute { /// Create a new execution context from this session.