Define RowFn and RowVisitor

vortex-array/Cargo.toml

@@ -79,6 +79,8 @@ cudarc = ["dep:cudarc"]
 table-display = ["dep:tabled"]
 _test-harness = ["dep:goldenfile", "dep:rstest", "dep:rstest_reuse"]
 serde = ["dep:serde", "vortex-buffer/serde", "vortex-mask/serde"]
+# Exposes experimental row-function APIs without compatibility guarantees.
+unstable_row_fns = []
 
 [dev-dependencies]
 divan = { workspace = true }
@@ -90,7 +92,11 @@ rstest = { workspace = true }
 serde_json = { workspace = true }
 serde_test = { workspace = true }
 test-with = { workspace = true }
-vortex-array = { path = ".", features = ["_test-harness", "table-display"] }
+vortex-array = { path = ".", features = [
+    "_test-harness",
+    "table-display",
+    "unstable_row_fns",
+] }
 
 [[bench]]
 name = "aggregate_max"

vortex-array/src/scalar_fn/mod.rs

@@ -6,6 +6,12 @@
 //! This module contains the [`ScalarFnVTable`] trait and all built-in scalar function
 //! implementations. Expressions ([`crate::expr::Expression`]) reference scalar functions
 //! at each node.
+//!
+//! Strict functions with row-at-a-time kernels can implement `unstable::row::RowFn`. It handles
+//! decoding, constants, null propagation, output construction, and validity. This API requires the
+//! `unstable_row_fns` feature and has no compatibility guarantees. Implement [`ScalarFnVTable`]
+//! directly for columnar kernels and functions that alias an input or can produce null from valid
+//! inputs.
 
 use vortex_session::registry::Id;
 
@@ -35,6 +41,12 @@ pub use options::*;
 mod signature;
 pub use signature::*;
 
+#[cfg(feature = "unstable_row_fns")]
+pub mod unstable;
+#[cfg(not(feature = "unstable_row_fns"))]
+#[allow(dead_code, unused_imports)]
+pub(crate) mod unstable;
+
 pub mod fns;
 pub mod internal;
 pub mod session;

vortex-array/src/scalar_fn/unstable/mod.rs

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Experimental scalar-function APIs without compatibility guarantees.
+//!
+//! These APIs can change or disappear without a deprecation period. External users must enable
+//! the corresponding `unstable_*` Cargo feature before importing them.
+
+pub mod row;

vortex-array/src/scalar_fn/unstable/row/mod.rs

@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Experimental support for strict scalar functions computed one row at a time.
+//!
+//! This module is experimental and has no compatibility guarantees. External users must enable
+//! the `unstable_row_fns` Cargo feature before importing it.
+//!
+//! A [`RowFn`] describes the typed operation while the framework owns columnar concerns such as
+//! decoding, constant handling, null propagation, allocation, and validity. Its
+//! [`RowFn::dispatch`] implementation uses a [`RowVisitor`] to select an [`ElementTuple`] and
+//! either an [`OutputElement`] or [`OutputSink`] for each supported dtype combination.
+//!
+//! Prepared visits move work derived from constant operands outside the hot loop. Deferred visits
+//! reduce compact failure evidence in that loop and retry only valid rows when null payloads may
+//! have caused the failure.
+
+mod row_fn;
+pub use row_fn::RowFn;
+
+mod types;
+pub use types::ElementTuple;
+pub use types::IndexedElementTuple;
+pub use types::InitializedElement;
+pub use types::InputElement;
+pub use types::OutputElement;
+pub use types::OutputSink;
+pub use types::SinkResult;
+pub use types::UninitElementSink;
+
+mod visitor;
+pub use visitor::RowVisitor;
+
+mod vtable;
+pub use vtable::execute_rows;
+pub use vtable::row_fn_return_dtype;

vortex-array/src/scalar_fn/unstable/row/row_fn.rs

@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! The [`RowFn`] contract for scalar functions whose natural kernel computes one row at a time.
+//!
+//! Implementations declare their arity and fallibility, then use [`RowFn::dispatch`] to select the
+//! typed row signature for each supported dtype combination. Optional methods provide
+//! serialization without putting persistence plumbing in the row kernel.
+
+use std::fmt::Debug;
+use std::fmt::Display;
+use std::hash::Hash;
+
+use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
+use vortex_session::VortexSession;
+
+use super::visitor::RowVisitor;
+use crate::dtype::DType;
+use crate::scalar_fn::ScalarFnId;
+
+/// A scalar function computed one row at a time.
+///
+/// Declare argument names and use [`dispatch`](Self::dispatch) to select element and output types.
+/// Every implementation receives the standard [`ScalarFnVTable`]. A public type that needs custom
+/// vtable hooks can delegate its row kernel through [`row_fn_return_dtype`] and [`execute_rows`].
+///
+/// [`ScalarFnVTable`]: crate::scalar_fn::ScalarFnVTable
+/// [`execute_rows`]: crate::scalar_fn::unstable::row::execute_rows
+/// [`row_fn_return_dtype`]: crate::scalar_fn::unstable::row::row_fn_return_dtype
+pub trait RowFn: 'static + Sized + Clone + Send + Sync {
+    /// Options for this function, or [`EmptyOptions`](crate::scalar_fn::EmptyOptions) for none.
+    type Options: 'static + Send + Sync + Clone + Debug + Display + PartialEq + Eq + Hash;
+
+    /// The arguments in display order. Its length is the function's exact arity.
+    const ARG_NAMES: &'static [&'static str];
+
+    /// Whether any dispatch can raise a semantic error.
+    ///
+    /// See [`ScalarFnVTable::is_fallible`](crate::scalar_fn::ScalarFnVTable::is_fallible) for a
+    /// more detailed explanation of semantic errors.
+    ///
+    /// The framework checks dispatched element and result types. A conservative `true` is allowed.
+    const FALLIBLE: bool;
+
+    /// Returns the ID of the scalar function.
+    fn id(&self) -> ScalarFnId;
+
+    /// Serialize this function's options, or return `None` when the function is not serializable.
+    fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
+        _ = options;
+        Ok(None)
+    }
+
+    /// Restore options written by [`serialize`](Self::serialize).
+    fn deserialize(
+        &self,
+        _metadata: &[u8],
+        _session: &VortexSession,
+    ) -> VortexResult<Self::Options> {
+        vortex_bail!("Expression {} is not deserializable", self.id())
+    }
+
+    /// Choose element types for these input dtypes and visit the framework with them.
+    ///
+    /// Planning and execution both call this method, so its result **must** depend only on
+    /// `options` and `args`. Cross-argument dtype validation belongs here.
+    fn dispatch<V: RowVisitor<Self::Options>>(
+        &self,
+        options: &Self::Options,
+        args: &[DType],
+        visitor: V,
+    ) -> VortexResult<V::VisitResult>;
+}

vortex-array/src/scalar_fn/unstable/row/types/element/bool.rs

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use vortex_buffer::BitBuffer;
+use vortex_error::VortexResult;
+use vortex_error::vortex_ensure;
+
+use crate::ArrayRef;
+use crate::ExecutionCtx;
+use crate::IntoArray;
+use crate::arrays::BoolArray;
+use crate::dtype::DType;
+use crate::dtype::Nullability;
+use crate::scalar_fn::unstable::row::InputElement;
+use crate::scalar_fn::unstable::row::OutputElement;
+use crate::validity::Validity;
+
+// SAFETY: the per-row view is a bit buffer, and its reported length is the buffer length.
+unsafe impl InputElement for bool {
+    type Column = BitBuffer;
+    type View<'a> = &'a BitBuffer;
+    type Elem<'a> = bool;
+
+    // Every bit of the buffer is readable, valid or not.
+    const DENSE_SAFE: bool = true;
+    const DECODE_FALLIBLE: bool = false;
+
+    fn validate(dtype: &DType) -> VortexResult<()> {
+        vortex_ensure!(
+            matches!(dtype, DType::Bool(_)),
+            "expected a Bool column, got {dtype}",
+        );
+        Ok(())
+    }
+
+    fn decode(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self::Column> {
+        Ok(array.execute::<BoolArray>(ctx)?.into_bit_buffer())
+    }
+
+    fn can_decode_null_tolerant(_array: &ArrayRef) -> VortexResult<bool> {
+        Ok(true)
+    }
+
+    fn get(column: &Self::Column, index: usize) -> bool {
+        column.value(index)
+    }
+
+    fn view(column: &Self::Column) -> Self::View<'_> {
+        column
+    }
+
+    fn view_len(view: &Self::View<'_>) -> usize {
+        view.len()
+    }
+
+    fn get_from_view<'a>(view: &Self::View<'a>, index: usize) -> bool
+    where
+        Self: 'a,
+    {
+        view.value(index)
+    }
+
+    unsafe fn get_from_view_unchecked<'a>(view: &Self::View<'a>, index: usize) -> bool
+    where
+        Self: 'a,
+    {
+        // SAFETY: forwarded from this method's contract.
+        unsafe { view.value_unchecked(index) }
+    }
+}
+
+impl OutputElement for bool {
+    fn element_dtype() -> DType {
+        DType::Bool(Nullability::NonNullable)
+    }
+
+    fn build(values: Vec<Self>) -> ArrayRef {
+        // `From<Vec<bool>>` uses the bulk bit-packing path.
+        BoolArray::new(BitBuffer::from(values), Validity::NonNullable).into_array()
+    }
+}

vortex-array/src/scalar_fn/unstable/row/types/element/input.rs

@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Typed decoding and row access for one input column.
+//!
+//! [`InputElement`] separates invocation-wide decoding from the checked and unchecked access paths
+//! used by row kernels.
+
+use vortex_error::VortexResult;
+
+use crate::ArrayRef;
+use crate::ExecutionCtx;
+use crate::dtype::DType;
+
+/// An element type that can be read row-wise out of an input column.
+///
+/// # Safety
+///
+/// For every view returned by [`view`](Self::view), every index below
+/// [`view_len`](Self::view_len) **must** satisfy the safety contract of
+/// [`get_from_view_unchecked`](Self::get_from_view_unchecked). Shared execution relies on this
+/// proof to perform unchecked reads after one pre-loop length check.
+pub unsafe trait InputElement: 'static {
+    /// The decoded column representation supporting `O(1)` row access.
+    type Column;
+
+    /// The row-loop view of a decoded column.
+    ///
+    /// This can borrow a cheaper representation than [`Column`](Self::Column). Primitive elements,
+    /// for example, expose a slice so its pointer and length are loop invariants rather than
+    /// re-reading a [`Buffer`](vortex_buffer::Buffer) descriptor for every row.
+    type View<'a>;
+
+    /// The borrowed element value handed to a row closure.
+    type Elem<'a>;
+
+    /// Whether every dense decode and access path tolerates rows that are null in the input.
+    ///
+    /// Arrays guarantee payloads only for valid rows. Set this to `true` only when every decode and
+    /// access method remains safe for null rows. Dense execution can pass unspecified values from
+    /// null rows to the row closure.
+    const DENSE_SAFE: bool;
+
+    /// Whether [`decode`](Self::decode) can fail on _legal_ input data.
+    ///
+    /// This excludes infrastructural failures such as IO or allocation.
+    const DECODE_FALLIBLE: bool;
+
+    /// Validate that `dtype` is an acceptable input column dtype for this element type.
+    fn validate(dtype: &DType) -> VortexResult<()>;
+
+    /// Decode `array` into its column representation.
+    ///
+    /// Called once per row-kernel invocation, including deferred-error retries. Hoist dtype checks,
+    /// downcasts, and other invocation-invariant work into this method.
+    fn decode(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self::Column>;
+
+    /// Whether [`decode_null_tolerant`](Self::decode_null_tolerant) can decode this array.
+    ///
+    /// The conservative default declines. An implementation whose ordinary decode is safe and
+    /// infallible over null payloads can return `true`. Other implementations can inspect `array`
+    /// and opt in only for supported representations.
+    fn can_decode_null_tolerant(_array: &ArrayRef) -> VortexResult<bool> {
+        Ok(false)
+    }
+
+    /// Decode `array` _without_ assuming every row is valid, or return `Ok(None)` when this element
+    /// cannot decode this particular array.
+    ///
+    /// Override this for a non-dense-safe representation that can still place safe placeholders in
+    /// null slots. The skip-invalid executor never reads those slots.
+    fn decode_null_tolerant(
+        array: ArrayRef,
+        ctx: &mut ExecutionCtx,
+    ) -> VortexResult<Option<Self::Column>> {
+        if Self::can_decode_null_tolerant(&array)? {
+            Self::decode(array, ctx).map(Some)
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Read one row without repeating batch-constant work from [`decode`](Self::decode).
+    fn get(column: &Self::Column, index: usize) -> Self::Elem<'_>;
+
+    /// Borrow the representation used when this argument varies within the batch.
+    ///
+    /// Called once before the hot loop. Constants do not use this view because the tuple adapter
+    /// keeps their one-row decoded representation separate.
+    fn view(column: &Self::Column) -> Self::View<'_>;
+
+    /// Number of rows addressable through a [`View`](Self::View).
+    ///
+    /// Every index below this length must be valid for
+    /// [`get_from_view_unchecked`](Self::get_from_view_unchecked).
+    fn view_len(view: &Self::View<'_>) -> usize;
+
+    /// Read one row from a [`View`](Self::View).
+    fn get_from_view<'a>(view: &Self::View<'a>, index: usize) -> Self::Elem<'a>
+    where
+        Self: 'a;
+
+    /// Read one row without checking that `index` is in bounds.
+    ///
+    /// # Safety
+    ///
+    /// `index` must be less than [`view_len`](Self::view_len) for `view`.
+    unsafe fn get_from_view_unchecked<'a>(view: &Self::View<'a>, index: usize) -> Self::Elem<'a>
+    where
+        Self: 'a,
+    {
+        Self::get_from_view(view, index)
+    }
+}

vortex-array/src/scalar_fn/unstable/row/types/element/mod.rs

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! The element types a row function can read and produce.
+//!
+//! [`InputElement::Elem`] can borrow from its decoded column. Owned row computations return an
+//! [`OutputElement`]. Runtime-shaped outputs use an
+//! [`OutputSink`](crate::scalar_fn::unstable::row::OutputSink).
+
+mod bool;
+
+mod input;
+pub use input::InputElement;
+
+mod output;
+pub use output::OutputElement;
+
+mod primitive;
+
+mod tuple;
+pub use tuple::ElementTuple;
+pub use tuple::IndexedElementTuple;