Split layouts into a commons crate

Cargo.toml

@@ -21,6 +21,7 @@ members = [
     "vortex-json",
     "vortex-compressor",
     "vortex-btrblocks",
+    "vortex-layout-commons",
     "vortex-layout",
     "vortex-scan",
     "vortex-file",
@@ -316,6 +317,7 @@ vortex-io = { version = "0.1.0", path = "./vortex-io", default-features = false
 vortex-ipc = { version = "0.1.0", path = "./vortex-ipc", default-features = false }
 vortex-json = { version = "0.1.0", path = "./vortex-json", default-features = false }
 vortex-layout = { version = "0.1.0", path = "./vortex-layout", default-features = false }
+vortex-layout-commons = { version = "0.1.0", path = "./vortex-layout-commons", default-features = false }
 vortex-mask = { version = "0.1.0", path = "./vortex-mask", default-features = false }
 vortex-metrics = { version = "0.1.0", path = "./vortex-metrics", default-features = false }
 vortex-onpair = { version = "0.1.0", path = "./encodings/onpair", default-features = false }

vortex-cuda/src/file.rs

@@ -174,7 +174,9 @@ mod tests {
     use vortex::array::dtype::StructFields;
     use vortex::buffer::ByteBuffer;
     use vortex::layout::layouts::flat::FlatLayout;
+    use vortex::layout::layouts::flat::FlatLayoutExt;
     use vortex::layout::layouts::zoned::ZonedLayout;
+    use vortex::layout::layouts::zoned::ZonedLayoutExt;
     use vortex::session::registry::ReadContext;
 
     use super::*;

vortex-layout-commons/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "vortex-layout-commons"
+authors = { workspace = true }
+categories = { workspace = true }
+description = "Core traits and types for Vortex layouts"
+edition = { workspace = true }
+homepage = { workspace = true }
+include = { workspace = true }
+keywords = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+arcref = { workspace = true }
+async-trait = { workspace = true }
+flatbuffers = { workspace = true }
+futures = { workspace = true, features = ["alloc", "async-await"] }
+itertools = { workspace = true }
+once_cell = { workspace = true, features = ["parking_lot"] }
+parking_lot = { workspace = true }
+pin-project-lite = { workspace = true }
+termtree = { workspace = true }
+vortex-array = { workspace = true }
+vortex-buffer = { workspace = true }
+vortex-error = { workspace = true }
+vortex-flatbuffers = { workspace = true, features = ["layout"] }
+vortex-mask = { workspace = true }
+vortex-session = { workspace = true }
+vortex-utils = { workspace = true }
+
+[dev-dependencies]
+rstest = { workspace = true }
+
+[lints]
+workspace = true

vortex-layout-commons/src/children.rs

@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use vortex_array::dtype::DType;
+use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
+
+use crate::LayoutRef;
+
+/// Abstract way of accessing the children of a layout.
+///
+/// This allows layout trees to use lazy serialized children as well as in-memory children.
+pub trait LayoutChildren: 'static + Send + Sync {
+    fn to_arc(&self) -> Arc<dyn LayoutChildren>;
+
+    fn child(&self, idx: usize, dtype: &DType) -> VortexResult<LayoutRef>;
+
+    fn child_row_count(&self, idx: usize) -> u64;
+
+    fn nchildren(&self) -> usize;
+
+    /// Returns `true` if the child at `idx` is known, without materializing it, to be indivisible.
+    ///
+    /// Implementations must conservatively return `false` when answering would require
+    /// materializing the child.
+    fn child_is_indivisible(&self, _idx: usize) -> bool {
+        false
+    }
+}
+
+impl Debug for dyn LayoutChildren {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LayoutChildren")
+            .field("nchildren", &self.nchildren())
+            .finish()
+    }
+}
+
+impl LayoutChildren for Arc<dyn LayoutChildren> {
+    fn to_arc(&self) -> Arc<dyn LayoutChildren> {
+        Arc::clone(self)
+    }
+
+    fn child(&self, idx: usize, dtype: &DType) -> VortexResult<LayoutRef> {
+        self.as_ref().child(idx, dtype)
+    }
+
+    fn child_row_count(&self, idx: usize) -> u64 {
+        self.as_ref().child_row_count(idx)
+    }
+
+    fn nchildren(&self) -> usize {
+        self.as_ref().nchildren()
+    }
+
+    fn child_is_indivisible(&self, idx: usize) -> bool {
+        self.as_ref().child_is_indivisible(idx)
+    }
+}
+
+/// In-memory owned layout children.
+#[derive(Clone)]
+pub struct OwnedLayoutChildren(Vec<LayoutRef>);
+
+impl OwnedLayoutChildren {
+    pub fn layout_children(children: Vec<LayoutRef>) -> Arc<dyn LayoutChildren> {
+        Arc::new(Self(children))
+    }
+}
+
+/// Create an in-memory child adapter from owned layout references.
+pub fn layout_children(children: Vec<LayoutRef>) -> Arc<dyn LayoutChildren> {
+    OwnedLayoutChildren::layout_children(children)
+}
+
+impl LayoutChildren for OwnedLayoutChildren {
+    fn to_arc(&self) -> Arc<dyn LayoutChildren> {
+        Arc::new(self.clone())
+    }
+
+    fn child(&self, idx: usize, dtype: &DType) -> VortexResult<LayoutRef> {
+        if idx >= self.0.len() {
+            vortex_bail!("Child index out of bounds: {} of {}", idx, self.0.len());
+        }
+        let child = &self.0[idx];
+        if child.dtype() != dtype {
+            vortex_bail!("Child dtype mismatch: {} != {}", child.dtype(), dtype);
+        }
+        Ok(Arc::clone(child))
+    }
+
+    fn child_row_count(&self, idx: usize) -> u64 {
+        self.0[idx].row_count()
+    }
+
+    fn nchildren(&self) -> usize {
+        self.0.len()
+    }
+
+    fn child_is_indivisible(&self, idx: usize) -> bool {
+        self.0[idx].dyn_is_indivisible()
+    }
+}