Skip to content

array slots #6870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
joseph-isaacs wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

array slots #6870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions encodings/alp/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,14 +218,22 @@ pub fn vortex_alp::ALPRDVTable::nbuffers(_array: &vortex_alp::ALPRDArray) -> usi

pub fn vortex_alp::ALPRDVTable::nchildren(array: &vortex_alp::ALPRDArray) -> usize

pub fn vortex_alp::ALPRDVTable::nslots(_array: &vortex_alp::ALPRDArray) -> usize

pub fn vortex_alp::ALPRDVTable::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>

pub fn vortex_alp::ALPRDVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>

pub fn vortex_alp::ALPRDVTable::slot(array: &vortex_alp::ALPRDArray, idx: usize) -> &core::option::Option<vortex_array::array::ArrayRef>

pub fn vortex_alp::ALPRDVTable::slot_name(_array: &vortex_alp::ALPRDArray, idx: usize) -> &str

pub fn vortex_alp::ALPRDVTable::stats(array: &vortex_alp::ALPRDArray) -> vortex_array::stats::array::StatsSetRef<'_>

pub fn vortex_alp::ALPRDVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec<vortex_array::array::ArrayRef>) -> vortex_error::VortexResult<()>

pub fn vortex_alp::ALPRDVTable::with_slots(array: &mut vortex_alp::ALPRDArray, slots: alloc::vec::Vec<core::option::Option<vortex_array::array::ArrayRef>>) -> vortex_error::VortexResult<()>

impl vortex_array::vtable::operations::OperationsVTable<vortex_alp::ALPRDVTable> for vortex_alp::ALPRDVTable

pub fn vortex_alp::ALPRDVTable::scalar_at(array: &vortex_alp::ALPRDArray, index: usize) -> vortex_error::VortexResult<vortex_array::scalar::Scalar>
Expand Down Expand Up @@ -322,14 +330,22 @@ pub fn vortex_alp::ALPVTable::nbuffers(_array: &vortex_alp::ALPArray) -> usize

pub fn vortex_alp::ALPVTable::nchildren(array: &vortex_alp::ALPArray) -> usize

pub fn vortex_alp::ALPVTable::nslots(_array: &vortex_alp::ALPArray) -> usize

pub fn vortex_alp::ALPVTable::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>

pub fn vortex_alp::ALPVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>

pub fn vortex_alp::ALPVTable::slot(array: &vortex_alp::ALPArray, idx: usize) -> &core::option::Option<vortex_array::array::ArrayRef>

pub fn vortex_alp::ALPVTable::slot_name(_array: &vortex_alp::ALPArray, idx: usize) -> &str

pub fn vortex_alp::ALPVTable::stats(array: &vortex_alp::ALPArray) -> vortex_array::stats::array::StatsSetRef<'_>

pub fn vortex_alp::ALPVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec<vortex_array::array::ArrayRef>) -> vortex_error::VortexResult<()>

pub fn vortex_alp::ALPVTable::with_slots(array: &mut vortex_alp::ALPArray, slots: alloc::vec::Vec<core::option::Option<vortex_array::array::ArrayRef>>) -> vortex_error::VortexResult<()>

impl vortex_array::vtable::operations::OperationsVTable<vortex_alp::ALPVTable> for vortex_alp::ALPVTable

pub fn vortex_alp::ALPVTable::scalar_at(array: &vortex_alp::ALPArray, index: usize) -> vortex_error::VortexResult<vortex_array::scalar::Scalar>
Expand Down
59 changes: 48 additions & 11 deletions encodings/alp/src/alp/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ impl VTable for ALPVTable {
}

fn len(array: &ALPArray) -> usize {
array.encoded.len()
array.encoded().len()
}

fn dtype(array: &ALPArray) -> &DType {
Expand All @@ -72,14 +72,14 @@ impl VTable for ALPVTable {

fn array_hash<H: std::hash::Hasher>(array: &ALPArray, state: &mut H, precision: Precision) {
array.dtype.hash(state);
array.encoded.array_hash(state, precision);
array.encoded().array_hash(state, precision);
array.exponents.hash(state);
array.patches.array_hash(state, precision);
}

fn array_eq(array: &ALPArray, other: &ALPArray, precision: Precision) -> bool {
array.dtype == other.dtype
&& array.encoded.array_eq(&other.encoded, precision)
&& array.encoded().array_eq(other.encoded(), precision)
&& array.exponents == other.exponents
&& array.patches.array_eq(&other.patches, precision)
}
Expand All @@ -96,6 +96,29 @@ impl VTable for ALPVTable {
None
}

fn nslots(_array: &ALPArray) -> usize {
NUM_SLOTS
}

fn slot(array: &ALPArray, idx: usize) -> &Option<ArrayRef> {
&array.slots[idx]
}

fn slot_name(_array: &ALPArray, idx: usize) -> &str {
SLOT_NAMES[idx]
}

fn with_slots(array: &mut ALPArray, slots: Vec<Option<ArrayRef>>) -> VortexResult<()> {
vortex_ensure!(
slots.len() == NUM_SLOTS,
"ALPArray expects {} slots, got {}",
NUM_SLOTS,
slots.len()
);
array.slots = slots;
Ok(())
}

fn nchildren(array: &ALPArray) -> usize {
1 + array.patches().map_or(0, patches_nchildren)
}
Expand Down Expand Up @@ -210,9 +233,11 @@ impl VTable for ALPVTable {
);

let mut children_iter = children.into_iter();
array.encoded = children_iter
.next()
.ok_or_else(|| vortex_err!("Expected encoded child"))?;
array.slots[ENCODED_SLOT] = Some(
children_iter
.next()
.ok_or_else(|| vortex_err!("Expected encoded child"))?,
);

if let Some((array_len, offset, _has_chunk_offsets)) = patches_info {
let indices = children_iter
Expand Down Expand Up @@ -260,9 +285,13 @@ impl VTable for ALPVTable {
}
}

pub(super) const ENCODED_SLOT: usize = 0;
pub(super) const NUM_SLOTS: usize = 1;
pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"];

#[derive(Clone, Debug)]
pub struct ALPArray {
encoded: ArrayRef,
slots: Vec<Option<ArrayRef>>,
patches: Option<Patches>,
dtype: DType,
exponents: Exponents,
Expand Down Expand Up @@ -433,7 +462,7 @@ impl ALPArray {

Ok(Self {
dtype,
encoded,
slots: vec![Some(encoded)],
exponents,
patches,
stats_set: Default::default(),
Expand All @@ -452,7 +481,7 @@ impl ALPArray {
) -> Self {
Self {
dtype,
encoded,
slots: vec![Some(encoded)],
exponents,
patches,
stats_set: Default::default(),
Expand All @@ -464,7 +493,9 @@ impl ALPArray {
}

pub fn encoded(&self) -> &ArrayRef {
&self.encoded
self.slots[ENCODED_SLOT]
.as_ref()
.vortex_expect("ALPArray encoded slot")
}

#[inline]
Expand All @@ -479,7 +510,13 @@ impl ALPArray {
/// Consumes the array and returns its parts.
#[inline]
pub fn into_parts(self) -> (ArrayRef, Exponents, Option<Patches>, DType) {
(self.encoded, self.exponents, self.patches, self.dtype)
let encoded = self
.slots
.into_iter()
.next()
.flatten()
.vortex_expect("ALPArray encoded slot");
(encoded, self.exponents, self.patches, self.dtype)
}
}

Expand Down
74 changes: 54 additions & 20 deletions encodings/alp/src/alp_rd/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ use vortex_array::vtable::patches_child;
use vortex_array::vtable::patches_child_name;
use vortex_array::vtable::patches_nchildren;
use vortex_buffer::Buffer;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_error::vortex_ensure;
Expand Down Expand Up @@ -76,7 +77,7 @@ impl VTable for ALPRDVTable {
}

fn len(array: &ALPRDArray) -> usize {
array.left_parts.len()
array.left_parts().len()
}

fn dtype(array: &ALPRDArray) -> &DType {
Expand All @@ -89,20 +90,20 @@ impl VTable for ALPRDVTable {

fn array_hash<H: std::hash::Hasher>(array: &ALPRDArray, state: &mut H, precision: Precision) {
array.dtype.hash(state);
array.left_parts.array_hash(state, precision);
array.left_parts().array_hash(state, precision);
array.left_parts_dictionary.array_hash(state, precision);
array.right_parts.array_hash(state, precision);
array.right_parts().array_hash(state, precision);
array.right_bit_width.hash(state);
array.left_parts_patches.array_hash(state, precision);
}

fn array_eq(array: &ALPRDArray, other: &ALPRDArray, precision: Precision) -> bool {
array.dtype == other.dtype
&& array.left_parts.array_eq(&other.left_parts, precision)
&& array.left_parts().array_eq(other.left_parts(), precision)
&& array
.left_parts_dictionary
.array_eq(&other.left_parts_dictionary, precision)
&& array.right_parts.array_eq(&other.right_parts, precision)
&& array.right_parts().array_eq(other.right_parts(), precision)
&& array.right_bit_width == other.right_bit_width
&& array
.left_parts_patches
Expand Down Expand Up @@ -162,7 +163,7 @@ impl VTable for ALPRDVTable {
right_bit_width: array.right_bit_width() as u32,
dict_len: array.left_parts_dictionary().len() as u32,
dict,
left_parts_ptype: array.left_parts.dtype().as_ptype() as i32,
left_parts_ptype: array.left_parts().dtype().as_ptype() as i32,
patches: array
.left_parts_patches()
.map(|p| p.to_metadata(array.len(), array.left_parts().dtype()))
Expand Down Expand Up @@ -255,6 +256,29 @@ impl VTable for ALPRDVTable {
)
}

fn nslots(_array: &ALPRDArray) -> usize {
NUM_SLOTS
}

fn slot(array: &ALPRDArray, idx: usize) -> &Option<ArrayRef> {
&array.slots[idx]
}

fn slot_name(_array: &ALPRDArray, idx: usize) -> &str {
SLOT_NAMES[idx]
}

fn with_slots(array: &mut ALPRDArray, slots: Vec<Option<ArrayRef>>) -> VortexResult<()> {
vortex_ensure!(
slots.len() == NUM_SLOTS,
"ALPRDArray expects {} slots, got {}",
NUM_SLOTS,
slots.len()
);
array.slots = slots;
Ok(())
}

fn with_children(array: &mut Self::Array, children: Vec<ArrayRef>) -> VortexResult<()> {
// Children: left_parts, right_parts, patches (if present): indices, values
let patches_info = array
Expand All @@ -272,12 +296,16 @@ impl VTable for ALPRDVTable {
);

let mut children_iter = children.into_iter();
array.left_parts = children_iter
.next()
.ok_or_else(|| vortex_err!("Expected left_parts child"))?;
array.right_parts = children_iter
.next()
.ok_or_else(|| vortex_err!("Expected right_parts child"))?;
array.slots[LEFT_PARTS_SLOT] = Some(
children_iter
.next()
.ok_or_else(|| vortex_err!("Expected left_parts child"))?,
);
array.slots[RIGHT_PARTS_SLOT] = Some(
children_iter
.next()
.ok_or_else(|| vortex_err!("Expected right_parts child"))?,
);

if let Some((array_len, offset)) = patches_info {
let indices = children_iter
Expand Down Expand Up @@ -356,13 +384,17 @@ impl VTable for ALPRDVTable {
}
}

pub(super) const LEFT_PARTS_SLOT: usize = 0;
pub(super) const RIGHT_PARTS_SLOT: usize = 1;
pub(super) const NUM_SLOTS: usize = 2;
pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["left_parts", "right_parts"];

#[derive(Clone, Debug)]
pub struct ALPRDArray {
dtype: DType,
left_parts: ArrayRef,
slots: Vec<Option<ArrayRef>>,
left_parts_patches: Option<Patches>,
left_parts_dictionary: Buffer<u16>,
right_parts: ArrayRef,
right_bit_width: u8,
stats_set: ArrayStats,
}
Expand Down Expand Up @@ -431,9 +463,8 @@ impl ALPRDArray {

Ok(Self {
dtype,
left_parts,
slots: vec![Some(left_parts), Some(right_parts)],
left_parts_dictionary,
right_parts,
right_bit_width,
left_parts_patches,
stats_set: Default::default(),
Expand All @@ -452,10 +483,9 @@ impl ALPRDArray {
) -> Self {
Self {
dtype,
left_parts,
slots: vec![Some(left_parts), Some(right_parts)],
left_parts_patches,
left_parts_dictionary,
right_parts,
right_bit_width,
stats_set: Default::default(),
}
Expand All @@ -474,12 +504,16 @@ impl ALPRDArray {
/// These are bit-packed and dictionary encoded, and cannot directly be interpreted without
/// the metadata of this array.
pub fn left_parts(&self) -> &ArrayRef {
&self.left_parts
self.slots[LEFT_PARTS_SLOT]
.as_ref()
.vortex_expect("ALPRDArray left_parts slot")
}

/// The rightmost (least significant) bits of the floating point values stored in the array.
pub fn right_parts(&self) -> &ArrayRef {
&self.right_parts
self.slots[RIGHT_PARTS_SLOT]
.as_ref()
.vortex_expect("ALPRDArray right_parts slot")
}

#[inline]
Expand Down
8 changes: 8 additions & 0 deletions encodings/bytebool/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,14 +122,22 @@ pub fn vortex_bytebool::ByteBoolVTable::nbuffers(_array: &vortex_bytebool::ByteB

pub fn vortex_bytebool::ByteBoolVTable::nchildren(array: &vortex_bytebool::ByteBoolArray) -> usize

pub fn vortex_bytebool::ByteBoolVTable::nslots(_array: &vortex_bytebool::ByteBoolArray) -> usize

pub fn vortex_bytebool::ByteBoolVTable::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>

pub fn vortex_bytebool::ByteBoolVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>

pub fn vortex_bytebool::ByteBoolVTable::slot(_array: &vortex_bytebool::ByteBoolArray, idx: usize) -> &core::option::Option<vortex_array::array::ArrayRef>

pub fn vortex_bytebool::ByteBoolVTable::slot_name(_array: &vortex_bytebool::ByteBoolArray, idx: usize) -> &str

pub fn vortex_bytebool::ByteBoolVTable::stats(array: &vortex_bytebool::ByteBoolArray) -> vortex_array::stats::array::StatsSetRef<'_>

pub fn vortex_bytebool::ByteBoolVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec<vortex_array::array::ArrayRef>) -> vortex_error::VortexResult<()>

pub fn vortex_bytebool::ByteBoolVTable::with_slots(array: &mut vortex_bytebool::ByteBoolArray, slots: alloc::vec::Vec<core::option::Option<vortex_array::array::ArrayRef>>) -> vortex_error::VortexResult<()>

impl vortex_array::vtable::operations::OperationsVTable<vortex_bytebool::ByteBoolVTable> for vortex_bytebool::ByteBoolVTable

pub fn vortex_bytebool::ByteBoolVTable::scalar_at(array: &vortex_bytebool::ByteBoolArray, index: usize) -> vortex_error::VortexResult<vortex_array::scalar::Scalar>
Loading
Loading