Skip to content

Commit 458273c

Browse files
jiaqizhojiaqizho
committed
Add FlatLayout range read for sub-segment IO
When a FlatLayout has its array_tree metadata inlined in the footer, we can figure out exactly which bytes of the segment are needed for a given row range without any IO. This lets us issue a single small read instead of fetching the entire segment, which is a big win for point lookups and narrow scans on wide tables. The range read planner walks the encoding tree via `VTable::plan_range_read`, where each encoding (Primitive, Bool, BitPacked, Delta, FoR, ZigZag, ALP, ALPRD, Dict, FixedSizeList, Constant, Null, Sequence, ByteBool, DateTimeParts, DecimalByteParts) declares its own buffer sub-ranges and child recursion strategy. If the resulting byte range is less than 50% of the full segment, we issue the targeted read; otherwise we fall back to reading the whole segment. To make Delta work with sub-ranged buffers, Delta::build() now derives child array lengths from `len + offset` instead of metadata.deltas_len. On disk, offset is always 0 so this is a no-op for the normal decode path, but it lets the range read pass a smaller decode_len without the decoder panicking on buffer size mismatch. Also adds `request_range()` to the SegmentSource trait with a default fallback implementation, efficient overrides in FileSegmentSource and BufferSegmentSource, a `RangeReadEnabled` session flag, and `ScanBuilder::with_split_row_indices` to generate per-index tight ranges for point lookups. Signed-off-by: jiaqizho <jiaqi.zhou@zilliz.com>
1 parent 2a0d1b7 commit 458273c

41 files changed

Lines changed: 2171 additions & 24 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.lock‎

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎encodings/alp/public-api.lock‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ pub fn vortex_alp::ALP::nbuffers(_array: &vortex_alp::ALPArray) -> usize
9090

9191
pub fn vortex_alp::ALP::nchildren(array: &vortex_alp::ALPArray) -> usize
9292

93+
pub fn vortex_alp::ALP::plan_range_read(metadata: &vortex_array::metadata::ProstMetadata<vortex_alp::ALPMetadata>, row_range: core::ops::range::Range<usize>, row_count: usize, dtype: &vortex_array::dtype::DType) -> core::option::Option<vortex_array::vtable::range_read::EncodingRangeRead>
94+
9395
pub fn vortex_alp::ALP::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
9496

9597
pub fn vortex_alp::ALP::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
@@ -246,6 +248,8 @@ pub fn vortex_alp::ALPRD::nbuffers(_array: &vortex_alp::ALPRDArray) -> usize
246248

247249
pub fn vortex_alp::ALPRD::nchildren(array: &vortex_alp::ALPRDArray) -> usize
248250

251+
pub fn vortex_alp::ALPRD::plan_range_read(metadata: &vortex_array::metadata::ProstMetadata<vortex_alp::ALPRDMetadata>, row_range: core::ops::range::Range<usize>, row_count: usize, dtype: &vortex_array::dtype::DType) -> core::option::Option<vortex_array::vtable::range_read::EncodingRangeRead>
252+
249253
pub fn vortex_alp::ALPRD::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
250254

251255
pub fn vortex_alp::ALPRD::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>

‎encodings/alp/src/alp/array.rs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::fmt::Debug;
55
use std::hash::Hash;
6+
use std::ops::Range;
67

78
use vortex_array::ArrayEq;
89
use vortex_array::ArrayHash;
@@ -25,6 +26,9 @@ use vortex_array::stats::ArrayStats;
2526
use vortex_array::stats::StatsSetRef;
2627
use vortex_array::vtable;
2728
use vortex_array::vtable::ArrayId;
29+
use vortex_array::vtable::ChildRangeRead;
30+
use vortex_array::vtable::EncodingRangeRead;
31+
use vortex_array::vtable::RangeDecodeInfo;
2832
use vortex_array::vtable::VTable;
2933
use vortex_array::vtable::ValidityChild;
3034
use vortex_array::vtable::ValidityVTableFromChild;
@@ -258,6 +262,38 @@ impl VTable for ALP {
258262
) -> VortexResult<Option<ArrayRef>> {
259263
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
260264
}
265+
266+
fn plan_range_read(
267+
metadata: &ProstMetadata<ALPMetadata>,
268+
row_range: Range<usize>,
269+
row_count: usize,
270+
dtype: &DType,
271+
) -> Option<EncodingRangeRead> {
272+
// Patches cannot be safely sub-ranged (global indices).
273+
if metadata.0.patches.is_some() {
274+
return None;
275+
}
276+
277+
// Child 0 = encoded values (f32→i32 or f64→i64).
278+
let child_dtype = match dtype {
279+
DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n),
280+
DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n),
281+
_ => return None,
282+
};
283+
284+
Some(EncodingRangeRead {
285+
buffer_sub_ranges: vec![],
286+
children: vec![ChildRangeRead::Recurse {
287+
row_range,
288+
row_count,
289+
dtype: child_dtype,
290+
}],
291+
decode_info: RangeDecodeInfo::FromChild {
292+
child_idx: 0,
293+
divisor: 1,
294+
},
295+
})
296+
}
261297
}
262298

263299
#[derive(Clone, Debug)]

‎encodings/alp/src/alp_rd/array.rs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::fmt::Debug;
55
use std::hash::Hash;
6+
use std::ops::Range;
67

78
use itertools::Itertools;
89
use vortex_array::ArrayEq;
@@ -29,6 +30,9 @@ use vortex_array::stats::StatsSetRef;
2930
use vortex_array::validity::Validity;
3031
use vortex_array::vtable;
3132
use vortex_array::vtable::ArrayId;
33+
use vortex_array::vtable::ChildRangeRead;
34+
use vortex_array::vtable::EncodingRangeRead;
35+
use vortex_array::vtable::RangeDecodeInfo;
3236
use vortex_array::vtable::VTable;
3337
use vortex_array::vtable::ValidityChild;
3438
use vortex_array::vtable::ValidityVTableFromChild;
@@ -354,6 +358,53 @@ impl VTable for ALPRD {
354358
) -> VortexResult<Option<ArrayRef>> {
355359
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
356360
}
361+
362+
fn plan_range_read(
363+
metadata: &ProstMetadata<ALPRDMetadata>,
364+
row_range: Range<usize>,
365+
row_count: usize,
366+
dtype: &DType,
367+
) -> Option<EncodingRangeRead> {
368+
// Patches cannot be safely sub-ranged (global indices).
369+
if metadata.0.patches.is_some() {
370+
return None;
371+
}
372+
373+
let left_parts_ptype = PType::try_from(metadata.0.left_parts_ptype).ok()?;
374+
let left_parts_dtype = DType::Primitive(left_parts_ptype, dtype.nullability());
375+
376+
let right_parts_dtype = match dtype {
377+
DType::Primitive(PType::F32, _) => {
378+
DType::Primitive(PType::U32, Nullability::NonNullable)
379+
}
380+
DType::Primitive(PType::F64, _) => {
381+
DType::Primitive(PType::U64, Nullability::NonNullable)
382+
}
383+
_ => return None,
384+
};
385+
386+
Some(EncodingRangeRead {
387+
buffer_sub_ranges: vec![],
388+
children: vec![
389+
// Child 0 = left_parts.
390+
ChildRangeRead::Recurse {
391+
row_range: row_range.clone(),
392+
row_count,
393+
dtype: left_parts_dtype,
394+
},
395+
// Child 1 = right_parts.
396+
ChildRangeRead::Recurse {
397+
row_range,
398+
row_count,
399+
dtype: right_parts_dtype,
400+
},
401+
],
402+
decode_info: RangeDecodeInfo::FromChild {
403+
child_idx: 0,
404+
divisor: 1,
405+
},
406+
})
407+
}
357408
}
358409

359410
#[derive(Clone, Debug)]

‎encodings/bytebool/public-api.lock‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ pub fn vortex_bytebool::ByteBool::nbuffers(_array: &vortex_bytebool::ByteBoolArr
6868

6969
pub fn vortex_bytebool::ByteBool::nchildren(array: &vortex_bytebool::ByteBoolArray) -> usize
7070

71+
pub fn vortex_bytebool::ByteBool::plan_range_read(_metadata: &vortex_array::metadata::EmptyMetadata, row_range: core::ops::range::Range<usize>, _row_count: usize, _dtype: &vortex_array::dtype::DType) -> core::option::Option<vortex_array::vtable::range_read::EncodingRangeRead>
72+
7173
pub fn vortex_bytebool::ByteBool::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
7274

7375
pub fn vortex_bytebool::ByteBool::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>

‎encodings/bytebool/src/array.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::fmt::Debug;
55
use std::hash::Hash;
6+
use std::ops::Range;
67

78
use vortex_array::ArrayEq;
89
use vortex_array::ArrayHash;
@@ -22,7 +23,10 @@ use vortex_array::stats::StatsSetRef;
2223
use vortex_array::validity::Validity;
2324
use vortex_array::vtable;
2425
use vortex_array::vtable::ArrayId;
26+
use vortex_array::vtable::BufferSubRange;
27+
use vortex_array::vtable::EncodingRangeRead;
2528
use vortex_array::vtable::OperationsVTable;
29+
use vortex_array::vtable::RangeDecodeInfo;
2630
use vortex_array::vtable::VTable;
2731
use vortex_array::vtable::ValidityHelper;
2832
use vortex_array::vtable::ValidityVTableFromValidityHelper;
@@ -199,6 +203,23 @@ impl VTable for ByteBool {
199203
) -> VortexResult<Option<ArrayRef>> {
200204
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
201205
}
206+
207+
fn plan_range_read(
208+
_metadata: &EmptyMetadata,
209+
row_range: Range<usize>,
210+
_row_count: usize,
211+
_dtype: &DType,
212+
) -> Option<EncodingRangeRead> {
213+
// 1 byte per boolean value
214+
Some(EncodingRangeRead {
215+
buffer_sub_ranges: vec![BufferSubRange::Range(row_range.start..row_range.end)],
216+
children: vec![],
217+
decode_info: RangeDecodeInfo::Leaf {
218+
decode_len: row_range.len(),
219+
post_slice: None,
220+
},
221+
})
222+
}
202223
}
203224

204225
#[derive(Clone, Debug)]

‎encodings/datetime-parts/public-api.lock‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ pub fn vortex_datetime_parts::DateTimeParts::nbuffers(_array: &vortex_datetime_p
8080

8181
pub fn vortex_datetime_parts::DateTimeParts::nchildren(_array: &vortex_datetime_parts::DateTimePartsArray) -> usize
8282

83+
pub fn vortex_datetime_parts::DateTimeParts::plan_range_read(metadata: &vortex_array::metadata::ProstMetadata<vortex_datetime_parts::DateTimePartsMetadata>, row_range: core::ops::range::Range<usize>, row_count: usize, dtype: &vortex_array::dtype::DType) -> core::option::Option<vortex_array::vtable::range_read::EncodingRangeRead>
84+
8385
pub fn vortex_datetime_parts::DateTimeParts::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
8486

8587
pub fn vortex_datetime_parts::DateTimeParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>

‎encodings/datetime-parts/src/array.rs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::fmt::Debug;
55
use std::hash::Hash;
6+
use std::ops::Range;
67

78
use vortex_array::ArrayEq;
89
use vortex_array::ArrayHash;
@@ -24,6 +25,9 @@ use vortex_array::stats::ArrayStats;
2425
use vortex_array::stats::StatsSetRef;
2526
use vortex_array::vtable;
2627
use vortex_array::vtable::ArrayId;
28+
use vortex_array::vtable::ChildRangeRead;
29+
use vortex_array::vtable::EncodingRangeRead;
30+
use vortex_array::vtable::RangeDecodeInfo;
2731
use vortex_array::vtable::VTable;
2832
use vortex_array::vtable::ValidityChild;
2933
use vortex_array::vtable::ValidityVTableFromChild;
@@ -244,6 +248,45 @@ impl VTable for DateTimeParts {
244248
) -> VortexResult<Option<ArrayRef>> {
245249
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
246250
}
251+
252+
fn plan_range_read(
253+
metadata: &ProstMetadata<DateTimePartsMetadata>,
254+
row_range: Range<usize>,
255+
row_count: usize,
256+
dtype: &DType,
257+
) -> Option<EncodingRangeRead> {
258+
let days_ptype = metadata.0.get_days_ptype().ok()?;
259+
let seconds_ptype = metadata.0.get_seconds_ptype().ok()?;
260+
let subseconds_ptype = metadata.0.get_subseconds_ptype().ok()?;
261+
262+
Some(EncodingRangeRead {
263+
buffer_sub_ranges: vec![],
264+
children: vec![
265+
// Child 0 = days (carries validity from parent dtype).
266+
ChildRangeRead::Recurse {
267+
row_range: row_range.clone(),
268+
row_count,
269+
dtype: DType::Primitive(days_ptype, dtype.nullability()),
270+
},
271+
// Child 1 = seconds (always non-nullable).
272+
ChildRangeRead::Recurse {
273+
row_range: row_range.clone(),
274+
row_count,
275+
dtype: DType::Primitive(seconds_ptype, Nullability::NonNullable),
276+
},
277+
// Child 2 = subseconds (always non-nullable).
278+
ChildRangeRead::Recurse {
279+
row_range,
280+
row_count,
281+
dtype: DType::Primitive(subseconds_ptype, Nullability::NonNullable),
282+
},
283+
],
284+
decode_info: RangeDecodeInfo::FromChild {
285+
child_idx: 0,
286+
divisor: 1,
287+
},
288+
})
289+
}
247290
}
248291

249292
#[derive(Clone, Debug)]

‎encodings/decimal-byte-parts/public-api.lock‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::nbuffers(_array: &vortex_dec
8080

8181
pub fn vortex_decimal_byte_parts::DecimalByteParts::nchildren(_array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> usize
8282

83+
pub fn vortex_decimal_byte_parts::DecimalByteParts::plan_range_read(metadata: &vortex_array::metadata::ProstMetadata<vortex_decimal_byte_parts::DecimalBytesPartsMetadata>, row_range: core::ops::range::Range<usize>, row_count: usize, dtype: &vortex_array::dtype::DType) -> core::option::Option<vortex_array::vtable::range_read::EncodingRangeRead>
84+
8385
pub fn vortex_decimal_byte_parts::DecimalByteParts::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
8486

8587
pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>

‎encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ mod rules;
66
mod slice;
77

88
use std::hash::Hash;
9+
use std::ops::Range;
910

1011
use prost::Message as _;
1112
use vortex_array::ArrayEq;
@@ -33,7 +34,10 @@ use vortex_array::stats::ArrayStats;
3334
use vortex_array::stats::StatsSetRef;
3435
use vortex_array::vtable;
3536
use vortex_array::vtable::ArrayId;
37+
use vortex_array::vtable::ChildRangeRead;
38+
use vortex_array::vtable::EncodingRangeRead;
3639
use vortex_array::vtable::OperationsVTable;
40+
use vortex_array::vtable::RangeDecodeInfo;
3741
use vortex_array::vtable::VTable;
3842
use vortex_array::vtable::ValidityChild;
3943
use vortex_array::vtable::ValidityHelper;
@@ -202,6 +206,27 @@ impl VTable for DecimalByteParts {
202206
) -> VortexResult<Option<ArrayRef>> {
203207
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
204208
}
209+
210+
fn plan_range_read(
211+
metadata: &ProstMetadata<DecimalBytesPartsMetadata>,
212+
row_range: Range<usize>,
213+
row_count: usize,
214+
dtype: &DType,
215+
) -> Option<EncodingRangeRead> {
216+
let child_dtype = DType::Primitive(metadata.zeroth_child_ptype(), dtype.nullability());
217+
Some(EncodingRangeRead {
218+
buffer_sub_ranges: vec![],
219+
children: vec![ChildRangeRead::Recurse {
220+
row_range,
221+
row_count,
222+
dtype: child_dtype,
223+
}],
224+
decode_info: RangeDecodeInfo::FromChild {
225+
child_idx: 0,
226+
divisor: 1,
227+
},
228+
})
229+
}
205230
}
206231

207232
/// This array encodes decimals as between 1-4 columns of primitive typed children.

0 commit comments

Comments
 (0)