Skip to content

Interned Identifiers #7068

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
gatesn wants to merge 8 commits into
base: develop
Choose a base branch
from
Draft

Interned Identifiers #7068

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
35e27b7
interned identifiers
gatesn Mar 19, 2026
921b934
Merge branch 'develop' into ngates/id
gatesn Mar 19, 2026
fbd5bec
interned identifiers
gatesn Mar 19, 2026
649cd3e
Interned Identifiers
gatesn Mar 19, 2026
6275b76
Interned Identifiers
gatesn Mar 19, 2026
d952863
Interned Identifiers
gatesn Mar 19, 2026
30ef9c5
Interned Identifiers
gatesn Mar 19, 2026
2232ae5
Interned Identifiers
gatesn Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ similar = "2.7.0"
sketches-ddsketch = "0.4.0"
smol = "2.0.2"
static_assertions = "1.1"
string-interner = "0.19.0"
strum = "0.28"
syn = { version = "2.0.113", features = ["full"] }
sysinfo = "0.38.0"
Expand Down
8 changes: 6 additions & 2 deletions encodings/alp/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ pub struct vortex_alp::ALP

impl vortex_alp::ALP

pub const vortex_alp::ALP::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_alp::ALP::ID: &'static str

pub fn vortex_alp::ALP::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_alp::ALP

Expand Down Expand Up @@ -174,7 +176,9 @@ pub struct vortex_alp::ALPRD

impl vortex_alp::ALPRD

pub const vortex_alp::ALPRD::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_alp::ALPRD::ID: &'static str

pub fn vortex_alp::ALPRD::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_alp::ALPRD

Expand Down
10 changes: 8 additions & 2 deletions encodings/alp/src/alp/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ impl VTable for ALP {
type ValidityVTable = ValidityVTableFromChild;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &ALPArray) -> usize {
Expand Down Expand Up @@ -273,7 +273,13 @@ pub struct ALPArray {
pub struct ALP;

impl ALP {
pub const ID: ArrayId = ArrayId::new_ref("vortex.alp");
pub const ID: &'static str = "vortex.alp";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
Comment on lines +276 to +282
Copy link
Contributor

@joseph-isaacs joseph-isaacs Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this not have an atomic? On read

Copy link
Contributor Author

@gatesn gatesn Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could force vtables to hold onto an instance of their ID by having VTable::id(&Array) -> &ArrayId?

But it means no more ZSTs for vtables

}

#[derive(Clone, prost::Message)]
Expand Down
10 changes: 8 additions & 2 deletions encodings/alp/src/alp_rd/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ impl VTable for ALPRD {
type ValidityVTable = ValidityVTableFromChild;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &ALPRDArray) -> usize {
Expand Down Expand Up @@ -371,7 +371,13 @@ pub struct ALPRDArray {
pub struct ALPRD;

impl ALPRD {
pub const ID: ArrayId = ArrayId::new_ref("vortex.alprd");
pub const ID: &'static str = "vortex.alprd";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}

impl ALPRDArray {
Expand Down
4 changes: 3 additions & 1 deletion encodings/bytebool/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ pub struct vortex_bytebool::ByteBool

impl vortex_bytebool::ByteBool

pub const vortex_bytebool::ByteBool::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_bytebool::ByteBool::ID: &'static str

pub fn vortex_bytebool::ByteBool::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_bytebool::ByteBool

Expand Down
10 changes: 8 additions & 2 deletions encodings/bytebool/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ impl VTable for ByteBool {
type ValidityVTable = ValidityVTableFromValidityHelper;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &ByteBoolArray) -> usize {
Expand Down Expand Up @@ -213,7 +213,13 @@ pub struct ByteBoolArray {
pub struct ByteBool;

impl ByteBool {
pub const ID: ArrayId = ArrayId::new_ref("vortex.bytebool");
pub const ID: &'static str = "vortex.bytebool";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}

impl ByteBoolArray {
Expand Down
4 changes: 3 additions & 1 deletion encodings/datetime-parts/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ pub struct vortex_datetime_parts::DateTimeParts

impl vortex_datetime_parts::DateTimeParts

pub const vortex_datetime_parts::DateTimeParts::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_datetime_parts::DateTimeParts::ID: &'static str

pub fn vortex_datetime_parts::DateTimeParts::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_datetime_parts::DateTimeParts

Expand Down
10 changes: 8 additions & 2 deletions encodings/datetime-parts/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ impl VTable for DateTimeParts {
type ValidityVTable = ValidityVTableFromChild;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &DateTimePartsArray) -> usize {
Expand Down Expand Up @@ -267,7 +267,13 @@ pub struct DateTimePartsArrayParts {
pub struct DateTimeParts;

impl DateTimeParts {
pub const ID: ArrayId = ArrayId::new_ref("vortex.datetimeparts");
pub const ID: &'static str = "vortex.datetimeparts";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}

impl DateTimePartsArray {
Expand Down
4 changes: 3 additions & 1 deletion encodings/decimal-byte-parts/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ pub struct vortex_decimal_byte_parts::DecimalByteParts

impl vortex_decimal_byte_parts::DecimalByteParts

pub const vortex_decimal_byte_parts::DecimalByteParts::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_decimal_byte_parts::DecimalByteParts::ID: &'static str

pub fn vortex_decimal_byte_parts::DecimalByteParts::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_decimal_byte_parts::DecimalByteParts

Expand Down
10 changes: 8 additions & 2 deletions encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ impl VTable for DecimalByteParts {
type ValidityVTable = ValidityVTableFromChild;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &DecimalBytePartsArray) -> usize {
Expand Down Expand Up @@ -274,7 +274,13 @@ impl DecimalBytePartsArray {
pub struct DecimalByteParts;

impl DecimalByteParts {
pub const ID: ArrayId = ArrayId::new_ref("vortex.decimal_byte_parts");
pub const ID: &'static str = "vortex.decimal_byte_parts";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}

/// Converts a DecimalBytePartsArray to its canonical DecimalArray representation.
Expand Down
16 changes: 12 additions & 4 deletions encodings/fastlanes/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,9 @@ pub struct vortex_fastlanes::BitPacked

impl vortex_fastlanes::BitPacked

pub const vortex_fastlanes::BitPacked::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_fastlanes::BitPacked::ID: &'static str

pub fn vortex_fastlanes::BitPacked::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_fastlanes::BitPacked

Expand Down Expand Up @@ -274,7 +276,9 @@ pub struct vortex_fastlanes::Delta

impl vortex_fastlanes::Delta

pub const vortex_fastlanes::Delta::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_fastlanes::Delta::ID: &'static str

pub fn vortex_fastlanes::Delta::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_fastlanes::Delta

Expand Down Expand Up @@ -402,7 +406,9 @@ pub struct vortex_fastlanes::FoR

impl vortex_fastlanes::FoR

pub const vortex_fastlanes::FoR::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_fastlanes::FoR::ID: &'static str

pub fn vortex_fastlanes::FoR::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_fastlanes::FoR

Expand Down Expand Up @@ -536,7 +542,9 @@ pub struct vortex_fastlanes::RLE

impl vortex_fastlanes::RLE

pub const vortex_fastlanes::RLE::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_fastlanes::RLE::ID: &'static str

pub fn vortex_fastlanes::RLE::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_fastlanes::RLE

Expand Down
10 changes: 8 additions & 2 deletions encodings/fastlanes/src/bitpacking/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ impl VTable for BitPacked {
type ValidityVTable = ValidityVTableFromValidityHelper;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &BitPackedArray) -> usize {
Expand Down Expand Up @@ -372,5 +372,11 @@ impl VTable for BitPacked {
pub struct BitPacked;

impl BitPacked {
pub const ID: ArrayId = ArrayId::new_ref("fastlanes.bitpacked");
pub const ID: &'static str = "fastlanes.bitpacked";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}
10 changes: 8 additions & 2 deletions encodings/fastlanes/src/delta/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ impl VTable for Delta {
type ValidityVTable = ValidityVTableFromChildSliceHelper;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &DeltaArray) -> usize {
Expand Down Expand Up @@ -201,7 +201,13 @@ impl VTable for Delta {
pub struct Delta;

impl Delta {
pub const ID: ArrayId = ArrayId::new_ref("fastlanes.delta");
pub const ID: &'static str = "fastlanes.delta";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}

#[cfg(test)]
Expand Down
10 changes: 8 additions & 2 deletions encodings/fastlanes/src/for/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ impl VTable for FoR {
type ValidityVTable = ValidityVTableFromChild;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &FoRArray) -> usize {
Expand Down Expand Up @@ -184,5 +184,11 @@ impl VTable for FoR {
pub struct FoR;

impl FoR {
pub const ID: ArrayId = ArrayId::new_ref("fastlanes.for");
pub const ID: &'static str = "fastlanes.for";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}
10 changes: 8 additions & 2 deletions encodings/fastlanes/src/rle/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ impl VTable for RLE {
type ValidityVTable = ValidityVTableFromChildSliceHelper;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &RLEArray) -> usize {
Expand Down Expand Up @@ -242,7 +242,13 @@ impl VTable for RLE {
pub struct RLE;

impl RLE {
pub const ID: ArrayId = ArrayId::new_ref("fastlanes.rle");
pub const ID: &'static str = "fastlanes.rle";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}

#[cfg(test)]
Expand Down
4 changes: 3 additions & 1 deletion encodings/fsst/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ pub struct vortex_fsst::FSST

impl vortex_fsst::FSST

pub const vortex_fsst::FSST::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_fsst::FSST::ID: &'static str

pub fn vortex_fsst::FSST::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_fsst::FSST

Expand Down
10 changes: 8 additions & 2 deletions encodings/fsst/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ impl VTable for FSST {
type ValidityVTable = ValidityVTableFromChild;

fn id(_array: &Self::Array) -> ArrayId {
Self::ID
Self::array_id()
}

fn len(array: &FSSTArray) -> usize {
Expand Down Expand Up @@ -393,7 +393,13 @@ impl Debug for FSSTArray {
pub struct FSST;

impl FSST {
pub const ID: ArrayId = ArrayId::new_ref("vortex.fsst");
pub const ID: &'static str = "vortex.fsst";

/// Returns the cached [`ArrayId`] for this encoding.
pub fn array_id() -> ArrayId {
static CACHED: std::sync::OnceLock<ArrayId> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| ArrayId::new(Self::ID))
}
}

impl FSSTArray {
Expand Down
4 changes: 3 additions & 1 deletion encodings/pco/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ pub struct vortex_pco::Pco

impl vortex_pco::Pco

pub const vortex_pco::Pco::ID: vortex_array::vtable::dyn_::ArrayId
pub const vortex_pco::Pco::ID: &'static str

pub fn vortex_pco::Pco::array_id() -> vortex_array::vtable::dyn_::ArrayId

impl core::fmt::Debug for vortex_pco::Pco

Expand Down
Loading
Loading