WIP: Variant Type

.github/workflows/ci.yml

@@ -217,7 +217,7 @@ jobs:
             target: wasm32-unknown-unknown
             env:
               rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"wasm_js\"'"
-            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest"
+            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant"
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'

Cargo.toml

@@ -55,6 +55,7 @@ members = [
     "benchmarks/duckdb-bench",
     "benchmarks/random-access-bench",
     "vortex-sqllogictest",
+    "encodings/parquet-variant",
 ]
 exclude = ["java/testfiles", "wasm-test"]
 resolver = "2"
@@ -86,16 +87,16 @@ arbitrary = "1.3.2"
 arc-swap = "1.8"
 arcref = "0.2.0"
 arrayref = "0.3.7"
-arrow-arith = "57.1"
-arrow-array = "57.1"
-arrow-buffer = "57.1"
-arrow-cast = "57.1"
-arrow-data = "57.1"
-arrow-ipc = "57.1"
-arrow-ord = "57.1"
-arrow-schema = "57.1"
-arrow-select = "57.1"
-arrow-string = "57.1"
+arrow-arith = "57.2"
+arrow-array = "57.2"
+arrow-buffer = "57.2"
+arrow-cast = "57.2"
+arrow-data = "57.2"
+arrow-ipc = "57.2"
+arrow-ord = "57.2"
+arrow-schema = "57.2"
+arrow-select = "57.2"
+arrow-string = "57.2"
 async-fs = "2.2.0"
 async-lock = "3.4"
 async-stream = "0.3.6"
@@ -178,7 +179,9 @@ opentelemetry = "0.31.0"
 opentelemetry-otlp = "0.31.0"
 opentelemetry_sdk = "0.31.0"
 parking_lot = { version = "0.12.3", features = ["nightly"] }
-parquet = "57.1"
+parquet = "57.2"
+parquet-variant = "57.2"
+parquet-variant-compute = "57.2"
 paste = "1.0.15"
 pco = "1.0.1"
 pin-project-lite = "0.2.15"

encodings/parquet-variant/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "vortex-parquet-variant"
+authors = { workspace = true }
+categories = { workspace = true }
+description = "Vortex Pco array"
+edition = { workspace = true }
+homepage = { workspace = true }
+include = { workspace = true }
+keywords = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+arrow-array = { workspace = true }
+parquet-variant-compute = { workspace = true }
+prost = { workspace = true }
+vortex-array = { workspace = true }
+vortex-buffer = { workspace = true }
+vortex-error = { workspace = true }
+vortex-mask = { workspace = true }
+vortex-proto = { workspace = true }
+vortex-session = { workspace = true }
+
+[dev-dependencies]
+arrow-array = { workspace = true }
+arrow-schema = { workspace = true }
+parquet-variant = { workspace = true }
+rstest = { workspace = true }
+vortex-array = { workspace = true, features = ["_test-harness"] }
+
+[package.metadata.cargo-machete]
+ignored = ["getrandom_v03"]

encodings/parquet-variant/public-api.lock

@@ -0,0 +1,121 @@
+pub mod vortex_parquet_variant
+
+pub struct vortex_parquet_variant::ParquetVariantArray
+
+impl vortex_parquet_variant::ParquetVariantArray
+
+pub fn vortex_parquet_variant::ParquetVariantArray::from_arrow_variant(arrow_variant: &parquet_variant_compute::variant_array::VariantArray) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
+
+pub fn vortex_parquet_variant::ParquetVariantArray::metadata_array(&self) -> &vortex_array::array::ArrayRef
+
+pub fn vortex_parquet_variant::ParquetVariantArray::try_new(metadata: vortex_array::array::ArrayRef, value: core::option::Option<vortex_array::array::ArrayRef>, typed_value: core::option::Option<vortex_array::array::ArrayRef>) -> vortex_error::VortexResult<Self>
+
+pub fn vortex_parquet_variant::ParquetVariantArray::typed_value_array(&self) -> core::option::Option<&vortex_array::array::ArrayRef>
+
+pub fn vortex_parquet_variant::ParquetVariantArray::value_array(&self) -> core::option::Option<&vortex_array::array::ArrayRef>
+
+impl vortex_parquet_variant::ParquetVariantArray
+
+pub fn vortex_parquet_variant::ParquetVariantArray::to_array(&self) -> vortex_array::array::ArrayRef
+
+impl core::clone::Clone for vortex_parquet_variant::ParquetVariantArray
+
+pub fn vortex_parquet_variant::ParquetVariantArray::clone(&self) -> vortex_parquet_variant::ParquetVariantArray
+
+impl core::convert::AsRef<dyn vortex_array::array::DynArray> for vortex_parquet_variant::ParquetVariantArray
+
+pub fn vortex_parquet_variant::ParquetVariantArray::as_ref(&self) -> &dyn vortex_array::array::DynArray
+
+impl core::convert::From<vortex_parquet_variant::ParquetVariantArray> for vortex_array::array::ArrayRef
+
+pub fn vortex_array::array::ArrayRef::from(value: vortex_parquet_variant::ParquetVariantArray) -> vortex_array::array::ArrayRef
+
+impl core::fmt::Debug for vortex_parquet_variant::ParquetVariantArray
+
+pub fn vortex_parquet_variant::ParquetVariantArray::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+impl core::ops::deref::Deref for vortex_parquet_variant::ParquetVariantArray
+
+pub type vortex_parquet_variant::ParquetVariantArray::Target = dyn vortex_array::array::DynArray
+
+pub fn vortex_parquet_variant::ParquetVariantArray::deref(&self) -> &Self::Target
+
+impl vortex_array::array::IntoArray for vortex_parquet_variant::ParquetVariantArray
+
+pub fn vortex_parquet_variant::ParquetVariantArray::into_array(self) -> vortex_array::array::ArrayRef
+
+pub struct vortex_parquet_variant::ParquetVariantMetadata
+
+pub vortex_parquet_variant::ParquetVariantMetadata::has_value: bool
+
+pub vortex_parquet_variant::ParquetVariantMetadata::typed_value_dtype: core::option::Option<vortex_array::dtype::DType>
+
+impl core::clone::Clone for vortex_parquet_variant::ParquetVariantMetadata
+
+pub fn vortex_parquet_variant::ParquetVariantMetadata::clone(&self) -> vortex_parquet_variant::ParquetVariantMetadata
+
+impl core::fmt::Debug for vortex_parquet_variant::ParquetVariantMetadata
+
+pub fn vortex_parquet_variant::ParquetVariantMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+pub struct vortex_parquet_variant::ParquetVariantVTable
+
+impl vortex_parquet_variant::ParquetVariantVTable
+
+pub const vortex_parquet_variant::ParquetVariantVTable::ID: vortex_array::vtable::dyn_::ArrayId
+
+impl core::fmt::Debug for vortex_parquet_variant::ParquetVariantVTable
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+impl vortex_array::vtable::VTable for vortex_parquet_variant::ParquetVariantVTable
+
+pub type vortex_parquet_variant::ParquetVariantVTable::Array = vortex_parquet_variant::ParquetVariantArray
+
+pub type vortex_parquet_variant::ParquetVariantVTable::Metadata = vortex_parquet_variant::ParquetVariantMetadata
+
+pub type vortex_parquet_variant::ParquetVariantVTable::OperationsVTable = vortex_array::vtable::NotSupported
+
+pub type vortex_parquet_variant::ParquetVariantVTable::ValidityVTable = vortex_parquet_variant::ParquetVariantVTable
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::array_eq(array: &vortex_parquet_variant::ParquetVariantArray, other: &vortex_parquet_variant::ParquetVariantArray, precision: vortex_array::hash::Precision) -> bool
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::array_hash<H: core::hash::Hasher>(array: &vortex_parquet_variant::ParquetVariantArray, state: &mut H, precision: vortex_array::hash::Precision)
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::buffer(_array: &vortex_parquet_variant::ParquetVariantArray, idx: usize) -> vortex_array::buffer::BufferHandle
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::buffer_name(_array: &vortex_parquet_variant::ParquetVariantArray, _idx: usize) -> core::option::Option<alloc::string::String>
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_parquet_variant::ParquetVariantArray>
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::child(array: &vortex_parquet_variant::ParquetVariantArray, idx: usize) -> vortex_array::array::ArrayRef
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::child_name(array: &vortex_parquet_variant::ParquetVariantArray, idx: usize) -> alloc::string::String
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::dtype(_array: &vortex_parquet_variant::ParquetVariantArray) -> &vortex_array::dtype::DType
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::len(array: &vortex_parquet_variant::ParquetVariantArray) -> usize
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::metadata(array: &vortex_parquet_variant::ParquetVariantArray) -> vortex_error::VortexResult<Self::Metadata>
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::nbuffers(_array: &vortex_parquet_variant::ParquetVariantArray) -> usize
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::nchildren(array: &vortex_parquet_variant::ParquetVariantArray) -> usize
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::stats(array: &vortex_parquet_variant::ParquetVariantArray) -> vortex_array::stats::array::StatsSetRef<'_>
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec<vortex_array::array::ArrayRef>) -> vortex_error::VortexResult<()>
+
+impl vortex_array::vtable::validity::ValidityVTable<vortex_parquet_variant::ParquetVariantVTable> for vortex_parquet_variant::ParquetVariantVTable
+
+pub fn vortex_parquet_variant::ParquetVariantVTable::validity(_array: &vortex_parquet_variant::ParquetVariantArray) -> vortex_error::VortexResult<vortex_array::validity::Validity>