Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions datafusion/common/src/scalar/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3675,6 +3675,22 @@ impl ScalarValue {
.with_field(field)
.build_fixed_size_list_scalar(list_size)
}
DataType::ListView(field) => {
let list_array = array.as_list_view::<i32>();
let nested_array = list_array.value(index);
// Store as List scalar since ScalarValue has no ListView variant.
SingleRowListArrayBuilder::new(nested_array)
.with_field(field)
.build_list_scalar()
}
DataType::LargeListView(field) => {
let list_array = array.as_list_view::<i64>();
let nested_array = list_array.value(index);
// Store as LargeList scalar since ScalarValue has no LargeListView variant.
SingleRowListArrayBuilder::new(nested_array)
.with_field(field)
.build_large_list_scalar()
}
DataType::Date32 => typed_cast!(array, index, as_date32_array, Date32)?,
DataType::Date64 => typed_cast!(array, index, as_date64_array, Date64)?,
DataType::Time32(TimeUnit::Second) => {
Expand Down
4 changes: 3 additions & 1 deletion datafusion/expr/src/expr_schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,9 @@ impl ExprSchemable for Expr {
match arg_data_type {
DataType::List(field)
| DataType::LargeList(field)
| DataType::FixedSizeList(field, _) => Ok(field.data_type().clone()),
| DataType::FixedSizeList(field, _)
| DataType::ListView(field)
| DataType::LargeListView(field) => Ok(field.data_type().clone()),
DataType::Struct(_) => Ok(arg_data_type),
DataType::Null => {
not_impl_err!("unnest() does not support null yet")
Expand Down
14 changes: 11 additions & 3 deletions datafusion/expr/src/logical_plan/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4194,7 +4194,9 @@ impl Unnest {
}
DataType::List(_)
| DataType::FixedSizeList(_, _)
| DataType::LargeList(_) => {
| DataType::LargeList(_)
| DataType::ListView(_)
| DataType::LargeListView(_) => {
list_columns.push((
index,
ColumnUnnestList {
Expand Down Expand Up @@ -4269,7 +4271,11 @@ fn get_unnested_columns(
let mut qualified_columns = Vec::with_capacity(1);

match data_type {
DataType::List(_) | DataType::FixedSizeList(_, _) | DataType::LargeList(_) => {
DataType::List(_)
| DataType::FixedSizeList(_, _)
| DataType::LargeList(_)
| DataType::ListView(_)
| DataType::LargeListView(_) => {
let data_type = get_unnested_list_datatype_recursive(data_type, depth)?;
let new_field = Arc::new(Field::new(
col_name, data_type,
Expand Down Expand Up @@ -4306,7 +4312,9 @@ fn get_unnested_list_datatype_recursive(
match data_type {
DataType::List(field)
| DataType::FixedSizeList(field, _)
| DataType::LargeList(field) => {
| DataType::LargeList(field)
| DataType::ListView(field)
| DataType::LargeListView(field) => {
if depth == 1 {
return Ok(field.data_type().clone());
}
Expand Down
33 changes: 32 additions & 1 deletion datafusion/physical-plan/src/unnest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ use crate::{

use arrow::array::{
Array, ArrayRef, AsArray, BooleanBufferBuilder, FixedSizeListArray, Int64Array,
LargeListArray, ListArray, PrimitiveArray, Scalar, StructArray, new_null_array,
LargeListArray, LargeListViewArray, ListArray, ListViewArray, PrimitiveArray, Scalar,
StructArray, new_null_array,
};
use arrow::compute::kernels::length::length;
use arrow::compute::kernels::zip::zip;
Expand Down Expand Up @@ -845,6 +846,30 @@ impl ListArrayType for FixedSizeListArray {
}
}

impl ListArrayType for ListViewArray {
fn values(&self) -> &ArrayRef {
self.values()
}

fn value_offsets(&self, row: usize) -> (i64, i64) {
let offset = self.value_offsets()[row] as i64;
let size = self.value_sizes()[row] as i64;
(offset, offset + size)
}
}

impl ListArrayType for LargeListViewArray {
fn values(&self) -> &ArrayRef {
self.values()
}

fn value_offsets(&self, row: usize) -> (i64, i64) {
let offset = self.value_offsets()[row];
let size = self.value_sizes()[row];
(offset, offset + size)
}
}

/// Unnest multiple list arrays according to the length array.
fn unnest_list_arrays(
list_arrays: &[ArrayRef],
Expand All @@ -861,6 +886,12 @@ fn unnest_list_arrays(
DataType::FixedSizeList(_, _) => {
Ok(list_array.as_fixed_size_list() as &dyn ListArrayType)
}
DataType::ListView(_) => {
Ok(list_array.as_list_view::<i32>() as &dyn ListArrayType)
}
DataType::LargeListView(_) => {
Ok(list_array.as_list_view::<i64>() as &dyn ListArrayType)
}
other => exec_err!("Invalid unnest datatype {other }"),
})
.collect::<Result<Vec<_>>>()?;
Expand Down
2 changes: 2 additions & 0 deletions datafusion/sql/src/expr/function.rs
Original file line number Diff line number Diff line change
Expand Up @@ -905,6 +905,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
DataType::List(_)
| DataType::LargeList(_)
| DataType::FixedSizeList(_, _)
| DataType::ListView(_)
| DataType::LargeListView(_)
| DataType::Struct(_) => Ok(()),
DataType::Null => {
not_impl_err!("unnest() does not support null yet")
Expand Down
4 changes: 3 additions & 1 deletion datafusion/sql/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,9 @@ impl RecursiveUnnestRewriter<'_> {
}
DataType::List(_)
| DataType::FixedSizeList(_, _)
| DataType::LargeList(_) => {
| DataType::LargeList(_)
| DataType::ListView(_)
| DataType::LargeListView(_) => {
push_projection_dedupl(
self.inner_projection_exprs,
expr_in_unnest.clone().alias(placeholder_name.clone()),
Expand Down
96 changes: 96 additions & 0 deletions datafusion/sqllogictest/test_files/unnest.slt
Original file line number Diff line number Diff line change
Expand Up @@ -1233,3 +1233,99 @@ physical_plan
# cleanup
statement ok
drop table t;

########################################
# Unnest ListView / LargeListView Tests #
########################################

## Basic unnest ListView in select list
query I
select unnest(arrow_cast([1,2,3], 'ListView(Int64)'));
----
1
2
3

## Basic unnest ListView in from clause
query I
select * from unnest(arrow_cast([1,2,3], 'ListView(Int64)'));
----
1
2
3

## Basic unnest LargeListView in select list
query I
select unnest(arrow_cast([1,2,3], 'LargeListView(Int64)'));
----
1
2
3

## Basic unnest LargeListView in from clause
query I
select * from unnest(arrow_cast([1,2,3], 'LargeListView(Int64)'));
----
1
2
3

## Unnest ListView with range
query I
select unnest(arrow_cast(range(1, 3), 'ListView(Int64)'));
----
1
2

## Unnest LargeListView with range
query I
select * from unnest(arrow_cast(range(1, 3), 'LargeListView(Int64)'));
----
1
2

## Multiple unnest with ListView columns from a table
query III
select
unnest(column1),
unnest(arrow_cast(column2, 'ListView(Int64)')),
unnest(arrow_cast(column4, 'LargeListView(Int64)'))
from unnest_table where column4 is not null;
----
1 7 13
2 NULL 14
3 NULL NULL
4 8 15
5 9 16
NULL 10 NULL
NULL NULL 17
NULL NULL 18

## Unnest ListView with null elements
query I
select unnest(arrow_cast([1, null, 3], 'ListView(Int64)'));
----
1
NULL
3

## Unnest empty ListView
query I
select unnest(arrow_cast([], 'ListView(Int64)'));
----

## Unnest ListView of strings
query T
select unnest(arrow_cast(['a','b','c'], 'ListView(Utf8)'));
----
a
b
c

## Unnest LargeListView of strings
query T
select unnest(arrow_cast(['a','b','c'], 'LargeListView(Utf8)'));
----
a
b
c
Loading