feat: Add LargeList support for JavaScript bindings

src/Arrow.dom.ts

@@ -57,6 +57,7 @@ export {
     Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond,
     Decimal,
     List,
+    LargeList,
     Struct, StructRow,
     Union, DenseUnion, SparseUnion,
     Dictionary,

src/Arrow.ts

@@ -45,6 +45,7 @@ export {
     Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond,
     Decimal,
     List,
+    LargeList,
     Struct,
     Union, DenseUnion, SparseUnion,
     Dictionary,

src/data.ts

@@ -256,7 +256,7 @@ export class Data<T extends DataType = DataType> {
 
 import {
     Dictionary,
-    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct,
+    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, LargeList, FixedSizeList, Map_, Struct,
     Float,
     Int,
     Date_,
@@ -377,6 +377,13 @@ class MakeDataVisitor extends Visitor {
         const { ['length']: length = valueOffsets.length - 1, ['nullCount']: nullCount = props['nullBitmap'] ? -1 : 0 } = props;
         return new Data(type, offset, length, nullCount, [valueOffsets, undefined, nullBitmap], [child]);
     }
+    public visitLargeList<T extends LargeList>(props: LargeListDataProps<T>) {
+        const { ['type']: type, ['offset']: offset = 0, ['child']: child } = props;
+        const nullBitmap = toUint8Array(props['nullBitmap']);
+        const valueOffsets = toBigInt64Array(props['valueOffsets']);
+        const { ['length']: length = valueOffsets.length - 1, ['nullCount']: nullCount = props['nullBitmap'] ? -1 : 0 } = props;
+        return new Data(type, offset, length, nullCount, [valueOffsets, undefined, nullBitmap], [child]);
+    }
     public visitStruct<T extends Struct>(props: StructDataProps<T>) {
         const { ['type']: type, ['offset']: offset = 0, ['children']: children = [] } = props;
         const nullBitmap = toUint8Array(props['nullBitmap']);
@@ -459,6 +466,7 @@ interface LargeBinaryDataProps<T extends LargeBinary> extends DataProps_<T> { va
 interface Utf8DataProps<T extends Utf8> extends DataProps_<T> { valueOffsets: ValueOffsetsBuffer; data?: DataBuffer<T> }
 interface LargeUtf8DataProps<T extends LargeUtf8> extends DataProps_<T> { valueOffsets: LargeValueOffsetsBuffer | ValueOffsetsBuffer; data?: DataBuffer<T> }
 interface ListDataProps<T extends List> extends DataProps_<T> { valueOffsets: ValueOffsetsBuffer; child: Data<T['valueType']> }
+interface LargeListDataProps<T extends LargeList> extends DataProps_<T> { valueOffsets: LargeValueOffsetsBuffer | ValueOffsetsBuffer; child: Data<T['valueType']> }
 interface FixedSizeListDataProps<T extends FixedSizeList> extends DataProps_<T> { child: Data<T['valueType']> }
 interface StructDataProps<T extends Struct> extends DataProps_<T> { children: Data[] }
 interface Map_DataProps<T extends Map_> extends DataProps_<T> { valueOffsets: ValueOffsetsBuffer; child: Data }
@@ -484,6 +492,7 @@ export type DataProps<T extends DataType> = (
     T extends Utf8 /*            */ ? Utf8DataProps<T> :
     T extends LargeUtf8 /*       */ ? LargeUtf8DataProps<T> :
     T extends List /*            */ ? ListDataProps<T> :
+    T extends LargeList /*       */ ? LargeListDataProps<T> :
     T extends FixedSizeList /*   */ ? FixedSizeListDataProps<T> :
     T extends Struct /*          */ ? StructDataProps<T> :
     T extends Map_ /*            */ ? Map_DataProps<T> :
@@ -512,6 +521,7 @@ export function makeData<T extends LargeBinary>(props: LargeBinaryDataProps<T>):
 export function makeData<T extends Utf8>(props: Utf8DataProps<T>): Data<T>;
 export function makeData<T extends LargeUtf8>(props: LargeUtf8DataProps<T>): Data<T>;
 export function makeData<T extends List>(props: ListDataProps<T>): Data<T>;
+export function makeData<T extends LargeList>(props: LargeListDataProps<T>): Data<T>;
 export function makeData<T extends FixedSizeList>(props: FixedSizeListDataProps<T>): Data<T>;
 export function makeData<T extends Struct>(props: StructDataProps<T>): Data<T>;
 export function makeData<T extends Map_>(props: Map_DataProps<T>): Data<T>;

src/enum.ts

@@ -70,6 +70,7 @@ export enum Type {
     Duration = 18, /** Measure of elapsed time in either seconds, milliseconds, microseconds or nanoseconds */
     LargeBinary = 19, /** Large variable-length bytes (no guarantee of UTF8-ness) */
     LargeUtf8 = 20, /** Large variable-length string as List<Char> */
+    LargeList = 21, /** A list of some logical data type with 64-bit offsets */
 
     Dictionary = -1, /** Dictionary aka Category type */
     Int8 = -2,

src/ipc/metadata/message.ts

@@ -58,7 +58,7 @@ import ByteBuffer = flatbuffers.ByteBuffer;
 import {
     DataType, Dictionary, TimeBitWidth,
     Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary,
-    List, FixedSizeList, Map_, Struct, Union,
+    List, LargeList, FixedSizeList, Map_, Struct, Union,
     Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, Duration,
 } from '../../type.js';
 
@@ -472,6 +472,7 @@ function decodeFieldType(f: _Field, children?: Field[]): DataType<any> {
         case Type['LargeUtf8']: return new LargeUtf8();
         case Type['Bool']: return new Bool();
         case Type['List']: return new List((children || [])[0]);
+        case Type['LargeList']: return new LargeList((children || [])[0]);
         case Type['Struct_']: return new Struct(children || []);
     }
 

src/type.ts

@@ -69,6 +69,7 @@ export abstract class DataType<TType extends Type = Type, TChildren extends Type
     /** @nocollapse */ static isInterval(x: any): x is Interval_ { return x?.typeId === Type.Interval; }
     /** @nocollapse */ static isDuration(x: any): x is Duration { return x?.typeId === Type.Duration; }
     /** @nocollapse */ static isList(x: any): x is List { return x?.typeId === Type.List; }
+    /** @nocollapse */ static isLargeList(x: any): x is LargeList { return x?.typeId === Type.LargeList; }
     /** @nocollapse */ static isStruct(x: any): x is Struct { return x?.typeId === Type.Struct; }
     /** @nocollapse */ static isUnion(x: any): x is Union_ { return x?.typeId === Type.Union; }
     /** @nocollapse */ static isFixedSizeBinary(x: any): x is FixedSizeBinary { return x?.typeId === Type.FixedSizeBinary; }
@@ -546,6 +547,32 @@ export class List<T extends DataType = any> extends DataType<Type.List, { [0]: T
     })(List.prototype);
 }
 
+/** @ignore */
+export interface LargeList<T extends DataType = any> extends DataType<Type.LargeList, { [0]: T }> {
+    TArray: Array<T>;
+    TValue: Vector<T>;
+    TOffsetArray: BigInt64Array;
+    OffsetArrayType: BigIntArrayConstructor<BigInt64Array>;
+}
+
+/** @ignore */
+export class LargeList<T extends DataType = any> extends DataType<Type.LargeList, { [0]: T }> {
+    constructor(child: Field<T>) {
+        super(Type.LargeList);
+        this.children = [child];
+    }
+    public declare readonly children: Field<T>[];
+    public toString() { return `LargeList<${this.valueType}>`; }
+    public get valueType(): T { return this.children[0].type as T; }
+    public get valueField(): Field<T> { return this.children[0] as Field<T>; }
+    public get ArrayType(): T['ArrayType'] { return this.valueType.ArrayType; }
+    protected static [Symbol.toStringTag] = ((proto: LargeList) => {
+        (<any>proto).children = null;
+        (<any>proto).OffsetArrayType = BigInt64Array;
+        return proto[Symbol.toStringTag] = 'LargeList';
+    })(LargeList.prototype);
+}
+
 /** @ignore */
 export interface Struct<T extends TypeMap = any> extends DataType<Type.Struct, T> {
     TArray: Array<StructRowProxy<T>>;

src/visitor.ts

@@ -45,6 +45,7 @@ export abstract class Visitor {
     public visitTime(_node: any, ..._args: any[]): any { return null; }
     public visitDecimal(_node: any, ..._args: any[]): any { return null; }
     public visitList(_node: any, ..._args: any[]): any { return null; }
+    public visitLargeList(_node: any, ..._args: any[]): any { return null; }
     public visitStruct(_node: any, ..._args: any[]): any { return null; }
     public visitUnion(_node: any, ..._args: any[]): any { return null; }
     public visitDictionary(_node: any, ..._args: any[]): any { return null; }
@@ -110,6 +111,7 @@ function getVisitFnByTypeId(visitor: Visitor, dtype: Type, throwIfNotFound = tru
         case Type.TimeNanosecond: fn = visitor.visitTimeNanosecond || visitor.visitTime; break;
         case Type.Decimal: fn = visitor.visitDecimal; break;
         case Type.List: fn = visitor.visitList; break;
+        case Type.LargeList: fn = visitor.visitLargeList; break;
         case Type.Struct: fn = visitor.visitStruct; break;
         case Type.Union: fn = visitor.visitUnion; break;
         case Type.DenseUnion: fn = visitor.visitDenseUnion || visitor.visitUnion; break;
@@ -205,6 +207,7 @@ function inferDType<T extends DataType>(type: T): Type {
             return Type.Duration;
         case Type.Map: return Type.Map;
         case Type.List: return Type.List;
+        case Type.LargeList: return Type.LargeList;
         case Type.Struct: return Type.Struct;
         case Type.Union:
             switch ((type as any as Union).mode) {

src/visitor/get.ts

@@ -28,7 +28,7 @@ import { uint16ToFloat64 } from '../util/math.js';
 import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum.js';
 import {
     DataType, Dictionary,
-    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct,
+    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, LargeList, FixedSizeList, Map_, Struct,
     Float, Float16, Float32, Float64,
     Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64,
     Date_, DateDay, DateMillisecond,
@@ -81,6 +81,7 @@ export interface GetVisitor extends Visitor {
     visitTimeNanosecond<T extends TimeNanosecond>(data: Data<T>, index: number): T['TValue'] | null;
     visitDecimal<T extends Decimal>(data: Data<T>, index: number): T['TValue'] | null;
     visitList<T extends List>(data: Data<T>, index: number): T['TValue'] | null;
+    visitLargeList<T extends LargeList>(data: Data<T>, index: number): T['TValue'] | null;
     visitStruct<T extends Struct>(data: Data<T>, index: number): T['TValue'] | null;
     visitUnion<T extends Union>(data: Data<T>, index: number): T['TValue'] | null;
     visitDenseUnion<T extends DenseUnion>(data: Data<T>, index: number): T['TValue'] | null;
@@ -222,6 +223,16 @@ const getList = <T extends List>(data: Data<T>, index: number): T['TValue'] => {
     return new Vector([slice]) as T['TValue'];
 };
 
+/** @ignore */
+const getLargeList = <T extends LargeList>(data: Data<T>, index: number): T['TValue'] => {
+    const { valueOffsets, stride, children } = data;
+    const begin = Number(valueOffsets[index * stride]);
+    const end = Number(valueOffsets[index * stride + 1]);
+    const child: Data<T['valueType']> = children[0];
+    const slice = child.slice(begin, end - begin);
+    return new Vector([slice]) as T['TValue'];
+};
+
 /** @ignore */
 const getMap = <T extends Map_>(data: Data<T>, index: number): T['TValue'] => {
     const { valueOffsets, children } = data;
@@ -350,6 +361,7 @@ GetVisitor.prototype.visitTimeMicrosecond = wrapGet(getTimeMicrosecond);
 GetVisitor.prototype.visitTimeNanosecond = wrapGet(getTimeNanosecond);
 GetVisitor.prototype.visitDecimal = wrapGet(getDecimal);
 GetVisitor.prototype.visitList = wrapGet(getList);
+GetVisitor.prototype.visitLargeList = wrapGet(getLargeList);
 GetVisitor.prototype.visitStruct = wrapGet(getStruct);
 GetVisitor.prototype.visitUnion = wrapGet(getUnion);
 GetVisitor.prototype.visitDenseUnion = wrapGet(getDenseUnion);

src/visitor/indexof.ts

@@ -24,7 +24,7 @@ import { getBool, BitIterator } from '../util/bit.js';
 import { createElementComparator } from '../util/vector.js';
 import {
     DataType, Dictionary,
-    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct,
+    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, LargeList, FixedSizeList, Map_, Struct,
     Float, Float16, Float32, Float64,
     Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64,
     Date_, DateDay, DateMillisecond,
@@ -77,6 +77,7 @@ export interface IndexOfVisitor extends Visitor {
     visitTimeNanosecond<T extends TimeNanosecond>(data: Data<T>, value: T['TValue'] | null, index?: number): number;
     visitDecimal<T extends Decimal>(data: Data<T>, value: T['TValue'] | null, index?: number): number;
     visitList<T extends List>(data: Data<T>, value: T['TValue'] | null, index?: number): number;
+    visitLargeList<T extends LargeList>(data: Data<T>, value: T['TValue'] | null, index?: number): number;
     visitStruct<T extends Struct>(data: Data<T>, value: T['TValue'] | null, index?: number): number;
     visitUnion<T extends Union>(data: Data<T>, value: T['TValue'] | null, index?: number): number;
     visitDenseUnion<T extends DenseUnion>(data: Data<T>, value: T['TValue'] | null, index?: number): number;
@@ -195,6 +196,7 @@ IndexOfVisitor.prototype.visitTimeMicrosecond = indexOfValue;
 IndexOfVisitor.prototype.visitTimeNanosecond = indexOfValue;
 IndexOfVisitor.prototype.visitDecimal = indexOfValue;
 IndexOfVisitor.prototype.visitList = indexOfValue;
+IndexOfVisitor.prototype.visitLargeList = indexOfValue;
 IndexOfVisitor.prototype.visitStruct = indexOfValue;
 IndexOfVisitor.prototype.visitUnion = indexOfValue;
 IndexOfVisitor.prototype.visitDenseUnion = indexOfUnion;

src/visitor/iterator.ts

@@ -21,7 +21,7 @@ import { Type, Precision } from '../enum.js';
 import { TypeToDataType } from '../interfaces.js';
 import {
     DataType, Dictionary,
-    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct,
+    Bool, Null, Utf8, LargeUtf8, Binary, LargeBinary, Decimal, FixedSizeBinary, List, LargeList, FixedSizeList, Map_, Struct,
     Float, Float16, Float32, Float64,
     Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64,
     Date_, DateDay, DateMillisecond,
@@ -75,6 +75,7 @@ export interface IteratorVisitor extends Visitor {
     visitTimeNanosecond<T extends TimeNanosecond>(vector: Vector<T>): IterableIterator<T['TValue'] | null>;
     visitDecimal<T extends Decimal>(vector: Vector<T>): IterableIterator<T['TValue'] | null>;
     visitList<T extends List>(vector: Vector<T>): IterableIterator<T['TValue'] | null>;
+    visitLargeList<T extends LargeList>(vector: Vector<T>): IterableIterator<T['TValue'] | null>;
     visitStruct<T extends Struct>(vector: Vector<T>): IterableIterator<T['TValue'] | null>;
     visitUnion<T extends Union>(vector: Vector<T>): IterableIterator<T['TValue'] | null>;
     visitDenseUnion<T extends DenseUnion>(vector: Vector<T>): IterableIterator<T['TValue'] | null>;
@@ -182,6 +183,7 @@ IteratorVisitor.prototype.visitTimeMicrosecond = vectorIterator;
 IteratorVisitor.prototype.visitTimeNanosecond = vectorIterator;
 IteratorVisitor.prototype.visitDecimal = vectorIterator;
 IteratorVisitor.prototype.visitList = vectorIterator;
+IteratorVisitor.prototype.visitLargeList = vectorIterator;
 IteratorVisitor.prototype.visitStruct = vectorIterator;
 IteratorVisitor.prototype.visitUnion = vectorIterator;
 IteratorVisitor.prototype.visitDenseUnion = vectorIterator;

src/visitor/jsontypeassembler.ts

@@ -75,6 +75,9 @@ export class JSONTypeAssembler extends Visitor {
     public visitList<T extends type.List>({ typeId }: T) {
         return { 'name': ArrowType[typeId].toLowerCase() };
     }
+    public visitLargeList<T extends type.LargeList>({ typeId }: T) {
+        return { 'name': ArrowType[typeId].toLowerCase() };
+    }
     public visitStruct<T extends type.Struct>({ typeId }: T) {
         return { 'name': ArrowType[typeId].toLowerCase() };
     }

src/visitor/typeassembler.ts

@@ -36,6 +36,7 @@ import { Timestamp } from '../fb/timestamp.js';
 import { Interval } from '../fb/interval.js';
 import { Duration } from '../fb/duration.js';
 import { List } from '../fb/list.js';
+import { LargeList } from '../fb/large-list.js';
 import { Struct_ as Struct } from '../fb/struct-.js';
 import { Union } from '../fb/union.js';
 import { DictionaryEncoding } from '../fb/dictionary-encoding.js';
@@ -129,6 +130,10 @@ export class TypeAssembler extends Visitor {
         List.startList(b);
         return List.endList(b);
     }
+    public visitLargeList<T extends type.LargeList>(_node: T, b: Builder) {
+        LargeList.startLargeList(b);
+        return LargeList.endLargeList(b);
+    }
     public visitStruct<T extends type.Struct>(_node: T, b: Builder) {
         Struct.startStruct_(b);
         return Struct.endStruct_(b);