Skip to content

Don't read OFFSET vector for FixedSizeList #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 23, 2018
Merged

Don't read OFFSET vector for FixedSizeList #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions js/src/data.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { popcnt_bit_range } from './util/bit';
import { VectorLike, Vector } from './vector';
import { VectorType, TypedArray, TypedArrayConstructor, Dictionary } from './type';
import { Int, Bool, FlatListType, List, FixedSizeList, Struct, Map_ } from './type';
import { DataType, FlatType, ListType, NestedType, DenseUnion, SparseUnion } from './type';
import { DataType, FlatType, ListType, NestedType, SingleNestedType, DenseUnion, SparseUnion } from './type';

export function toTypedArray<T extends TypedArray>(ArrayType: TypedArrayConstructor<T>, values?: T | ArrayLike<number> | Iterable<number> | null): T {
if (!ArrayType && ArrayBuffer.isView(values)) { return values; }
Expand All @@ -46,7 +46,7 @@ export interface DataTypes<T extends DataType> {
/* [Type.Struct]*/ 13: NestedData<Struct>;
/* [Type.Union]*/ 14: UnionData;
/* [Type.FixedSizeBinary]*/ 15: FlatData<T>;
/* [Type.FixedSizeList]*/ 16: ListData<FixedSizeList<T>>;
/* [Type.FixedSizeList]*/ 16: SingleNestedData<FixedSizeList<T>>;
/* [Type.Map]*/ 17: NestedData<Map_>;
/* [Type.DenseUnion]*/ DenseUnion: DenseUnionData;
/*[Type.SparseUnion]*/ SparseUnion: SparseUnionData;
Expand Down Expand Up @@ -195,15 +195,21 @@ export class NestedData<T extends NestedType = NestedType> extends BaseData<T> {
}
}

export class ListData<T extends ListType> extends NestedData<T> {
public /* [VectorType.OFFSET]:*/ 0: Int32Array;
public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
export class SingleNestedData<T extends SingleNestedType> extends NestedData<T> {
protected _valuesData: Data<T>;
public get values() { return this._valuesData; }
public get valueOffsets() { return this[VectorType.OFFSET]; }
constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable<number>, valueChildData: Data<T>, offset?: number, nullCount?: number) {
constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueChildData: Data<T>, offset?: number, nullCount?: number) {
super(type, length, nullBitmap, [valueChildData], offset, nullCount);
this._valuesData = valueChildData;
}
}

export class ListData<T extends ListType> extends SingleNestedData<T> {
public /* [VectorType.OFFSET]:*/ 0: Int32Array;
public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
public get valueOffsets() { return this[VectorType.OFFSET]; }
constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, valueOffsets: Iterable<number>, valueChildData: Data<T>, offset?: number, nullCount?: number) {
super(type, length, nullBitmap, valueChildData, offset, nullCount);
this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
}
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
Expand Down
7 changes: 5 additions & 2 deletions js/src/ipc/reader/vector.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { RecordBatch } from '../../recordbatch';
import { TypeVisitor } from '../../visitor';
import { FlatType, NestedType, ListType } from '../../type';
import { Message, FieldMetadata, BufferMetadata } from '../metadata';
import { FlatData, ListData, NestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data';
import { FlatData, ListData, NestedData, SingleNestedData, DenseUnionData, SparseUnionData, BoolData, FlatListData, DictionaryData } from '../../data';
import {
Schema, Field,
Dictionary,
Expand Down Expand Up @@ -89,7 +89,7 @@ export abstract class TypeDataLoader extends TypeVisitor {
public visitStruct (type: Struct) { return this.visitNestedType(type); }
public visitUnion (type: Union) { return this.visitUnionType(type); }
public visitFixedSizeBinary(type: FixedSizeBinary) { return this.visitFlatType(type); }
public visitFixedSizeList (type: FixedSizeList) { return this.visitListType(type); }
public visitFixedSizeList (type: FixedSizeList) { return this.visitFixedSizeListType(type); }
public visitMap (type: Map_) { return this.visitNestedType(type); }
public visitDictionary (type: Dictionary) {
return new DictionaryData(type, this.dictionaries.get(type.id)!, this.visit(type.indicies));
Expand Down Expand Up @@ -117,6 +117,9 @@ export abstract class TypeDataLoader extends TypeVisitor {
protected visitListType<T extends ListType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
return new ListData<T>(type, length, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children![0].type), 0, nullCount);
}
protected visitFixedSizeListType<T extends FixedSizeList>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
return new SingleNestedData<T>(type, length, this.readNullBitmap(type, nullCount), this.visit(type.children![0].type), 0, nullCount);
}
protected visitNestedType<T extends NestedType>(type: T, { length, nullCount }: FieldMetadata = this.getFieldMetadata()) {
return new NestedData<T>(type, length, this.readNullBitmap(type, nullCount), this.visitFields(type.children), 0, nullCount);
}
Expand Down
3 changes: 2 additions & 1 deletion js/src/type.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,9 @@ export type PrimitiveType = NumericType | FixedSizeType;

export type FlatListType = Utf8 | Binary; // <-- these types have `offset`, `data`, and `validity` buffers
export type FlatType = Bool | PrimitiveType | FlatListType; // <-- these types have `data` and `validity` buffers
export type ListType = List<any> | FixedSizeList<any>; // <-- these types have `offset` and `validity` buffers
export type ListType = List<any>; // <-- these types have `offset` and `validity` buffers
export type NestedType = Map_ | Struct | List<any> | FixedSizeList<any> | Union<any>; // <-- these types have `validity` buffer and nested childData
export type SingleNestedType = List<any> | FixedSizeList<any>; // <-- these are nested types that can only have a single child

/**
* *
Expand Down
2 changes: 1 addition & 1 deletion js/src/vector.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ export class ListVector<T extends DataType = DataType> extends ListVectorBase<Li
}
}

export class FixedSizeListVector extends ListVectorBase<FixedSizeList> {
export class FixedSizeListVector extends Vector<FixedSizeList> {
constructor(data: Data<FixedSizeList>, view: View<FixedSizeList> = new FixedSizeListView(data)) {
super(data, view);
}
Expand Down
17 changes: 12 additions & 5 deletions js/src/vector/list.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,13 @@ export const decodeUtf8 = ((decoder) =>
decoder.decode.bind(decoder) as (input?: ArrayBufferLike | ArrayBufferView) => string
)(new TextDecoder('utf-8'));

export abstract class ListViewBase<T extends (ListType | FlatListType)> implements View<T> {
export abstract class ListViewBase<T extends (ListType | FlatListType | FixedSizeList)> implements View<T> {
public length: number;
public values: T['TArray'];
public valueOffsets?: Int32Array;
constructor(data: Data<T>) {
this.length = data.length;
this.values = data.values;
this.valueOffsets = data.valueOffsets;
}
public clone(data: Data<T>): this {
return new (<any> this.constructor)(data) as this;
Expand Down Expand Up @@ -64,7 +63,15 @@ export abstract class ListViewBase<T extends (ListType | FlatListType)> implemen
protected abstract setList(values: T['TArray'], index: number, value: T['TValue'], valueOffsets?: Int32Array): void;
}

export class ListView<T extends DataType> extends ListViewBase<List<T>> {
export abstract class VariableListViewBase<T extends (ListType | FlatListType)> extends ListViewBase<T> {
constructor(data: Data<T>) {
super(data)
this.length = data.length;
this.valueOffsets = data.valueOffsets;
}
}

export class ListView<T extends DataType> extends VariableListViewBase<List<T>> {
constructor(data: Data<List<T>>) {
super(data);
this.values = createVector(data.values);
Expand Down Expand Up @@ -101,7 +108,7 @@ export class FixedSizeListView<T extends DataType> extends ListViewBase<FixedSiz
}
}

export class BinaryView extends ListViewBase<Binary> {
export class BinaryView extends VariableListViewBase<Binary> {
protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) {
return values.subarray(valueOffsets[index], valueOffsets[index + 1]);
}
Expand All @@ -111,7 +118,7 @@ export class BinaryView extends ListViewBase<Binary> {
}
}

export class Utf8View extends ListViewBase<Utf8> {
export class Utf8View extends VariableListViewBase<Utf8> {
protected getList(values: Uint8Array, index: number, valueOffsets: Int32Array) {
return decodeUtf8(values.subarray(valueOffsets[index], valueOffsets[index + 1]));
}
Expand Down