Skip to content

ARROW-4523,ARROW-4524: [JS] Add row proxy generation benchmark, improve performance #3601

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

ARROW-4523,ARROW-4524: [JS] Add row proxy generation benchmark, improve performance #3601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fb7a0f0
add row proxy benchmark
TheNeuralBit Feb 10, 2019
8a5d162
Improve Row proxy performance
TheNeuralBit Feb 10, 2019
da4d97c
Switch back to Object.create with no extra property descriptor
TheNeuralBit Feb 11, 2019
4079439
linter fixes
TheNeuralBit Feb 11, 2019
7bb6e4f
Update comparator, switch to Symbols for internal variables in Row
TheNeuralBit Feb 12, 2019
2442545
Remove inner class, don't re-define columns
TheNeuralBit Feb 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions js/perf/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,12 @@ for (let { name, buffers } of require('./table_config')) {
for (let {name, buffers, countBys, counts} of require('./table_config')) {
const table = Table.from(buffers);

const tableIterateSuiteName = `Table Iterate "${name}"`;
const dfCountBySuiteName = `DataFrame Count By "${name}"`;
const dfFilterCountSuiteName = `DataFrame Filter-Scan Count "${name}"`;
const dfDirectCountSuiteName = `DataFrame Direct Count "${name}"`;

suites.push(createTestSuite(tableIterateSuiteName, createTableIterateTest(table)));
suites.push(...countBys.map((countBy) => createTestSuite(dfCountBySuiteName, createDataFrameCountByTest(table, countBy))));
suites.push(...counts.map(({ col, test, value }) => createTestSuite(dfFilterCountSuiteName, createDataFrameFilterCountTest(table, col, test, value))));
suites.push(...counts.map(({ col, test, value }) => createTestSuite(dfDirectCountSuiteName, createDataFrameDirectCountTest(table, col, test, value))));
Expand Down Expand Up @@ -135,6 +137,15 @@ function createGetByIndexTest(vector, name) {
};
}

function createTableIterateTest(table) {
let value;
return {
async: true,
name: `length: ${table.length}\n`,
fn() { for (value of table) {} }
};
}

function createDataFrameDirectCountTest(table, column, test, value) {
let sum, colidx = table.schema.fields.findIndex((c)=>c.name === column);

Expand Down
137 changes: 91 additions & 46 deletions js/src/util/vector.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
// under the License.

import { Vector } from '../vector';
import { Row } from '../vector/row';
import { Row, kLength } from '../vector/row';
import { compareArrayLike } from '../util/buffer';

/** @ignore */
Expand Down Expand Up @@ -75,60 +75,105 @@ export function createElementComparator(search: any) {
}
// Compare Array-likes
if (Array.isArray(search)) {
const n = (search as any).length;
const fns = [] as ((x: any) => boolean)[];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator((search as any)[i]);
}
return (value: any) => {
if (!value || value.length !== n) { return false; }
// Handle the case where the search element is an Array, but the
// values are Rows or Vectors, e.g. list.indexOf(['foo', 'bar'])
if ((value instanceof Row) || (value instanceof Vector)) {
for (let i = -1, n = value.length; ++i < n;) {
if (!(fns[i]((value as any).get(i)))) { return false; }
}
return true;
}
for (let i = -1, n = value.length; ++i < n;) {
if (!(fns[i](value[i]))) { return false; }
return createArrayLikeComparator(search);
}
// Compare Rows
if (search instanceof Row) {
return createRowComparator(search);
}
// Compare Vectors
if (search instanceof Vector) {
return createVectorComparator(search);
}
// Compare non-empty Objects
const keys = Object.keys(search);
if (keys.length > 0) {
return createObjectKeysComparator(search, keys);
}
// No valid comparator
return () => false;
}

/** @ignore */
function createArrayLikeComparator(search: ArrayLike<any>) {
const n = search.length;
const fns = [] as ((x: any) => boolean)[];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator((search as any)[i]);
}
return (value: any) => {
if (!value) { return false; }
// Handle the case where the search element is an Array, but the
// values are Rows or Vectors, e.g. list.indexOf(['foo', 'bar'])
if (value instanceof Row) {
if (value[kLength] !== n) { return false; }
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) { return false; }
}
return true;
};
}
// Compare Rows and Vectors
if ((search instanceof Row) || (search instanceof Vector)) {
const n = search.length;
const C = search.constructor as any;
const fns = [] as ((x: any) => boolean)[];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator((search as any).get(i));
}
return (value: any) => {
if (!(value instanceof C)) { return false; }
if (!(value.length === n)) { return false; }
if (value.length !== n) { return false; }
if (value instanceof Vector) {
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) { return false; }
}
return true;
};
}
for (let i = -1; ++i < n;) {
if (!(fns[i](value[i]))) { return false; }
}
return true;
};
}

/** @ignore */
function createRowComparator(search: Row<any>) {
const n = search[kLength];
const C = search.constructor as any;
const fns = [] as ((x: any) => boolean)[];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search.get(i));
}
// Compare non-empty Objects
const keys = Object.keys(search);
if (keys.length > 0) {
const n = keys.length;
const fns = [] as ((x: any) => boolean)[];
return (value: any) => {
if (!(value instanceof C)) { return false; }
if (!(value[kLength] === n)) { return false; }
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search[keys[i]]);
if (!(fns[i](value.get(i)))) { return false; }
}
return (value: any) => {
if (!value || typeof value !== 'object') { return false; }
for (let i = -1; ++i < n;) {
if (!(fns[i](value[keys[i]]))) { return false; }
}
return true;
};
return true;
};
}

/** @ignore */
function createVectorComparator(search: Vector<any>) {
const n = search.length;
const C = search.constructor as any;
const fns = [] as ((x: any) => boolean)[];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator((search as any).get(i));
}
// No valid comparator
return () => false;
return (value: any) => {
if (!(value instanceof C)) { return false; }
if (!(value.length === n)) { return false; }
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) { return false; }
}
return true;
};
}

/** @ignore */
function createObjectKeysComparator(search: any, keys: string[]) {
const n = keys.length;
const fns = [] as ((x: any) => boolean)[];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search[keys[i]]);
}
return (value: any) => {
if (!value || typeof value !== 'object') { return false; }
for (let i = -1; ++i < n;) {
if (!(fns[i](value[keys[i]]))) { return false; }
}
return true;
};
}
8 changes: 4 additions & 4 deletions js/src/vector/map.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

import { Row } from './row';
import { RowProxyGenerator } from './row';
import { Vector } from '../vector';
import { BaseVector } from './base';
import { DataType, Map_, Struct } from '../type';
Expand All @@ -25,8 +25,8 @@ export class MapVector<T extends { [key: string]: DataType } = any> extends Base
return Vector.new(this.data.clone(new Struct<T>(this.type.children)));
}
// @ts-ignore
private _rowProxy: Row<T>;
public get rowProxy(): Row<T> {
return this._rowProxy || (this._rowProxy = Row.new<T>(this.type.children || [], true));
private _rowProxy: RowProxyGenerator<T>;
public get rowProxy(): RowProxyGenerator<T> {
return this._rowProxy || (this._rowProxy = RowProxyGenerator.new<T>(this, this.type.children || [], true));
}
}
123 changes: 68 additions & 55 deletions js/src/vector/row.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,84 +17,97 @@

import { Field } from '../schema';
import { MapVector } from '../vector/map';
import { DataType, RowLike } from '../type';
import { DataType } from '../type';
import { valueToString } from '../util/pretty';
import { StructVector } from '../vector/struct';

/** @ignore */ const columnDescriptor = { enumerable: true, configurable: false, get: () => {} };
/** @ignore */ const lengthDescriptor = { writable: false, enumerable: false, configurable: false, value: -1 };
/** @ignore */ const rowIndexDescriptor = { writable: false, enumerable: false, configurable: true, value: null as any };
/** @ignore */ export const kLength = Symbol.for('length');
/** @ignore */ export const kParent = Symbol.for('parent');
/** @ignore */ export const kRowIndex = Symbol.for('rowIndex');
/** @ignore */ const columnDescriptor = { enumerable: true, configurable: false, get: null as any };
/** @ignore */ const rowLengthDescriptor = { writable: false, enumerable: false, configurable: false, value: -1 };
/** @ignore */ const rowParentDescriptor = { writable: false, enumerable: false, configurable: false, value: null as any };
/** @ignore */ const row = { parent: rowParentDescriptor, rowIndex: rowIndexDescriptor };

/** @ignore */
export class Row<T extends { [key: string]: DataType }> implements Iterable<T[keyof T]['TValue']> {
[key: string]: T[keyof T]['TValue'];
/** @nocollapse */
public static new<T extends { [key: string]: DataType }>(schemaOrFields: T | Field[], fieldsAreEnumerable = false): RowLike<T> & Row<T> {
let schema: T, fields: Field[];
if (Array.isArray(schemaOrFields)) {
fields = schemaOrFields;
} else {
schema = schemaOrFields;
fieldsAreEnumerable = true;
fields = Object.keys(schema).map((x) => new Field(x, schema[x]));
}
return new Row<T>(fields, fieldsAreEnumerable) as RowLike<T> & Row<T>;
}
// @ts-ignore
private parent: TParent;
public [kParent]: MapVector<T> | StructVector<T>;
// @ts-ignore
private rowIndex: number;
public [kRowIndex]: number;
// @ts-ignore
public readonly length: number;
private constructor(fields: Field[], fieldsAreEnumerable: boolean) {
lengthDescriptor.value = fields.length;
Object.defineProperty(this, 'length', lengthDescriptor);
fields.forEach((field, columnIndex) => {
columnDescriptor.get = this._bindGetter(columnIndex);
// set configurable to true to ensure Object.defineProperty
// doesn't throw in the case of duplicate column names
columnDescriptor.configurable = true;
columnDescriptor.enumerable = fieldsAreEnumerable;
Object.defineProperty(this, field.name, columnDescriptor);
columnDescriptor.configurable = false;
columnDescriptor.enumerable = !fieldsAreEnumerable;
Object.defineProperty(this, columnIndex, columnDescriptor);
columnDescriptor.get = null as any;
});
}
*[Symbol.iterator](this: RowLike<T>) {
for (let i = -1, n = this.length; ++i < n;) {
public readonly [kLength]: number;
*[Symbol.iterator]() {
for (let i = -1, n = this[kLength]; ++i < n;) {
yield this[i];
}
}
private _bindGetter(colIndex: number) {
return function (this: Row<T>) {
let child = this.parent.getChildAt(colIndex);
return child ? child.get(this.rowIndex) : null;
};
}
public get<K extends keyof T>(key: K) { return (this as any)[key] as T[K]['TValue']; }
public bind<TParent extends MapVector<T> | StructVector<T>>(parent: TParent, rowIndex: number) {
rowIndexDescriptor.value = rowIndex;
rowParentDescriptor.value = parent;
const bound = Object.create(this, row);
rowIndexDescriptor.value = null;
rowParentDescriptor.value = null;
return bound as RowLike<T>;
}
public toJSON(): any {
return DataType.isStruct(this.parent.type) ? [...this] :
return DataType.isStruct(this[kParent].type) ? [...this] :
Object.getOwnPropertyNames(this).reduce((props: any, prop: string) => {
return (props[prop] = (this as any)[prop]) && props || props;
}, {});
}
public toString() {
return DataType.isStruct(this.parent.type) ?
return DataType.isStruct(this[kParent].type) ?
[...this].map((x) => valueToString(x)).join(', ') :
Object.getOwnPropertyNames(this).reduce((props: any, prop: string) => {
return (props[prop] = valueToString((this as any)[prop])) && props || props;
}, {});
}
}

/** @ignore */
export class RowProxyGenerator<T extends { [key: string]: DataType }> {
/** @nocollapse */
public static new<T extends { [key: string]: DataType }>(parent: MapVector<T> | StructVector<T>, schemaOrFields: T | Field[], fieldsAreEnumerable = false): RowProxyGenerator<T> {
let schema: T, fields: Field[];
if (Array.isArray(schemaOrFields)) {
fields = schemaOrFields;
} else {
schema = schemaOrFields;
fieldsAreEnumerable = true;
fields = Object.keys(schema).map((x) => new Field(x, schema[x]));
}
return new RowProxyGenerator<T>(parent, fields, fieldsAreEnumerable);
}

private rowPrototype: Row<T>;

private constructor(parent: MapVector<T> | StructVector<T>, fields: Field[], fieldsAreEnumerable: boolean) {
const proto = Object.create(Row.prototype);

rowParentDescriptor.value = parent;
rowLengthDescriptor.value = fields.length;
Object.defineProperty(proto, kParent, rowParentDescriptor);
Object.defineProperty(proto, kLength, rowLengthDescriptor);
fields.forEach((field, columnIndex) => {
if (!proto.hasOwnProperty(field.name)) {
columnDescriptor.enumerable = fieldsAreEnumerable;
columnDescriptor.get || (columnDescriptor.get = this._bindGetter(columnIndex));
Object.defineProperty(proto, field.name, columnDescriptor);
}
if (!proto.hasOwnProperty(columnIndex)) {
columnDescriptor.enumerable = !fieldsAreEnumerable;
columnDescriptor.get || (columnDescriptor.get = this._bindGetter(columnIndex));
Object.defineProperty(proto, columnIndex, columnDescriptor);
}
columnDescriptor.get = null as any;
});

this.rowPrototype = proto;
}

private _bindGetter(columnIndex: number) {
return function(this: Row<T>) {
const child = this[kParent].getChildAt(columnIndex);
return child ? child.get(this[kRowIndex]) : null;
};
}
public bind(rowIndex: number) {
const bound = Object.create(this.rowPrototype);
bound[kRowIndex] = rowIndex;
return bound;
//return new this.RowProxy(rowIndex);
}
}
8 changes: 4 additions & 4 deletions js/src/vector/struct.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

import { Row } from './row';
import { RowProxyGenerator } from './row';
import { Vector } from '../vector';
import { BaseVector } from './base';
import { DataType, Map_, Struct } from '../type';
Expand All @@ -25,8 +25,8 @@ export class StructVector<T extends { [key: string]: DataType } = any> extends B
return Vector.new(this.data.clone(new Map_<T>(this.type.children, keysSorted)));
}
// @ts-ignore
private _rowProxy: Row<T>;
public get rowProxy(): Row<T> {
return this._rowProxy || (this._rowProxy = Row.new<T>(this.type.children || [], false));
private _rowProxy: RowProxyGenerator<T>;
public get rowProxy(): RowProxyGenerator<T> {
return this._rowProxy || (this._rowProxy = RowProxyGenerator.new<T>(this, this.type.children || [], false));
}
}
2 changes: 1 addition & 1 deletion js/src/visitor/get.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ const getNested = <
S extends { [key: string]: DataType },
V extends Vector<Map_<S>> | Vector<Struct<S>>
>(vector: V, index: number): V['TValue'] => {
return vector.rowProxy.bind(vector, index);
return vector.rowProxy.bind(index);
};

/* istanbul ignore next */
Expand Down