From 906e4532dffd8a61d61e560542e76706671d51a8 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 28 Jan 2016 13:29:26 -0800 Subject: [PATCH] using introspection strategy --- README.md | 5 +- config.yaml.sample | 2 +- docs/security.md | 19 +++ package.json | 20 +-- src/common/models/colors/colors.mocha.ts | 7 +- .../models/data-source/data-source.mocha.ts | 4 +- src/common/models/data-source/data-source.ts | 24 ++- .../models/dimension/dimension.mocha.ts | 4 +- src/common/models/essence/essence.mocha.ts | 8 +- .../filter-clause/filter-clause.mocha.ts | 4 +- src/common/models/filter/filter.mocha.ts | 4 +- .../models/highlight/highlight.mocha.ts | 4 +- src/common/models/max-time/max-time.mocha.ts | 4 +- src/common/models/measure/measure.mocha.ts | 156 ++++++++++++++++-- src/common/models/measure/measure.ts | 48 ++++-- .../models/refresh-rule/refresh-rule.mocha.ts | 4 +- src/common/models/sort-on/sort-on.mocha.ts | 4 +- .../split-combine/split-combine.mocha.ts | 4 +- src/common/models/splits/splits.mocha.ts | 4 +- src/common/models/stage/stage.mocha.ts | 4 +- .../models/time-preset/time-preset.mocha.ts | 4 +- src/server/config.ts | 46 +++--- src/server/utils/executor/executor.ts | 10 +- 23 files changed, 285 insertions(+), 108 deletions(-) create mode 100644 docs/security.md diff --git a/README.md b/README.md index d77150b63..0211bc487 100644 --- a/README.md +++ b/README.md @@ -125,18 +125,17 @@ Then you are ready to **Recent improvements:** +- All new introspection code - Better comparison behavior and legend interaction - Support for query time lookups (ex. `$language.lookup('wikipedia-language-lookup')`) - Support for the extract function (ex. `resourceVersion: $resource.extract("(\d+\.\d+\.\d+)")`) -- Custom dimensions (ex. `$user.substr(0,1)`) -- Better legend for time series comparison **We will be working on:** - Additional visualizations (bar chart, geo, heatmap) - Exclusion filters -- Better time selection - String / RegExp filters +- Removing strict limits on dimension values - Better time selection with date picker - Various additions, improvements and fixes to make the app more complete diff --git a/config.yaml.sample b/config.yaml.sample index fa16b3f9d..635fb3b1e 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -4,7 +4,7 @@ # The port on which the Pivot server will listen on port: 9090 -# Run in verbose mode and print the querries sent to the server +# Run in verbose mode and print the queries sent to the server #verbose: true # A Druid broker node that can serve data (only used if you have Druid based data source) diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 000000000..5e7c2d899 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,19 @@ +# Security in Pivot + +Pivot does not currently have a concept of a user. Everyone who has access to a given Pivot server has equal access to that server. +Pivot can act as a 'gatekeeper' for Druid or any supported datasource via the config. + +### Data source level access + +It is possible to restrict which data sources users have access to by explicitly defining in the config all the data sources that you want the users to see and disabling source discovery. +This will prevent any data sources not explicitly defined from being quarried through Pivot. + +### Column level access + +It is possible restrict which columns users have access to by explicitly defining all the dimensions and measures that you want the users to see and disabling introspection. +Any query asking for a column that was not explicitly defined in the dimensions or measures will fail. + +### Row level access + +A Pivot dataSource can define a `subsetFilter` that is a boolean Plywood filter clause that will be silently applied to all queries made to that data source. +For example if you wanted your users to only see the data for "United States" you could add `subsetFilter: $country == "United States"` to the data source definition. diff --git a/package.json b/package.json index 4ce95a8b7..37295dbdd 100644 --- a/package.json +++ b/package.json @@ -29,20 +29,20 @@ "body-parser": "1.14.2", "chronoshift": "0.3.7", "compression": "1.6.1", - "d3": "3.5.13", + "d3": "3.5.14", "debug": "2.2.0", "express": "4.13.4", - "express-handlebars": "2.0.1", + "express-handlebars": "3.0.0", "fs-promise": "0.3.1", "immutable": "3.7.6", - "immutable-class": "0.3.8", + "immutable-class": "0.3.9", "js-yaml": "3.5.2", "lz-string": "1.4.4", "morgan": "1.6.1", "nopt": "3.0.6", "numeral": "1.5.3", - "plywood": "0.8.1", - "plywood-druid-requester": "1.2.5", + "plywood": "0.8.3", + "plywood-druid-requester": "1.2.6", "plywood-mysql-requester": "1.1.5", "q": "1.4.1", "qajax": "1.3.0", @@ -51,18 +51,18 @@ "react-dom": "0.14.6" }, "devDependencies": { - "chai": "3.4.1", - "fs-extra": "0.26.4", + "chai": "3.5.0", + "fs-extra": "0.26.5", "gulp": "3.9.0", "gulp-watch": "4.3.5", - "jsdom": "7.2.2", + "jsdom": "8.0.0", "laborer": "2.4.5", "mkdirp": "0.5.1", - "mocha": "2.3.4", + "mocha": "2.4.4", "react-addons-test-utils": "0.14.6", "replace": "0.3.0", "run-sequence": "1.1.5", - "sinon": "1.17.2", + "sinon": "1.17.3", "underscore": "1.8.3" } } diff --git a/src/common/models/colors/colors.mocha.ts b/src/common/models/colors/colors.mocha.ts index 8b4946b76..8e2c90915 100644 --- a/src/common/models/colors/colors.mocha.ts +++ b/src/common/models/colors/colors.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { Colors } from './colors'; +import { Colors, ColorsJS } from './colors'; describe('Colors', () => { it('is an immutable class', () => { - testImmutableClass(Colors, [ + testImmutableClass(Colors, [ { dimension: 'country', limit: 5 @@ -35,8 +35,7 @@ describe('Colors', () => { }, { dimension: 'country', - values: { '0': 'USA', '1': 'UK', '2': 'India' }, - sameAsLimit: true + values: { '0': 'USA', '1': 'UK', '2': 'India' } } ]); }); diff --git a/src/common/models/data-source/data-source.mocha.ts b/src/common/models/data-source/data-source.mocha.ts index 63ca12610..802c2659b 100644 --- a/src/common/models/data-source/data-source.mocha.ts +++ b/src/common/models/data-source/data-source.mocha.ts @@ -4,12 +4,12 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { DataSource } from './data-source'; +import { DataSource, DataSourceJS } from './data-source'; describe('DataSource', () => { it('is an immutable class', () => { - testImmutableClass(DataSource, [ + testImmutableClass(DataSource, [ { name: 'twitter', title: 'Twitter', diff --git a/src/common/models/data-source/data-source.ts b/src/common/models/data-source/data-source.ts index 0fda6e580..1d6855b5e 100644 --- a/src/common/models/data-source/data-source.ts +++ b/src/common/models/data-source/data-source.ts @@ -432,13 +432,12 @@ export class DataSource implements Instance { case 'STRING': if (attribute.special === 'unique') { if (!autofillMeasures) continue; - expression = $main.countDistinct($(name)); - if (this.getMeasureByExpression(expression)) continue; - measures = measures.push(new Measure({ - name, - expression, - format: Measure.INTEGER_FORMAT - })); + + var newMeasures = Measure.measuresFromAttributeInfo(attribute); + newMeasures.forEach((newMeasure) => { + if (this.getMeasureByExpression(newMeasure.expression)) return; + measures = measures.push(newMeasure); + }); } else { if (!autofillDimensions) continue; expression = $(name); @@ -460,14 +459,13 @@ export class DataSource implements Instance { break; case 'NUMBER': - if (attribute.special === 'histogram') continue; if (!autofillMeasures) continue; - expression = Measure.getExpressionForName(name); - - if (this.getMeasureByExpression(expression)) continue; - var newMeasure = new Measure({ name, expression }); - measures = (name === 'count') ? measures.unshift(newMeasure) : measures.push(newMeasure); + var newMeasures = Measure.measuresFromAttributeInfo(attribute); + newMeasures.forEach((newMeasure) => { + if (this.getMeasureByExpression(newMeasure.expression)) return; + measures = (name === 'count') ? measures.unshift(newMeasure) : measures.push(newMeasure); + }); break; default: diff --git a/src/common/models/dimension/dimension.mocha.ts b/src/common/models/dimension/dimension.mocha.ts index 8ef0f091f..0f22e958d 100644 --- a/src/common/models/dimension/dimension.mocha.ts +++ b/src/common/models/dimension/dimension.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { Dimension } from './dimension'; +import { Dimension, DimensionJS } from './dimension'; describe('Dimension', () => { it('is an immutable class', () => { - testImmutableClass(Dimension, [ + testImmutableClass(Dimension, [ { name: 'country', title: 'important countries', diff --git a/src/common/models/essence/essence.mocha.ts b/src/common/models/essence/essence.mocha.ts index a9f914138..0e688ef9c 100644 --- a/src/common/models/essence/essence.mocha.ts +++ b/src/common/models/essence/essence.mocha.ts @@ -5,11 +5,11 @@ import { testImmutableClass } from 'immutable-class/build/tester'; import { List } from 'immutable'; import { $, Expression } from 'plywood'; -import { Essence } from './essence'; -import { DataSource } from "../data-source/data-source"; +import { Essence, EssenceJS } from './essence'; +import { DataSource, DataSourceJS } from "../data-source/data-source"; describe('Essence', () => { - var dataSourceJS = { + var dataSourceJS: DataSourceJS = { name: 'twitter', title: 'Twitter', engine: 'druid', @@ -71,7 +71,7 @@ describe('Essence', () => { } ]; it('is an immutable class', () => { - testImmutableClass(Essence, [ + testImmutableClass(Essence, [ { dataSource: 'twitter', visualization: 'viz1', diff --git a/src/common/models/filter-clause/filter-clause.mocha.ts b/src/common/models/filter-clause/filter-clause.mocha.ts index 8c11b3258..5c9f342bc 100644 --- a/src/common/models/filter-clause/filter-clause.mocha.ts +++ b/src/common/models/filter-clause/filter-clause.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { FilterClause } from './filter-clause'; +import { FilterClause, FilterClauseJS } from './filter-clause'; describe('FilterClause', () => { it('is an immutable class', () => { - testImmutableClass(FilterClause, [ + testImmutableClass(FilterClause, [ { expression: { op: 'ref', name: 'language' }, values: { diff --git a/src/common/models/filter/filter.mocha.ts b/src/common/models/filter/filter.mocha.ts index 919dd1191..639a58571 100644 --- a/src/common/models/filter/filter.mocha.ts +++ b/src/common/models/filter/filter.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { Filter } from "./filter"; +import { Filter, FilterJS } from "./filter"; describe('Filter', () => { it('is an immutable class', () => { - testImmutableClass(Filter, [ + testImmutableClass(Filter, [ { op: 'literal', value: true }, { "op": "chain", "expression": { "op": "ref", "name": "language" }, diff --git a/src/common/models/highlight/highlight.mocha.ts b/src/common/models/highlight/highlight.mocha.ts index 184e3455c..06a9b951d 100644 --- a/src/common/models/highlight/highlight.mocha.ts +++ b/src/common/models/highlight/highlight.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { Highlight } from './highlight'; +import { Highlight, HighlightJS } from './highlight'; describe('Highlight', () => { it('is an immutable class', () => { - testImmutableClass(Highlight, [ + testImmutableClass(Highlight, [ { owner: 'Sunkist', delta: { diff --git a/src/common/models/max-time/max-time.mocha.ts b/src/common/models/max-time/max-time.mocha.ts index e0d028616..9ffc84cb5 100644 --- a/src/common/models/max-time/max-time.mocha.ts +++ b/src/common/models/max-time/max-time.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { MaxTime } from './max-time'; +import { MaxTime, MaxTimeJS } from './max-time'; describe('MaxTime', () => { it('is an immutable class', () => { - testImmutableClass(MaxTime, [ + testImmutableClass(MaxTime, [ { time: new Date("2015-10-15T19:20:00Z"), updated: new Date("2015-10-15T19:20:13Z") diff --git a/src/common/models/measure/measure.mocha.ts b/src/common/models/measure/measure.mocha.ts index 28c49d1fd..84e507dc0 100644 --- a/src/common/models/measure/measure.mocha.ts +++ b/src/common/models/measure/measure.mocha.ts @@ -3,12 +3,12 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; -import { $, Expression } from 'plywood'; -import { Measure } from './measure'; +import { $, Expression, AttributeInfo } from 'plywood'; +import { Measure, MeasureJS } from './measure'; describe('Measure', () => { it('is an immutable class', () => { - testImmutableClass(Measure, [ + testImmutableClass(Measure, [ { name: 'price', title: 'Price', @@ -22,23 +22,153 @@ describe('Measure', () => { ]); }); - describe('.getExpressionForName', () => { + describe('.measuresFromAttributeInfo', () => { it('works with sum', () => { - var ex1 = Measure.getExpressionForName('price'); - var ex2 = $('main').sum('$price'); - expect(ex1.toJS()).to.deep.equal(ex2.toJS()); + var attribute = AttributeInfo.fromJS({ + "name": "price", + "type": "NUMBER", + "unsplitable": true, + "makerAction": { + "action": "sum", + "expression": { + "name": "price", + "op": "ref" + } + } + }); + var measures = Measure.measuresFromAttributeInfo(attribute).map((m => m.toJS())); + expect(measures).to.deep.equal([ + { + "expression": { + "action": { + "action": "sum", + "expression": { + "name": "price", + "op": "ref" + } + }, + "expression": { + "name": "main", + "op": "ref" + }, + "op": "chain" + }, + "name": "price", + "title": "Price" + } + ]); }); it('works with min', () => { - var ex1 = Measure.getExpressionForName('min_price'); - var ex2 = $('main').min('$min_price'); - expect(ex1.toJS()).to.deep.equal(ex2.toJS()); + var attribute = AttributeInfo.fromJS({ + "name": "price", + "type": "NUMBER", + "unsplitable": true, + "makerAction": { + "action": "min", + "expression": { + "name": "price", + "op": "ref" + } + } + }); + var measures = Measure.measuresFromAttributeInfo(attribute).map((m => m.toJS())); + expect(measures).to.deep.equal([ + { + "expression": { + "action": { + "action": "min", + "expression": { + "name": "price", + "op": "ref" + } + }, + "expression": { + "name": "main", + "op": "ref" + }, + "op": "chain" + }, + "name": "price", + "title": "Price" + } + ]); }); it('works with max', () => { - var ex1 = Measure.getExpressionForName('max_price'); - var ex2 = $('main').max('$max_price'); - expect(ex1.toJS()).to.deep.equal(ex2.toJS()); + var attribute = AttributeInfo.fromJS({ + "name": "price", + "type": "NUMBER", + "unsplitable": true, + "makerAction": { + "action": "max", + "expression": { + "name": "price", + "op": "ref" + } + } + }); + var measures = Measure.measuresFromAttributeInfo(attribute).map((m => m.toJS())); + expect(measures).to.deep.equal([ + { + "expression": { + "action": { + "action": "max", + "expression": { + "name": "price", + "op": "ref" + } + }, + "expression": { + "name": "main", + "op": "ref" + }, + "op": "chain" + }, + "name": "price", + "title": "Price" + } + ]); }); + + it('works with histogram', () => { + var attribute = AttributeInfo.fromJS({ + "name": "delta_hist", + "special": "histogram", + "type": "NUMBER" + }); + var measures = Measure.measuresFromAttributeInfo(attribute).map((m => m.toJS())); + expect(measures).to.deep.equal([]); + }); + + it('works with max', () => { + var attribute = AttributeInfo.fromJS({ + "name": "unique_page", + "special": "unique", + "type": "STRING" + }); + var measures = Measure.measuresFromAttributeInfo(attribute).map((m => m.toJS())); + expect(measures).to.deep.equal([ + { + "expression": { + "action": { + "action": "countDistinct", + "expression": { + "name": "unique_page", + "op": "ref" + } + }, + "expression": { + "name": "main", + "op": "ref" + }, + "op": "chain" + }, + "name": "unique_page", + "title": "Unique Page" + } + ]); + }); + }); }); diff --git a/src/common/models/measure/measure.ts b/src/common/models/measure/measure.ts index c81bf26bc..265eebc69 100644 --- a/src/common/models/measure/measure.ts +++ b/src/common/models/measure/measure.ts @@ -2,7 +2,7 @@ import { Class, Instance, isInstanceOf } from 'immutable-class'; import * as numeral from 'numeral'; -import { $, Expression, ExpressionJS, ApplyAction } from 'plywood'; +import { $, Expression, ExpressionJS, Action, ApplyAction, AttributeInfo } from 'plywood'; import { makeTitle } from '../../utils/general/general'; function formatFnFactory(format: string): (n: number) => string { @@ -35,17 +35,41 @@ export class Measure implements Instance { return isInstanceOf(candidate, Measure); } - static getExpressionForName(name: string): Expression { + static measuresFromAttributeInfo(attribute: AttributeInfo): Measure[] { + var name = attribute.name; var $main = $('main'); var ref = $(name); - var nameWithoutUnderscores = name.replace(/_/g, ' '); - if (/\bmin\b/.test(nameWithoutUnderscores)) { - return $main.min(ref); - } else if (/\bmax\b/.test(nameWithoutUnderscores)) { - return $main.max(ref); - } else { - return $main.sum(ref); + + if (attribute.special) { + if (attribute.special === 'unique') { + return [ + new Measure({ + name, + expression: $main.countDistinct(ref) + }) + ]; + } else { // ToDo: handle: 'histogram' + return []; + } + } + + var expression = $main.sum(ref); + var makerAction = attribute.makerAction; + if (makerAction) { + switch (makerAction.action) { + case 'min': + expression = $main.min(ref); + break; + + case 'max': + expression = $main.max(ref); + break; + + //default: // sum, count + } } + + return [new Measure({ name, expression })]; } static fromJS(parameters: MeasureJS): Measure { @@ -68,7 +92,11 @@ export class Measure implements Instance { var name = parameters.name; this.name = name; this.title = parameters.title || makeTitle(name); - this.expression = parameters.expression || Measure.getExpressionForName(name); + + var expression = parameters.expression; + if (!expression) throw new Error('must have expression'); + this.expression = expression; + var format = parameters.format || Measure.DEFAULT_FORMAT; if (format[0] === '(') throw new Error('can not have format that uses ( )'); this.format = format; diff --git a/src/common/models/refresh-rule/refresh-rule.mocha.ts b/src/common/models/refresh-rule/refresh-rule.mocha.ts index 8b55ac2b8..f7a541fcb 100644 --- a/src/common/models/refresh-rule/refresh-rule.mocha.ts +++ b/src/common/models/refresh-rule/refresh-rule.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { RefreshRule } from './refresh-rule'; +import { RefreshRule, RefreshRuleJS } from './refresh-rule'; describe('RefreshRule', () => { it('is an immutable class', () => { - testImmutableClass(RefreshRule, [ + testImmutableClass(RefreshRule, [ { rule: 'fixed', time: new Date("2015-10-15T19:21:00Z") diff --git a/src/common/models/sort-on/sort-on.mocha.ts b/src/common/models/sort-on/sort-on.mocha.ts index 7f8252d16..e299d245c 100644 --- a/src/common/models/sort-on/sort-on.mocha.ts +++ b/src/common/models/sort-on/sort-on.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { SortOn } from './sort-on'; +import { SortOn, SortOnJS } from './sort-on'; describe('SortOn', () => { it('is an immutable class', () => { - testImmutableClass(SortOn, [ + testImmutableClass(SortOn, [ { measure: { diff --git a/src/common/models/split-combine/split-combine.mocha.ts b/src/common/models/split-combine/split-combine.mocha.ts index 7a79ef153..96dd1bc04 100644 --- a/src/common/models/split-combine/split-combine.mocha.ts +++ b/src/common/models/split-combine/split-combine.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { SplitCombine } from './split-combine'; +import { SplitCombine, SplitCombineJS } from './split-combine'; describe('SplitCombine', () => { it('is an immutable class', () => { - testImmutableClass(SplitCombine, [ + testImmutableClass(SplitCombine, [ { expression: { op: 'ref', name: 'language' } }, diff --git a/src/common/models/splits/splits.mocha.ts b/src/common/models/splits/splits.mocha.ts index 6949fb266..0e816df82 100644 --- a/src/common/models/splits/splits.mocha.ts +++ b/src/common/models/splits/splits.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { Splits } from './splits'; +import { Splits, SplitsJS } from './splits'; describe('Splits', () => { it('is an immutable class', () => { - testImmutableClass(Splits, [ + testImmutableClass(Splits, [ [ { expression: { op: 'ref', name: 'language' } diff --git a/src/common/models/stage/stage.mocha.ts b/src/common/models/stage/stage.mocha.ts index 4a6701eaa..b08c45e2f 100644 --- a/src/common/models/stage/stage.mocha.ts +++ b/src/common/models/stage/stage.mocha.ts @@ -4,11 +4,11 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { Stage } from './stage'; +import { Stage, StageJS } from './stage'; describe('Stage', () => { it('is an immutable class', () => { - testImmutableClass(Stage, [ + testImmutableClass(Stage, [ { x: 10, y: 5, diff --git a/src/common/models/time-preset/time-preset.mocha.ts b/src/common/models/time-preset/time-preset.mocha.ts index 4d65e9f61..b2e13f47a 100644 --- a/src/common/models/time-preset/time-preset.mocha.ts +++ b/src/common/models/time-preset/time-preset.mocha.ts @@ -4,13 +4,13 @@ import { expect } from 'chai'; import { testImmutableClass } from 'immutable-class/build/tester'; import { $, Expression } from 'plywood'; -import { TimePreset } from './time-preset'; +import { TimePreset, TimePresetJS } from './time-preset'; describe('TimePreset', () => { it('is an immutable class', () => { - testImmutableClass(TimePreset, [ + testImmutableClass(TimePreset, [ { name: 'range1', timeRange: { diff --git a/src/server/config.ts b/src/server/config.ts index 71cef47cf..d04530778 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -12,7 +12,7 @@ export interface PivotConfig { verbose?: boolean; druidHost?: string; timeout?: number; - useSegmentMetadata?: string; + introspectionStrategy?: string; sourceListScan?: string; sourceListRefreshInterval?: number; dataSources?: DataSourceJS[]; @@ -39,21 +39,25 @@ Possible usage: pivot --example wiki pivot --druid your.broker.host:8082 - --help Print this help message - --version Display the version number - -v, --verbose Display the DB queries that are being made - -p, --port The port pivot will run on - --example Start pivot with some example data (overrides all other options) - -c, --config The configuration YAML files to use - - --print-config Prints out the auto generated config - --with-comments Adds comments when printing the auto generated config - --data-sources-only Only print the data sources in the auto generated config - - -f, --file Start pivot on top of this file based data source (must be JSON, CSV, or TSV) - - -d, --druid The Druid broker node to connect to - --use-segment-metadata Use the segmentMetadata query for introspection instead of GET /druid/v2/datasources/... + --help Print this help message + --version Display the version number + -v, --verbose Display the DB queries that are being made + -p, --port The port pivot will run on + --example Start pivot with some example data (overrides all other options) + -c, --config The configuration YAML files to use + + --print-config Prints out the auto generated config + --with-comments Adds comments when printing the auto generated config + --data-sources-only Only print the data sources in the auto generated config + + -f, --file Start pivot on top of this file based data source (must be JSON, CSV, or TSV) + + -d, --druid The Druid broker node to connect to + --introspection-strategy Druid introspection strategy + Possible values: + * segment-metadata-fallback - (default) use the segmentMetadata and fallback to GET route + * segment-metadata-only - only use the segmentMetadata query + * datasource-get - only use GET /druid/v2/datasources/DATASOURCE route ` ); } @@ -75,7 +79,7 @@ function parseArgs() { "file": String, "druid": String, - "use-segment-metadata": Boolean + "introspection-strategy": String }, { "v": ["--verbose"], @@ -160,7 +164,7 @@ export const PORT = parseInt(parsedArgs['port'] || config.port, 10); export const DRUID_HOST = parsedArgs['druid'] || config.druidHost; export const TIMEOUT = parseInt(config.timeout, 10) || 30000; -export const USE_SEGMENT_METADATA = Boolean(parsedArgs["use-segment-metadata"] || config.useSegmentMetadata); +export const INTROSPECTION_STRATEGY = String(parsedArgs["introspection-strategy"] || config.introspectionStrategy || 'segment-metadata-fallback'); export const SOURCE_LIST_SCAN = START_SERVER ? config.sourceListScan : 'disable'; export const SOURCE_LIST_REFRESH_INTERVAL = START_SERVER ? (parseInt(config.sourceListRefreshInterval, 10) || 10000) : 0; @@ -205,7 +209,7 @@ var fileDirectory = path.join(__dirname, '../..'); export const DATA_SOURCE_MANAGER: DataSourceManager = dataSourceManagerFactory({ dataSources: DATA_SOURCES, druidRequester, - dataSourceFiller: dataSourceFillerFactory(druidRequester, fileDirectory, TIMEOUT, USE_SEGMENT_METADATA), + dataSourceFiller: dataSourceFillerFactory(druidRequester, fileDirectory, TIMEOUT, INTROSPECTION_STRATEGY), sourceListScan: SOURCE_LIST_SCAN, sourceListRefreshInterval: SOURCE_LIST_REFRESH_INTERVAL, log: PRINT_CONFIG ? null : (line: string) => console.log(line) @@ -246,11 +250,11 @@ if (PRINT_CONFIG) { } } - if (USE_SEGMENT_METADATA) { + if (INTROSPECTION_STRATEGY) { if (withComments) { lines.push("# Use the segmentMetadata query for introspection (only works well with Druid >= 0.8.2)"); } - lines.push(`useSegmentMetadata: true`, ''); + lines.push(`noSegmentMetadata: true`, ''); } lines.push("# Should new datasources automatically be added"); diff --git a/src/server/utils/executor/executor.ts b/src/server/utils/executor/executor.ts index 1aa9f9fc6..05fd03698 100644 --- a/src/server/utils/executor/executor.ts +++ b/src/server/utils/executor/executor.ts @@ -91,7 +91,7 @@ export function getFileData(filePath: string): Q.Promise { }); } -export function externalFactory(dataSource: DataSource, druidRequester: Requester.PlywoodRequester, timeout: number, useSegmentMetadata: boolean): Q.Promise { +export function externalFactory(dataSource: DataSource, druidRequester: Requester.PlywoodRequester, timeout: number, introspectionStrategy: string): Q.Promise { var filter: ExpressionJS = null; if (dataSource.subsetFilter) { filter = dataSource.subsetFilter.toJS(); @@ -115,7 +115,7 @@ export function externalFactory(dataSource: DataSource, druidRequester: Requeste timeAttribute: dataSource.timeAttribute.name, customAggregations: dataSource.options['customAggregations'], attributes: deduceAttributes(dataSource), - useSegmentMetadata, + introspectionStrategy, filter, context, requester: druidRequester @@ -127,7 +127,7 @@ export function externalFactory(dataSource: DataSource, druidRequester: Requeste timeAttribute: dataSource.timeAttribute.name, attributeOverrides: dataSource.options['attributeOverrides'], customAggregations: dataSource.options['customAggregations'], - useSegmentMetadata, + introspectionStrategy, filter, context, requester: druidRequester @@ -153,7 +153,7 @@ export function externalFactory(dataSource: DataSource, druidRequester: Requeste } } -export function dataSourceFillerFactory(druidRequester: Requester.PlywoodRequester, fileDirectory: string, timeout: number, useSegmentMetadata: boolean) { +export function dataSourceFillerFactory(druidRequester: Requester.PlywoodRequester, fileDirectory: string, timeout: number, introspectionStrategy: string) { return function(dataSource: DataSource): Q.Promise { switch (dataSource.engine) { case 'native': @@ -181,7 +181,7 @@ export function dataSourceFillerFactory(druidRequester: Requester.PlywoodRequest }); case 'druid': - return externalFactory(dataSource, druidRequester, timeout, useSegmentMetadata).then((external) => { + return externalFactory(dataSource, druidRequester, timeout, introspectionStrategy).then((external) => { var executor = basicExecutorFactory({ datasets: { main: external } });