diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/fixtures.ts b/superset-frontend/packages/superset-ui-chart-controls/src/fixtures.ts similarity index 98% rename from superset-frontend/packages/superset-ui-chart-controls/test/fixtures.ts rename to superset-frontend/packages/superset-ui-chart-controls/src/fixtures.ts index 92ce4af405779..71c4dd31189ba 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/fixtures.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/fixtures.ts @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -import { Dataset } from '@superset-ui/chart-controls'; import { DatasourceType } from '@superset-ui/core'; +import { Dataset } from './types'; export const TestDataset: Dataset = { column_format: {}, diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/index.ts b/superset-frontend/packages/superset-ui-chart-controls/src/index.ts index 6aac9b50043d4..962d9ac0ab670 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/index.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/index.ts @@ -38,3 +38,4 @@ export * from './shared-controls/emitFilterControl'; export * from './shared-controls/components'; export * from './types'; export * from './shared-controls/mixins'; +export * from './fixtures'; diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/sections/sections.tsx b/superset-frontend/packages/superset-ui-chart-controls/src/sections/sections.tsx index 6214e2ba9c345..4535f1996dd72 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/sections/sections.tsx +++ b/superset-frontend/packages/superset-ui-chart-controls/src/sections/sections.tsx @@ -38,19 +38,23 @@ export const legacyTimeseriesTime: ControlPanelSectionConfig = { ], }; -export const genericTime: ControlPanelSectionConfig = { - ...baseTimeSection, - controlSetRows: [ - ['granularity_sqla'], - [hasGenericChartAxes ? null : 'time_grain_sqla'], - ['time_range'], - ], -}; +export const genericTime: ControlPanelSectionConfig = hasGenericChartAxes + ? { controlSetRows: [] } + : { + ...baseTimeSection, + controlSetRows: [ + ['granularity_sqla'], + ['time_grain_sqla'], + ['time_range'], + ], + }; -export const legacyRegularTime: ControlPanelSectionConfig = { - ...baseTimeSection, - controlSetRows: [['granularity_sqla'], ['time_range']], -}; +export const legacyRegularTime: ControlPanelSectionConfig = hasGenericChartAxes + ? { controlSetRows: [] } + : { + ...baseTimeSection, + controlSetRows: [['granularity_sqla'], ['time_range']], + }; export const datasourceAndVizType: ControlPanelSectionConfig = { label: t('Datasource & Chart Type'), diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/dndControls.tsx b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/dndControls.tsx index cf1cee2c169f0..840c5c3611a89 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/dndControls.tsx +++ b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/dndControls.tsx @@ -40,8 +40,9 @@ import { ColumnMeta, FilterOption, temporalColumnMixin, + datePickerInAdhocFilterMixin, + xAxisMixin, } from '..'; -import { xAxisMixin } from './mixins'; type Control = { savedMetrics?: Metric[] | null; @@ -149,6 +150,7 @@ export const dndAdhocFilterControl: SharedControlConfig< datasource, }), provideFormDataToProps: true, + ...datePickerInAdhocFilterMixin, }; export const dndAdhocMetricsControl: SharedControlConfig< diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/mixins.tsx b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/mixins.tsx index c268104d3d1f6..0b19b96c7f6cf 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/mixins.tsx +++ b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/mixins.tsx @@ -17,7 +17,9 @@ * under the License. */ import { + ensureIsArray, hasGenericChartAxes, + NO_TIME_RANGE, QueryFormData, t, validateNonEmpty, @@ -63,3 +65,57 @@ export const temporalColumnMixin: Pick = { }; }, }; + +export const datePickerInAdhocFilterMixin: Pick< + BaseControlConfig, + 'initialValue' +> = { + initialValue: (control: ControlState, state: ControlPanelState | null) => { + // skip initialValue if + // 1) GENERIC_CHART_AXES is disabled + // 2) there was a time filter in adhoc filters + if ( + !hasGenericChartAxes || + ensureIsArray(control.value).findIndex( + (flt: any) => flt?.operator === 'TEMPORAL_RANGE', + ) > -1 + ) { + return undefined; + } + + // should migrate original granularity_sqla and time_range into adhoc filter + // 1) granularity_sqla and time_range are existed + if (state?.form_data?.granularity_sqla && state?.form_data?.time_range) { + return [ + ...ensureIsArray(control.value), + { + clause: 'WHERE', + subject: state.form_data.granularity_sqla, + operator: 'TEMPORAL_RANGE', + comparator: state.form_data.time_range, + expressionType: 'SIMPLE', + }, + ]; + } + + // should apply the default time filter into adhoc filter + // 1) temporal column is existed in current datasource + const temporalColumn = + state?.datasource && + getTemporalColumns(state.datasource).defaultTemporalColumn; + if (hasGenericChartAxes && temporalColumn) { + return [ + ...ensureIsArray(control.value), + { + clause: 'WHERE', + subject: temporalColumn, + operator: 'TEMPORAL_RANGE', + comparator: state?.common?.conf?.DEFAULT_TIME_FILTER || NO_TIME_RANGE, + expressionType: 'SIMPLE', + }, + ]; + } + + return undefined; + }, +}; diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/types.ts b/superset-frontend/packages/superset-ui-chart-controls/src/types.ts index 8c9ce71874e69..fcb19e702d85f 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/types.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/types.ts @@ -22,9 +22,9 @@ import type { AdhocColumn, Column, DatasourceType, + JsonObject, JsonValue, Metric, - QueryColumn, QueryFormColumn, QueryFormData, QueryFormMetric, @@ -80,12 +80,15 @@ export interface Dataset { description: string | null; uid?: string; owners?: Owner[]; + filter_select?: boolean; + filter_select_enabled?: boolean; } export interface ControlPanelState { form_data: QueryFormData; datasource: Dataset | QueryResponse | null; controls: ControlStateMapping; + common: JsonObject; } /** @@ -449,9 +452,7 @@ export type ColorFormatters = { export default {}; -export function isColumnMeta( - column: AdhocColumn | ColumnMeta | QueryColumn, -): column is ColumnMeta { +export function isColumnMeta(column: AnyDict): column is ColumnMeta { return !!column && 'column_name' in column; } diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/utils/getTemporalColumns.ts b/superset-frontend/packages/superset-ui-chart-controls/src/utils/getTemporalColumns.ts index caae8dfd52d0c..718308d241a6e 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/utils/getTemporalColumns.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/utils/getTemporalColumns.ts @@ -29,9 +29,9 @@ import { isQueryResponse, } from '@superset-ui/chart-controls'; -export const getTemporalColumns = ( +export function getTemporalColumns( datasource: ValueOf>, -) => { +) { const rv: { temporalColumns: ColumnMeta[] | QueryColumn[]; defaultTemporalColumn: string | null | undefined; @@ -61,4 +61,17 @@ export const getTemporalColumns = ( } return rv; -}; +} + +export function isTemporalColumn( + columnName: string, + datasource: ValueOf>, +): boolean { + const columns = getTemporalColumns(datasource).temporalColumns; + for (let i = 0; i < columns.length; i += 1) { + if (columns[i].column_name === columnName) { + return true; + } + } + return false; +} diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/utils/index.ts b/superset-frontend/packages/superset-ui-chart-controls/src/utils/index.ts index ff0b8db638187..4fa4243c1e850 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/utils/index.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/utils/index.ts @@ -24,4 +24,4 @@ export { default as mainMetric } from './mainMetric'; export { default as columnChoices } from './columnChoices'; export * from './defineSavedMetrics'; export * from './getStandardizedControls'; -export { getTemporalColumns } from './getTemporalColumns'; +export * from './getTemporalColumns'; diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/getTemporalColumns.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/utils/getTemporalColumns.test.ts index 8b94752c89aeb..1921540ea6046 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/getTemporalColumns.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/getTemporalColumns.test.ts @@ -17,8 +17,12 @@ * under the License. */ import { testQueryResponse, testQueryResults } from '@superset-ui/core'; -import { Dataset, getTemporalColumns } from '../../src'; -import { TestDataset } from '../fixtures'; +import { + Dataset, + getTemporalColumns, + isTemporalColumn, + TestDataset, +} from '../../src'; test('get temporal columns from a Dataset', () => { expect(getTemporalColumns(TestDataset)).toEqual({ @@ -93,3 +97,8 @@ test('should accept empty Dataset or queryResponse', () => { defaultTemporalColumn: undefined, }); }); + +test('should determine temporal columns in a Dataset', () => { + expect(isTemporalColumn('ds', TestDataset)).toBeTruthy(); + expect(isTemporalColumn('num', TestDataset)).toBeFalsy(); +}); diff --git a/superset-frontend/packages/superset-ui-core/src/query/buildQueryContext.ts b/superset-frontend/packages/superset-ui-core/src/query/buildQueryContext.ts index c5b2b3780b9aa..e66c1e201e868 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/buildQueryContext.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/buildQueryContext.ts @@ -20,11 +20,16 @@ import buildQueryObject from './buildQueryObject'; import DatasourceKey from './DatasourceKey'; import { QueryFieldAliases, QueryFormData } from './types/QueryFormData'; -import { QueryContext, QueryObject } from './types/Query'; +import { + BinaryQueryObjectFilterClause, + QueryContext, + QueryObject, +} from './types/Query'; import { SetDataMaskHook } from '../chart'; import { JsonObject } from '../connection'; import { normalizeTimeColumn } from './normalizeTimeColumn'; -import { isXAxisSet } from './getXAxis'; +import { hasGenericChartAxes, isXAxisSet } from './getXAxis'; +import { ensureIsArray } from '../utils'; const WRAP_IN_ARRAY = (baseQueryObject: QueryObject) => [baseQueryObject]; @@ -48,15 +53,26 @@ export default function buildQueryContext( ? { buildQuery: options, queryFields: {} } : options || {}; let queries = buildQuery(buildQueryObject(formData, queryFields)); + // --- query mutator begin --- + // todo(Yongjie): move the query mutator into buildQueryObject instead of buildQueryContext queries.forEach(query => { if (Array.isArray(query.post_processing)) { // eslint-disable-next-line no-param-reassign query.post_processing = query.post_processing.filter(Boolean); } + if (hasGenericChartAxes && query.time_range) { + // eslint-disable-next-line no-param-reassign + query.filters = ensureIsArray(query.filters).map(flt => + flt?.op === 'TEMPORAL_RANGE' + ? ({ ...flt, val: query.time_range } as BinaryQueryObjectFilterClause) + : flt, + ); + } }); if (isXAxisSet(formData)) { queries = queries.map(query => normalizeTimeColumn(formData, query)); } + // --- query mutator end --- return { datasource: new DatasourceKey(formData.datasource).toObject(), force: formData.force || false, diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/Operator.ts b/superset-frontend/packages/superset-ui-core/src/query/types/Operator.ts index 754766bef262d..10385767614e6 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/types/Operator.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/types/Operator.ts @@ -31,6 +31,7 @@ const BINARY_OPERATORS = [ 'ILIKE', 'LIKE', 'REGEX', + 'TEMPORAL_RANGE', ] as const; /** List of operators that require another operand that is a set */ diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/Query.ts b/superset-frontend/packages/superset-ui-core/src/query/types/Query.ts index a2d6df39b89a6..420420eaedc00 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/types/Query.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/types/Query.ts @@ -258,6 +258,7 @@ export const CtasEnum = { export type QueryColumn = { name: string; + column_name?: string; type: string | null; is_dttm: boolean; }; diff --git a/superset-frontend/packages/superset-ui-core/test/query/buildQueryContext.test.ts b/superset-frontend/packages/superset-ui-core/test/query/buildQueryContext.test.ts index 4a9e71a6dd89e..e14648a984f96 100644 --- a/superset-frontend/packages/superset-ui-core/test/query/buildQueryContext.test.ts +++ b/superset-frontend/packages/superset-ui-core/test/query/buildQueryContext.test.ts @@ -18,6 +18,7 @@ */ import { buildQueryContext } from '@superset-ui/core'; import * as queryModule from '../../src/query/normalizeTimeColumn'; +import * as getXAxisModule from '../../src/query/getXAxis'; describe('buildQueryContext', () => { it('should build datasource for table sources and apply defaults', () => { @@ -98,6 +99,7 @@ describe('buildQueryContext', () => { ]), ); }); + // todo(Yongjie): move these test case into buildQueryObject.test.ts it('should remove undefined value in post_processing', () => { const queryContext = buildQueryContext( { @@ -124,12 +126,9 @@ describe('buildQueryContext', () => { ]); }); it('should call normalizeTimeColumn if GENERIC_CHART_AXES is enabled and has x_axis', () => { - // @ts-ignore - const spy = jest.spyOn(window, 'window', 'get').mockImplementation(() => ({ - featureFlags: { - GENERIC_CHART_AXES: true, - }, - })); + Object.defineProperty(getXAxisModule, 'hasGenericChartAxes', { + value: true, + }); const spyNormalizeTimeColumn = jest.spyOn( queryModule, 'normalizeTimeColumn', @@ -144,16 +143,12 @@ describe('buildQueryContext', () => { () => [{}], ); expect(spyNormalizeTimeColumn).toBeCalled(); - spy.mockRestore(); spyNormalizeTimeColumn.mockRestore(); }); it("shouldn't call normalizeTimeColumn if GENERIC_CHART_AXES is disabled", () => { - // @ts-ignore - const spy = jest.spyOn(window, 'window', 'get').mockImplementation(() => ({ - featureFlags: { - GENERIC_CHART_AXES: false, - }, - })); + Object.defineProperty(getXAxisModule, 'hasGenericChartAxes', { + value: false, + }); const spyNormalizeTimeColumn = jest.spyOn( queryModule, 'normalizeTimeColumn', @@ -167,7 +162,43 @@ describe('buildQueryContext', () => { () => [{}], ); expect(spyNormalizeTimeColumn).not.toBeCalled(); - spy.mockRestore(); spyNormalizeTimeColumn.mockRestore(); }); + it('should orverride time filter if GENERIC_CHART_AXES is enabled', () => { + Object.defineProperty(getXAxisModule, 'hasGenericChartAxes', { + value: true, + }); + + const queryContext = buildQueryContext( + { + datasource: '5__table', + viz_type: 'table', + }, + () => [ + { + filters: [ + { + col: 'col1', + op: 'TEMPORAL_RANGE', + val: '2001 : 2002', + }, + { + col: 'col2', + op: 'IN', + val: ['a', 'b'], + }, + ], + time_range: '1990 : 1991', + }, + ], + ); + expect(queryContext.queries[0].filters).toEqual([ + { col: 'col1', op: 'TEMPORAL_RANGE', val: '1990 : 1991' }, + { + col: 'col2', + op: 'IN', + val: ['a', 'b'], + }, + ]); + }); }); diff --git a/superset-frontend/packages/superset-ui-core/test/query/getAxis.test.ts b/superset-frontend/packages/superset-ui-core/test/query/getAxis.test.ts index 6db1c150eaf33..010bd9fc67591 100644 --- a/superset-frontend/packages/superset-ui-core/test/query/getAxis.test.ts +++ b/superset-frontend/packages/superset-ui-core/test/query/getAxis.test.ts @@ -18,54 +18,9 @@ */ import { isXAxisSet } from '@superset-ui/core'; -describe('GENERIC_CHART_AXES is enabled', () => { - let windowSpy: any; - - beforeAll(() => { - // @ts-ignore - windowSpy = jest.spyOn(window, 'window', 'get').mockImplementation(() => ({ - featureFlags: { - GENERIC_CHART_AXES: true, - }, - })); - }); - - afterAll(() => { - windowSpy.mockRestore(); - }); - - it('isEnabledAxies when FF is disabled', () => { - expect( - isXAxisSet({ datasource: '123', viz_type: 'table' }), - ).not.toBeTruthy(); - expect( - isXAxisSet({ datasource: '123', viz_type: 'table', x_axis: 'axis' }), - ).toBeTruthy(); - }); -}); - -describe('GENERIC_CHART_AXES is disabled', () => { - let windowSpy: any; - - beforeAll(() => { - // @ts-ignore - windowSpy = jest.spyOn(window, 'window', 'get').mockImplementation(() => ({ - featureFlags: { - GENERIC_CHART_AXES: false, - }, - })); - }); - - afterAll(() => { - windowSpy.mockRestore(); - }); - - it('isEnabledAxies when FF is disabled', () => { - expect( - isXAxisSet({ datasource: '123', viz_type: 'table' }), - ).not.toBeTruthy(); - expect( - isXAxisSet({ datasource: '123', viz_type: 'table', x_axis: 'axis' }), - ).toBeTruthy(); - }); +test('isXAxisSet', () => { + expect(isXAxisSet({ datasource: '123', viz_type: 'table' })).not.toBeTruthy(); + expect( + isXAxisSet({ datasource: '123', viz_type: 'table', x_axis: 'axis' }), + ).toBeTruthy(); }); diff --git a/superset-frontend/plugins/plugin-chart-handlebars/src/plugin/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-handlebars/src/plugin/controlPanel.tsx index 38615455c5eef..e34394b3043a2 100644 --- a/superset-frontend/plugins/plugin-chart-handlebars/src/plugin/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-handlebars/src/plugin/controlPanel.tsx @@ -49,7 +49,7 @@ import { styleControlSetItem } from './controls/style'; const config: ControlPanelConfig = { controlPanelSections: [ - sections.legacyTimeseriesTime, + sections.genericTime, { label: t('Query'), expanded: true, diff --git a/superset-frontend/src/components/Modal/Modal.tsx b/superset-frontend/src/components/Modal/Modal.tsx index 2ccd8dcf5dabd..608e8c5592414 100644 --- a/superset-frontend/src/components/Modal/Modal.tsx +++ b/superset-frontend/src/components/Modal/Modal.tsx @@ -57,6 +57,7 @@ export interface ModalProps { draggableConfig?: DraggableProps; destroyOnClose?: boolean; maskClosable?: boolean; + zIndex?: number; } interface StyledModalProps { diff --git a/superset-frontend/src/explore/actions/hydrateExplore.test.ts b/superset-frontend/src/explore/actions/hydrateExplore.test.ts index 353a811396d47..7f363adef6fd6 100644 --- a/superset-frontend/src/explore/actions/hydrateExplore.test.ts +++ b/superset-frontend/src/explore/actions/hydrateExplore.test.ts @@ -86,6 +86,7 @@ test('creates hydrate action from initial data', () => { standalone: null, force: null, saveAction: null, + common: {}, }, }, }), @@ -159,6 +160,7 @@ test('creates hydrate action with existing state', () => { standalone: null, force: null, saveAction: null, + common: {}, }, }, }), diff --git a/superset-frontend/src/explore/actions/hydrateExplore.ts b/superset-frontend/src/explore/actions/hydrateExplore.ts index 1f4f510b5e4e1..f9adc2fe056f7 100644 --- a/superset-frontend/src/explore/actions/hydrateExplore.ts +++ b/superset-frontend/src/explore/actions/hydrateExplore.ts @@ -136,6 +136,7 @@ export const hydrateExplore = force: getUrlParam(URL_PARAMS.force), metadata, saveAction, + common, }; // apply initial mapStateToProps for all controls, must execute AFTER diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/DateFilterLabel.tsx b/superset-frontend/src/explore/components/controls/DateFilterControl/DateFilterLabel.tsx index 4b84ccf20423b..1eca21ed1fe9b 100644 --- a/superset-frontend/src/explore/components/controls/DateFilterControl/DateFilterLabel.tsx +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/DateFilterLabel.tsx @@ -21,6 +21,7 @@ import { css, styled, t, useTheme, NO_TIME_RANGE } from '@superset-ui/core'; import Button from 'src/components/Button'; import ControlHeader from 'src/explore/components/ControlHeader'; import Label, { Type } from 'src/components/Label'; +import Modal from 'src/components/Modal'; import { Divider } from 'src/components'; import Icons from 'src/components/Icons'; import Select from 'src/components/Select/Select'; @@ -32,10 +33,12 @@ import ControlPopover from '../ControlPopover/ControlPopover'; import { DateFilterControlProps, FrameType } from './types'; import { + DATE_FILTER_TEST_KEY, fetchTimeRange, FRAME_OPTIONS, getDateFilterControlTestId, guessFrame, + useDefaultTimeFilter, } from './utils'; import { CommonFrame, @@ -44,7 +47,6 @@ import { AdvancedFrame, } from './components'; -const StyledPopover = styled(ControlPopover)``; const StyledRangeType = styled(Select)` width: 272px; `; @@ -121,12 +123,15 @@ const IconWrapper = styled.span` export default function DateFilterLabel(props: DateFilterControlProps) { const { - value = NO_TIME_RANGE, onChange, type, onOpenPopover = noOp, onClosePopover = noOp, + overlayStyle = 'Popover', } = props; + const defaultTimeFilter = useDefaultTimeFilter(); + + const value = props.value ?? defaultTimeFilter; const [actualTimeRange, setActualTimeRange] = useState(value); const [show, setShow] = useState(false); @@ -137,6 +142,7 @@ export default function DateFilterLabel(props: DateFilterControlProps) { const [validTimeRange, setValidTimeRange] = useState(false); const [evalResponse, setEvalResponse] = useState(value); const [tooltipTitle, setTooltipTitle] = useState(value); + const theme = useTheme(); useEffect(() => { if (value === NO_TIME_RANGE) { @@ -180,6 +186,7 @@ export default function DateFilterLabel(props: DateFilterControlProps) { setValidTimeRange(true); } setLastFetchedTimeRange(value); + setEvalResponse(actualRange || value); }); }, [value]); @@ -225,7 +232,7 @@ export default function DateFilterLabel(props: DateFilterControlProps) { setShow(false); } - const togglePopover = () => { + const toggleOverlay = () => { if (show) { onHide(); onClosePopover(); @@ -242,8 +249,6 @@ export default function DateFilterLabel(props: DateFilterControlProps) { setFrame(value); } - const theme = useTheme(); - const overlayContent = (
{t('RANGE TYPE')}
@@ -266,7 +271,9 @@ export default function DateFilterLabel(props: DateFilterControlProps) { {frame === 'Custom' && ( )} - {frame === 'No filter' &&
} + {frame === 'No filter' && ( +
+ )}
{t('Actual time range')}
@@ -285,7 +292,7 @@ export default function DateFilterLabel(props: DateFilterControlProps) { cta key="cancel" onClick={onHide} - data-test="cancel-button" + data-test={DATE_FILTER_TEST_KEY.cancelButton} > {t('CANCEL')} @@ -310,25 +317,56 @@ export default function DateFilterLabel(props: DateFilterControlProps) { ); - return ( + const popoverContent = ( + + + + + + ); + + const modalContent = ( <> - - + + + - - - - + {overlayContent} + + + ); + + return ( + <> + + {overlayStyle === 'Modal' ? modalContent : popoverContent} ); } diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/components/CommonFrame.tsx b/superset-frontend/src/explore/components/controls/DateFilterControl/components/CommonFrame.tsx index 4b76ab648a157..d3de2b9991b19 100644 --- a/superset-frontend/src/explore/components/controls/DateFilterControl/components/CommonFrame.tsx +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/components/CommonFrame.tsx @@ -22,6 +22,7 @@ import { Radio } from 'src/components/Radio'; import { COMMON_RANGE_OPTIONS, COMMON_RANGE_SET, + DATE_FILTER_TEST_KEY, } from 'src/explore/components/controls/DateFilterControl/utils'; import { CommonRangeType, @@ -38,7 +39,12 @@ export function CommonFrame(props: FrameComponentProps) { return ( <> -
{t('Configure Time Range: Last...')}
+
+ {t('Configure Time Range: Last...')} +
props.onChange(e.target.value)} diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/index.ts b/superset-frontend/src/explore/components/controls/DateFilterControl/index.ts index 199ecdc559d71..801b13937adc2 100644 --- a/superset-frontend/src/explore/components/controls/DateFilterControl/index.ts +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/index.ts @@ -21,4 +21,5 @@ export { DATE_FILTER_CONTROL_TEST_ID, fetchTimeRange, guessFrame, + DATE_FILTER_TEST_KEY, } from './utils'; diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/components/AdvancedFrame.test.tsx b/superset-frontend/src/explore/components/controls/DateFilterControl/tests/AdvancedFrame.test.tsx similarity index 97% rename from superset-frontend/src/explore/components/controls/DateFilterControl/components/AdvancedFrame.test.tsx rename to superset-frontend/src/explore/components/controls/DateFilterControl/tests/AdvancedFrame.test.tsx index f7f13add422a1..327f020b7c563 100644 --- a/superset-frontend/src/explore/components/controls/DateFilterControl/components/AdvancedFrame.test.tsx +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/tests/AdvancedFrame.test.tsx @@ -19,7 +19,7 @@ import React from 'react'; import { render, screen } from 'spec/helpers/testing-library'; import userEvent from '@testing-library/user-event'; -import { AdvancedFrame } from '.'; +import { AdvancedFrame } from '../components'; test('renders with default props', () => { render(); diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/components/CustomFrame.test.tsx b/superset-frontend/src/explore/components/controls/DateFilterControl/tests/CustomFrame.test.tsx similarity index 99% rename from superset-frontend/src/explore/components/controls/DateFilterControl/components/CustomFrame.test.tsx rename to superset-frontend/src/explore/components/controls/DateFilterControl/tests/CustomFrame.test.tsx index 4ba151afe0c5e..83cad48bdb439 100644 --- a/superset-frontend/src/explore/components/controls/DateFilterControl/components/CustomFrame.test.tsx +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/tests/CustomFrame.test.tsx @@ -22,7 +22,7 @@ import { Provider } from 'react-redux'; import configureStore from 'redux-mock-store'; import { render, screen } from 'spec/helpers/testing-library'; import userEvent from '@testing-library/user-event'; -import { CustomFrame } from '.'; +import { CustomFrame } from '../components'; jest.useFakeTimers(); diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/tests/DateFilterLabel.test.tsx b/superset-frontend/src/explore/components/controls/DateFilterControl/tests/DateFilterLabel.test.tsx new file mode 100644 index 0000000000000..89533439fd4e9 --- /dev/null +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/tests/DateFilterLabel.test.tsx @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import thunk from 'redux-thunk'; +import { Provider } from 'react-redux'; +import configureStore from 'redux-mock-store'; + +import { render, screen } from 'spec/helpers/testing-library'; +import userEvent from '@testing-library/user-event'; + +import { NO_TIME_RANGE } from '@superset-ui/core'; +import DateFilterLabel from '..'; +import { DateFilterControlProps } from '../types'; +import { DATE_FILTER_TEST_KEY } from '../utils'; + +const mockStore = configureStore([thunk]); + +function setup( + props: Omit, + store: any = mockStore({}), +) { + return ( + + + + ); +} + +test('DateFilter with default props', () => { + render(setup({ onChange: () => {} })); + // label + expect(screen.getByText(NO_TIME_RANGE)).toBeInTheDocument(); + + // should be popover by default + userEvent.click(screen.getByText(NO_TIME_RANGE)); + expect( + screen.getByTestId(DATE_FILTER_TEST_KEY.popoverOverlay), + ).toBeInTheDocument(); +}); + +test('DateFilter shoule be applied the overlayStyle props', () => { + render(setup({ onChange: () => {}, overlayStyle: 'Modal' })); + // should be Modal as overlay + userEvent.click(screen.getByText(NO_TIME_RANGE)); + expect( + screen.getByTestId(DATE_FILTER_TEST_KEY.modalOverlay), + ).toBeInTheDocument(); +}); + +test('DateFilter shoule be applied the global config time_filter from the store', () => { + render( + setup( + { onChange: () => {} }, + mockStore({ + common: { conf: { DEFAULT_TIME_FILTER: 'Last week' } }, + }), + ), + ); + // the label should be 'Last week' + expect(screen.getByText('Last week')).toBeInTheDocument(); + + userEvent.click(screen.getByText('Last week')); + expect( + screen.getByTestId(DATE_FILTER_TEST_KEY.commonFrame), + ).toBeInTheDocument(); +}); diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/utils/utils.test.ts b/superset-frontend/src/explore/components/controls/DateFilterControl/tests/utils.test.ts similarity index 100% rename from superset-frontend/src/explore/components/controls/DateFilterControl/utils/utils.test.ts rename to superset-frontend/src/explore/components/controls/DateFilterControl/tests/utils.test.ts diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/types.ts b/superset-frontend/src/explore/components/controls/DateFilterControl/types.ts index b7c69ea0cbab5..c14204919e95c 100644 --- a/superset-frontend/src/explore/components/controls/DateFilterControl/types.ts +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/types.ts @@ -99,4 +99,5 @@ export interface DateFilterControlProps { type?: Type; onOpenPopover?: () => void; onClosePopover?: () => void; + overlayStyle?: 'Modal' | 'Popover'; } diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/utils/constants.ts b/superset-frontend/src/explore/components/controls/DateFilterControl/utils/constants.ts index 1424ab918d0cd..e6ec228cb2ab3 100644 --- a/superset-frontend/src/explore/components/controls/DateFilterControl/utils/constants.ts +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/utils/constants.ts @@ -137,3 +137,11 @@ export const DATE_FILTER_CONTROL_TEST_ID = 'date-filter-control'; export const getDateFilterControlTestId = testWithId( DATE_FILTER_CONTROL_TEST_ID, ); + +export enum DATE_FILTER_TEST_KEY { + commonFrame = 'common-frame', + modalOverlay = 'modal-overlay', + popoverOverlay = 'time-range-trigger', + noFilter = 'no-filter', + cancelButton = 'cancel-button', +} diff --git a/superset-frontend/src/explore/components/controls/DateFilterControl/utils/dateFilterUtils.ts b/superset-frontend/src/explore/components/controls/DateFilterControl/utils/dateFilterUtils.ts index 05317d48e0f4f..08162cedf02d0 100644 --- a/superset-frontend/src/explore/components/controls/DateFilterControl/utils/dateFilterUtils.ts +++ b/superset-frontend/src/explore/components/controls/DateFilterControl/utils/dateFilterUtils.ts @@ -17,8 +17,9 @@ * under the License. */ import rison from 'rison'; -import { SupersetClient, NO_TIME_RANGE } from '@superset-ui/core'; +import { SupersetClient, NO_TIME_RANGE, JsonObject } from '@superset-ui/core'; import { getClientErrorObject } from 'src/utils/getClientErrorObject'; +import { useSelector } from 'react-redux'; import { COMMON_RANGE_VALUES_SET, CALENDAR_RANGE_VALUES_SET, @@ -84,3 +85,11 @@ export const fetchTimeRange = async ( }; } }; + +export function useDefaultTimeFilter() { + return ( + useSelector( + (state: JsonObject) => state?.common?.conf?.DEFAULT_TIME_FILTER, + ) ?? NO_TIME_RANGE + ); +} diff --git a/superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndAdhocFilterOption.tsx b/superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndAdhocFilterOption.tsx index bc65ef5e8a7c6..2b9b72b459a1e 100644 --- a/superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndAdhocFilterOption.tsx +++ b/superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndAdhocFilterOption.tsx @@ -22,6 +22,7 @@ import { DndItemType } from 'src/explore/components/DndItemType'; import AdhocFilterPopoverTrigger from 'src/explore/components/controls/FilterControl/AdhocFilterPopoverTrigger'; import AdhocFilter from 'src/explore/components/controls/FilterControl/AdhocFilter'; import { OptionSortType } from 'src/explore/types'; +import { useGetTimeRangeLabel } from 'src/explore/components/controls/FilterControl/utils'; import OptionWrapper from './OptionWrapper'; export interface DndAdhocFilterOptionProps { @@ -45,6 +46,8 @@ export default function DndAdhocFilterOption({ partitionColumn, index, }: DndAdhocFilterOptionProps) { + const { actualTimeRange, title } = useGetTimeRangeLabel(adhocFilter); + return ( { window.featureFlags = {}; }); +const mockStore = configureStore([thunk]); +const store = mockStore({}); + +function setup({ + value = undefined, + formData = baseFormData, + columns = [], +}: { + value?: AdhocFilter; + formData?: QueryFormData; + columns?: ColumnMeta[]; +} = {}) { + return ( + + + + ); +} + test('renders with default props', async () => { - render(, { useDnd: true }); + render(setup(), { useDnd: true }); expect( await screen.findByText('Drop columns or metrics here'), ).toBeInTheDocument(); @@ -68,7 +103,7 @@ test('renders with value', async () => { sqlExpression: 'COUNT(*)', expressionType: EXPRESSION_TYPES.SQL, }); - render(, { + render(setup({ value }), { useDnd: true, }); expect(await screen.findByText('COUNT(*)')).toBeInTheDocument(); @@ -76,14 +111,13 @@ test('renders with value', async () => { test('renders options with saved metric', async () => { render( - , + }, + }), { useDnd: true, }, @@ -95,17 +129,16 @@ test('renders options with saved metric', async () => { test('renders options with column', async () => { render( - , + ], + }), { useDnd: true, }, @@ -121,14 +154,13 @@ test('renders options with adhoc metric', async () => { metric_name: 'avg__num', }); render( - , + }, + }), { useDnd: true, }, diff --git a/superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndFilterSelect.tsx b/superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndFilterSelect.tsx index 5d023faedb3f1..8c4d6f8e831b6 100644 --- a/superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndFilterSelect.tsx +++ b/superset-frontend/src/explore/components/controls/DndColumnSelectControl/DndFilterSelect.tsx @@ -19,6 +19,7 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { FeatureFlag, + hasGenericChartAxes, isFeatureEnabled, logging, Metric, @@ -27,7 +28,12 @@ import { SupersetClient, t, } from '@superset-ui/core'; -import { ColumnMeta, withDndFallback } from '@superset-ui/chart-controls'; +import { + ColumnMeta, + isColumnMeta, + isTemporalColumn, + withDndFallback, +} from '@superset-ui/chart-controls'; import { OPERATOR_ENUM_TO_OPERATOR_TYPE, Operators, @@ -50,6 +56,7 @@ import { DndItemType } from 'src/explore/components/DndItemType'; import { ControlComponentProps } from 'src/explore/components/Control'; import AdhocFilterControl from '../FilterControl/AdhocFilterControl'; import DndAdhocFilterOption from './DndAdhocFilterOption'; +import { useDefaultTimeFilter } from '../DateFilterControl/utils'; const EMPTY_OBJECT = {}; const DND_ACCEPTED_TYPES = [ @@ -324,6 +331,7 @@ const DndFilterSelect = (props: DndFilterSelectProps) => { togglePopover(true); }, [togglePopover]); + const defaultTimeFilter = useDefaultTimeFilter(); const adhocFilter = useMemo(() => { if (isSavedMetric(droppedItem)) { return new AdhocFilter({ @@ -346,6 +354,15 @@ const DndFilterSelect = (props: DndFilterSelectProps) => { config.operator = OPERATOR_ENUM_TO_OPERATOR_TYPE[Operators.IN].operation; config.operatorId = Operators.IN; } + if ( + hasGenericChartAxes && + isColumnMeta(droppedItem) && + isTemporalColumn(droppedItem?.column_name, props.datasource) + ) { + config.operator = Operators.TEMPORAL_RANGE; + config.operatorId = Operators.TEMPORAL_RANGE; + config.comparator = defaultTimeFilter; + } return new AdhocFilter(config); }, [droppedItem]); diff --git a/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilter/index.js b/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilter/index.js index 29486709b14c7..3b54a7dc1a7aa 100644 --- a/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilter/index.js +++ b/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilter/index.js @@ -62,7 +62,9 @@ function translateToSql(adhocMetric, { useSimple } = {}) { const { subject, comparator } = adhocMetric; const operator = adhocMetric.operator && - CUSTOM_OPERATIONS.indexOf(adhocMetric.operator) >= 0 + // 'LATEST PARTITION' supported callback only + adhocMetric.operator === + OPERATOR_ENUM_TO_OPERATOR_TYPE[Operators.LATEST_PARTITION].operation ? OPERATORS_TO_SQL[adhocMetric.operator](adhocMetric) : OPERATORS_TO_SQL[adhocMetric.operator]; return getSimpleSQLExpression(subject, operator, comparator); diff --git a/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent/AdhocFilterEditPopoverSimpleTabContent.test.tsx b/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent/AdhocFilterEditPopoverSimpleTabContent.test.tsx index 93ace13d95754..9d1b68bc6f75d 100644 --- a/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent/AdhocFilterEditPopoverSimpleTabContent.test.tsx +++ b/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent/AdhocFilterEditPopoverSimpleTabContent.test.tsx @@ -18,8 +18,12 @@ */ /* eslint-disable no-unused-expressions */ import React from 'react'; +import * as redux from 'react-redux'; import sinon from 'sinon'; import { shallow } from 'enzyme'; +import thunk from 'redux-thunk'; +import { Provider } from 'react-redux'; +import configureStore from 'redux-mock-store'; import AdhocFilter, { EXPRESSION_TYPES, @@ -37,6 +41,7 @@ import * as featureFlags from 'src/featureFlags'; import userEvent from '@testing-library/user-event'; import fetchMock from 'fetch-mock'; +import { TestDataset } from '@superset-ui/chart-controls'; import AdhocFilterEditPopoverSimpleTabContent, { useSimpleTabFilterProps, Props, @@ -99,10 +104,11 @@ const getAdvancedDataTypeTestProps = (overrides?: Record) => { onChange, options: [{ type: 'DOUBLE', column_name: 'advancedDataType', id: 5 }], datasource: { - id: 'test-id', - columns: [], - type: 'postgres', - filter_select: false, + ...TestDataset, + ...{ + columns: [], + filter_select: false, + }, }, partitionColumn: 'test', ...overrides, @@ -114,15 +120,18 @@ const getAdvancedDataTypeTestProps = (overrides?: Record) => { function setup(overrides?: Record) { const onChange = sinon.spy(); const validHandler = sinon.spy(); + const spy = jest.spyOn(redux, 'useSelector'); + spy.mockReturnValue({}); const props = { adhocFilter: simpleAdhocFilter, onChange, options, datasource: { - id: 'test-id', - columns: [], - type: 'postgres', - filter_select: false, + ...TestDataset, + ...{ + columns: [], + filter_select: false, + }, }, partitionColumn: 'test', ...overrides, @@ -372,14 +381,19 @@ fetchMock.get(ADVANCED_DATA_TYPE_ENDPOINT_INVALID, { values: [], }, }); +const mockStore = configureStore([thunk]); +const store = mockStore({}); describe('AdhocFilterEditPopoverSimpleTabContent Advanced data Type Test', () => { const setupFilter = async (props: Props) => { await act(async () => { render( - - - , + + + + + , + , ); }); }; diff --git a/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent/index.tsx b/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent/index.tsx index 225efd5201b05..c15123ef3498b 100644 --- a/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent/index.tsx +++ b/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterEditPopoverSimpleTabContent/index.tsx @@ -19,7 +19,14 @@ import React, { useEffect, useState } from 'react'; import FormItem from 'src/components/Form/FormItem'; import { Select } from 'src/components'; -import { t, SupersetClient, SupersetTheme, styled } from '@superset-ui/core'; +import { + t, + SupersetClient, + SupersetTheme, + styled, + hasGenericChartAxes, + isDefined, +} from '@superset-ui/core'; import { Operators, OPERATORS_OPTIONS, @@ -39,8 +46,14 @@ import { Tooltip } from 'src/components/Tooltip'; import { Input } from 'src/components/Input'; import { optionLabel } from 'src/utils/common'; import { FeatureFlag, isFeatureEnabled } from 'src/featureFlags'; -import { ColumnMeta } from '@superset-ui/chart-controls'; +import { + ColumnMeta, + Dataset, + isTemporalColumn, +} from '@superset-ui/chart-controls'; import useAdvancedDataTypes from './useAdvancedDataTypes'; +import { useDatePickerInAdhocFilter } from '../utils'; +import { useDefaultTimeFilter } from '../../DateFilterControl/utils'; const StyledInput = styled(Input)` margin-bottom: ${({ theme }) => theme.gridUnit * 4}px; @@ -88,12 +101,7 @@ export interface Props { adhocFilter: AdhocFilter; onChange: (filter: AdhocFilter) => void; options: ColumnType[]; - datasource: { - id: string; - columns: ColumnMeta[]; - type: string; - filter_select: boolean; - }; + datasource: Dataset; partitionColumn: string; operators?: Operators[]; validHandler: (isValid: boolean) => void; @@ -106,6 +114,8 @@ export interface AdvancedDataTypesState { } export const useSimpleTabFilterProps = (props: Props) => { + const defaultTimeFilter = useDefaultTimeFilter(); + const isOperatorRelevant = (operator: Operators, subject: string) => { const column = props.datasource.columns?.find( col => col.column_name === subject, @@ -116,10 +126,14 @@ export const useSimpleTabFilterProps = (props: Props) => { !!column && (column.type === 'INT' || column.type === 'INTEGER'); const isColumnFunction = !!column && !!column.expression; - if (operator && CUSTOM_OPERATORS.has(operator)) { + if (operator && operator === Operators.LATEST_PARTITION) { const { partitionColumn } = props; return partitionColumn && subject && subject === partitionColumn; } + if (operator && operator === Operators.TEMPORAL_RANGE) { + // hide the TEMPORAL_RANGE operator + return false; + } if (operator === Operators.IS_TRUE || operator === Operators.IS_FALSE) { return isColumnBoolean || isColumnNumber || isColumnFunction; } @@ -152,17 +166,33 @@ export const useSimpleTabFilterProps = (props: Props) => { subject = option.label; clause = CLAUSES.HAVING; } - const { operator, operatorId } = props.adhocFilter; + let { operator, operatorId, comparator } = props.adhocFilter; + operator = + operator && operatorId && isOperatorRelevant(operatorId, subject) + ? OPERATOR_ENUM_TO_OPERATOR_TYPE[operatorId].operation + : null; + if (!isDefined(operator)) { + // if operator is `null`, use the `IN` and reset the comparator. + operator = Operators.IN; + operatorId = Operators.IN; + comparator = undefined; + } + + if (hasGenericChartAxes && isTemporalColumn(id, props.datasource)) { + subject = id; + operator = Operators.TEMPORAL_RANGE; + operatorId = Operators.TEMPORAL_RANGE; + comparator = defaultTimeFilter; + } + props.onChange( props.adhocFilter.duplicateWith({ subject, clause, - operator: - operator && operatorId && isOperatorRelevant(operatorId, subject) - ? OPERATOR_ENUM_TO_OPERATOR_TYPE[operatorId].operation - : null, + operator, expressionType: EXPRESSION_TYPES.SIMPLE, operatorId, + comparator, }), ); }; @@ -221,12 +251,23 @@ export const useSimpleTabFilterProps = (props: Props) => { }), ); }; + const onDatePickerChange = (columnName: string, timeRange: string) => { + props.onChange( + props.adhocFilter.duplicateWith({ + subject: columnName, + operator: Operators.TEMPORAL_RANGE, + comparator: timeRange, + expressionType: EXPRESSION_TYPES.SIMPLE, + }), + ); + }; return { onSubjectChange, onOperatorChange, onComparatorChange, isOperatorRelevant, clearOperator, + onDatePickerChange, }; }; @@ -236,6 +277,7 @@ const AdhocFilterEditPopoverSimpleTabContent: React.FC = props => { onOperatorChange, isOperatorRelevant, onComparatorChange, + onDatePickerChange, } = useSimpleTabFilterProps(props); const [suggestions, setSuggestions] = useState< Record<'label' | 'value', any>[] @@ -343,6 +385,16 @@ const AdhocFilterEditPopoverSimpleTabContent: React.FC = props => { const labelText = comparator && comparator.length > 0 && createSuggestionsPlaceholder(); + const datePicker = useDatePickerInAdhocFilter({ + columnName: props.adhocFilter.subject, + timeRange: + props.adhocFilter.operator === Operators.TEMPORAL_RANGE + ? props.adhocFilter.comparator + : undefined, + datasource: props.datasource, + onChange: onDatePickerChange, + }); + useEffect(() => { const refreshComparatorSuggestions = () => { const { datasource } = props; @@ -375,7 +427,9 @@ const AdhocFilterEditPopoverSimpleTabContent: React.FC = props => { }); } }; - refreshComparatorSuggestions(); + if (!datePicker) { + refreshComparatorSuggestions(); + } }, [props.adhocFilter.subject]); useEffect(() => { @@ -481,7 +535,7 @@ const AdhocFilterEditPopoverSimpleTabContent: React.FC = props => { return ( <> {subjectComponent} - {operatorsAndOperandComponent} + {datePicker ?? operatorsAndOperandComponent} ); }; diff --git a/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterOption/index.tsx b/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterOption/index.tsx index f1f1c902b0171..a8bffe9dd5cf6 100644 --- a/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterOption/index.tsx +++ b/superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterOption/index.tsx @@ -23,6 +23,7 @@ import AdhocFilterPopoverTrigger from 'src/explore/components/controls/FilterCon import AdhocFilter from 'src/explore/components/controls/FilterControl/AdhocFilter'; import { OptionSortType } from 'src/explore/types'; import { Operators } from 'src/explore/constants'; +import { useGetTimeRangeLabel } from '../utils'; export interface AdhocFilterOptionProps { adhocFilter: AdhocFilter; @@ -51,6 +52,8 @@ export default function AdhocFilterOption({ sections, operators, }: AdhocFilterOptionProps) { + const { actualTimeRange, title } = useGetTimeRangeLabel(adhocFilter); + return ( void; +} + +export const useDatePickerInAdhocFilter = ({ + columnName, + timeRange, + datasource, + onChange, +}: DatePickerInFilterProps): React.ReactElement | undefined => { + const onTimeRangeChange = (val: string) => onChange(columnName, val); + + return hasGenericChartAxes && isTemporalColumn(columnName, datasource) ? ( + <> + + + + ) : undefined; +}; diff --git a/superset-frontend/src/explore/components/controls/FilterControl/utils/useDatePickerInAdhocfilter.test.ts b/superset-frontend/src/explore/components/controls/FilterControl/utils/useDatePickerInAdhocfilter.test.ts new file mode 100644 index 0000000000000..fdafba2d989c5 --- /dev/null +++ b/superset-frontend/src/explore/components/controls/FilterControl/utils/useDatePickerInAdhocfilter.test.ts @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { renderHook } from '@testing-library/react-hooks'; +import { TestDataset } from '@superset-ui/chart-controls'; +import * as supersetCoreModule from '@superset-ui/core'; +import { useDatePickerInAdhocFilter } from './useDatePickerInAdhocFilter'; + +test('should return undefined if Generic Axis is disabled', () => { + Object.defineProperty(supersetCoreModule, 'hasGenericChartAxes', { + value: false, + }); + const { result } = renderHook(() => + useDatePickerInAdhocFilter({ + columnName: 'ds', + datasource: TestDataset, + onChange: jest.fn(), + }), + ); + expect(result.current).toBeUndefined(); +}); + +test('should return undefined if column is not temporal', () => { + Object.defineProperty(supersetCoreModule, 'hasGenericChartAxes', { + value: true, + }); + const { result } = renderHook(() => + useDatePickerInAdhocFilter({ + columnName: 'gender', + datasource: TestDataset, + onChange: jest.fn(), + }), + ); + expect(result.current).toBeUndefined(); +}); + +test('should return JSX', () => { + Object.defineProperty(supersetCoreModule, 'hasGenericChartAxes', { + value: true, + }); + const { result } = renderHook(() => + useDatePickerInAdhocFilter({ + columnName: 'ds', + datasource: TestDataset, + onChange: jest.fn(), + }), + ); + expect(result.current).not.toBeUndefined(); +}); diff --git a/superset-frontend/src/explore/components/controls/FilterControl/utils/useGetTimeRangeLabel.test.ts b/superset-frontend/src/explore/components/controls/FilterControl/utils/useGetTimeRangeLabel.test.ts new file mode 100644 index 0000000000000..0d39ef8a27041 --- /dev/null +++ b/superset-frontend/src/explore/components/controls/FilterControl/utils/useGetTimeRangeLabel.test.ts @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { renderHook } from '@testing-library/react-hooks'; +import { NO_TIME_RANGE } from '@superset-ui/core'; +import { Operators } from 'src/explore/constants'; +import * as FetchTimeRangeModule from 'src/explore/components/controls/DateFilterControl'; +import { useGetTimeRangeLabel } from './useGetTimeRangeLabel'; +import AdhocFilter, { CLAUSES, EXPRESSION_TYPES } from '../AdhocFilter'; + +test('should return empty object if operator is not TEMPORAL_RANGE', () => { + const adhocFilter = new AdhocFilter({ + expressionType: EXPRESSION_TYPES.SIMPLE, + subject: 'value', + operator: '>', + comparator: '10', + clause: CLAUSES.WHERE, + }); + const { result } = renderHook(() => useGetTimeRangeLabel(adhocFilter)); + expect(result.current).toEqual({}); +}); + +test('should return empty object if expressionType is SQL', () => { + const adhocFilter = new AdhocFilter({ + expressionType: EXPRESSION_TYPES.SQL, + subject: 'temporal column', + operator: Operators.TEMPORAL_RANGE, + comparator: 'Last week', + clause: CLAUSES.WHERE, + }); + const { result } = renderHook(() => useGetTimeRangeLabel(adhocFilter)); + expect(result.current).toEqual({}); +}); + +test('should get "No filter" label', () => { + const adhocFilter = new AdhocFilter({ + expressionType: EXPRESSION_TYPES.SIMPLE, + subject: 'temporal column', + operator: Operators.TEMPORAL_RANGE, + comparator: NO_TIME_RANGE, + clause: CLAUSES.WHERE, + }); + const { result } = renderHook(() => useGetTimeRangeLabel(adhocFilter)); + expect(result.current).toEqual({ + actualTimeRange: 'temporal column (No filter)', + title: 'No filter', + }); +}); + +test('should get actualTimeRange and title', async () => { + jest + .spyOn(FetchTimeRangeModule, 'fetchTimeRange') + .mockResolvedValue({ value: 'MOCK TIME' }); + + const adhocFilter = new AdhocFilter({ + expressionType: EXPRESSION_TYPES.SIMPLE, + subject: 'temporal column', + operator: Operators.TEMPORAL_RANGE, + comparator: 'Last week', + clause: CLAUSES.WHERE, + }); + + const { result } = await renderHook(() => useGetTimeRangeLabel(adhocFilter)); + expect(result.current).toEqual({ + actualTimeRange: 'MOCK TIME', + title: 'Last week', + }); +}); + +test('should get actualTimeRange and title when gets an error', async () => { + jest + .spyOn(FetchTimeRangeModule, 'fetchTimeRange') + .mockResolvedValue({ error: 'MOCK ERROR' }); + + const adhocFilter = new AdhocFilter({ + expressionType: EXPRESSION_TYPES.SIMPLE, + subject: 'temporal column', + operator: Operators.TEMPORAL_RANGE, + comparator: 'Last week', + clause: CLAUSES.WHERE, + }); + + const { result } = await renderHook(() => useGetTimeRangeLabel(adhocFilter)); + expect(result.current).toEqual({ + actualTimeRange: 'temporal column (Last week)', + title: 'MOCK ERROR', + }); +}); diff --git a/superset-frontend/src/explore/components/controls/FilterControl/utils/useGetTimeRangeLabel.tsx b/superset-frontend/src/explore/components/controls/FilterControl/utils/useGetTimeRangeLabel.tsx new file mode 100644 index 0000000000000..abc2ad5b27c91 --- /dev/null +++ b/superset-frontend/src/explore/components/controls/FilterControl/utils/useGetTimeRangeLabel.tsx @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useEffect, useState } from 'react'; +import { NO_TIME_RANGE } from '@superset-ui/core'; +import { fetchTimeRange } from 'src/explore/components/controls/DateFilterControl'; +import { Operators } from 'src/explore/constants'; +import AdhocFilter, { EXPRESSION_TYPES } from '../AdhocFilter'; + +interface Results { + actualTimeRange?: string; + title?: string; +} + +export const useGetTimeRangeLabel = (adhocFilter: AdhocFilter): Results => { + const [actualTimeRange, setActualTimeRange] = useState({}); + + useEffect(() => { + if ( + adhocFilter.operator !== Operators.TEMPORAL_RANGE || + adhocFilter.expressionType !== EXPRESSION_TYPES.SIMPLE + ) { + setActualTimeRange({}); + } + if ( + adhocFilter.operator === Operators.TEMPORAL_RANGE && + adhocFilter.comparator === NO_TIME_RANGE + ) { + setActualTimeRange({ + actualTimeRange: `${adhocFilter.subject} (${NO_TIME_RANGE})`, + title: NO_TIME_RANGE, + }); + } + + if ( + adhocFilter.operator === Operators.TEMPORAL_RANGE && + adhocFilter.expressionType === EXPRESSION_TYPES.SIMPLE && + adhocFilter.comparator !== NO_TIME_RANGE && + actualTimeRange.title !== adhocFilter.comparator + ) { + fetchTimeRange(adhocFilter.comparator, adhocFilter.subject).then( + ({ value, error }) => { + if (error) { + setActualTimeRange({ + actualTimeRange: `${adhocFilter.subject} (${adhocFilter.comparator})`, + title: error, + }); + } else { + setActualTimeRange({ + actualTimeRange: value ?? '', + title: adhocFilter.comparator, + }); + } + }, + ); + } + }, [adhocFilter]); + + return actualTimeRange; +}; diff --git a/superset-frontend/src/explore/constants.ts b/superset-frontend/src/explore/constants.ts index 56b8ca055fcf9..f45dbf2043594 100644 --- a/superset-frontend/src/explore/constants.ts +++ b/superset-frontend/src/explore/constants.ts @@ -45,6 +45,7 @@ export enum Operators { LATEST_PARTITION = 'LATEST_PARTITION', IS_TRUE = 'IS_TRUE', IS_FALSE = 'IS_FALSE', + TEMPORAL_RANGE = 'TEMPORAL_RANGE', } export interface OperatorType { @@ -80,6 +81,10 @@ export const OPERATOR_ENUM_TO_OPERATOR_TYPE: { }, [Operators.IS_TRUE]: { display: 'Is true', operation: '==' }, [Operators.IS_FALSE]: { display: 'Is false', operation: '==' }, + [Operators.TEMPORAL_RANGE]: { + display: 'TEMPORAL_RANGE', + operation: 'TEMPORAL_RANGE', + }, }; export const OPERATORS_OPTIONS = Object.values(Operators) as Operators[]; @@ -96,7 +101,10 @@ export const HAVING_OPERATORS = [ export const MULTI_OPERATORS = new Set([Operators.IN, Operators.NOT_IN]); // CUSTOM_OPERATORS will show operator in simple mode, // but will generate customized sqlExpression -export const CUSTOM_OPERATORS = new Set([Operators.LATEST_PARTITION]); +export const CUSTOM_OPERATORS = new Set([ + Operators.LATEST_PARTITION, + Operators.TEMPORAL_RANGE, +]); // DISABLE_INPUT_OPERATORS will disable filter value input // in adhocFilter control export const DISABLE_INPUT_OPERATORS = [ diff --git a/superset-frontend/src/explore/controlUtils/controlUtils.test.tsx b/superset-frontend/src/explore/controlUtils/controlUtils.test.tsx index 1b9cf1ea55c73..5feefc3481aa1 100644 --- a/superset-frontend/src/explore/controlUtils/controlUtils.test.tsx +++ b/superset-frontend/src/explore/controlUtils/controlUtils.test.tsx @@ -59,6 +59,7 @@ describe('controlUtils', () => { }, controls: {}, form_data: { datasource: '1__table', viz_type: 'table' }, + common: {}, }; beforeAll(() => { diff --git a/superset-frontend/src/explore/types.ts b/superset-frontend/src/explore/types.ts index 50205ce0a5b56..85240c919d3b9 100644 --- a/superset-frontend/src/explore/types.ts +++ b/superset-frontend/src/explore/types.ts @@ -112,6 +112,7 @@ export interface ExplorePageState { controlsTransferred: string[]; standalone: boolean; force: boolean; + common: JsonObject; }; sliceEntities?: JsonObject; // propagated from Dashboard view } diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py index 0a7ad4bada7b3..17cc5fd4119d9 100644 --- a/superset/common/query_context_processor.py +++ b/superset/common/query_context_processor.py @@ -35,6 +35,7 @@ from superset.common.query_actions import get_query_results from superset.common.utils import dataframe_utils from superset.common.utils.query_cache_manager import QueryCacheManager +from superset.common.utils.time_range_utils import get_since_until_from_query_object from superset.connectors.base.models import BaseDatasource from superset.constants import CacheRegion from superset.exceptions import ( @@ -56,6 +57,7 @@ get_column_names_from_columns, get_column_names_from_metrics, get_metric_names, + get_xaxis_label, normalize_dttm_col, TIME_COMPARISON, ) @@ -314,15 +316,36 @@ def processing_time_offsets( # pylint: disable=too-many-locals,too-many-stateme rv_dfs: List[pd.DataFrame] = [df] time_offsets = query_object.time_offsets - outer_from_dttm = query_object.from_dttm - outer_to_dttm = query_object.to_dttm + outer_from_dttm, outer_to_dttm = get_since_until_from_query_object(query_object) + if not outer_from_dttm or not outer_to_dttm: + raise QueryObjectValidationError( + _( + "An enclosed time range (both start and end) must be specified " + "when using a Time Comparison." + ) + ) for offset in time_offsets: try: + # pylint: disable=line-too-long + # Since the xaxis is also a column name for the time filter, xaxis_label will be set as granularity + # these query object are equivalent: + # 1) { granularity: 'dttm_col', time_range: '2020 : 2021', time_offsets: ['1 year ago']} + # 2) { columns: [ + # {label: 'dttm_col', sqlExpression: 'dttm_col', "columnType": "BASE_AXIS" } + # ], + # time_offsets: ['1 year ago'], + # filters: [{col: 'dttm_col', op: 'TEMPORAL_RANGE', val: '2020 : 2021'}], + # } query_object_clone.from_dttm = get_past_or_future( offset, outer_from_dttm, ) query_object_clone.to_dttm = get_past_or_future(offset, outer_to_dttm) + + xaxis_label = get_xaxis_label(query_object.columns) + query_object_clone.granularity = ( + query_object_clone.granularity or xaxis_label + ) except ValueError as ex: raise QueryObjectValidationError(str(ex)) from ex # make sure subquery use main query where clause @@ -330,14 +353,12 @@ def processing_time_offsets( # pylint: disable=too-many-locals,too-many-stateme query_object_clone.inner_to_dttm = outer_to_dttm query_object_clone.time_offsets = [] query_object_clone.post_processing = [] + query_object_clone.filter = [ + flt + for flt in query_object_clone.filter + if flt.get("col") != xaxis_label + ] - if not query_object.from_dttm or not query_object.to_dttm: - raise QueryObjectValidationError( - _( - "An enclosed time range (both start and end) must be specified " - "when using a Time Comparison." - ) - ) # `offset` is added to the hash function cache_key = self.query_cache_key(query_object_clone, time_offset=offset) cache = QueryCacheManager.get( diff --git a/superset/common/query_object.py b/superset/common/query_object.py index ac86273b27098..94cf2a74ccaa9 100644 --- a/superset/common/query_object.py +++ b/superset/common/query_object.py @@ -19,7 +19,7 @@ import json import logging -from datetime import datetime, timedelta +from datetime import datetime from pprint import pformat from typing import Any, Dict, List, NamedTuple, Optional, TYPE_CHECKING @@ -46,7 +46,6 @@ json_int_dttm_ser, QueryObjectFilterClause, ) -from superset.utils.date_parser import parse_human_timedelta from superset.utils.hashing import md5_sha_from_dict if TYPE_CHECKING: @@ -106,7 +105,7 @@ class QueryObject: # pylint: disable=too-many-instance-attributes series_limit: int series_limit_metric: Optional[Metric] time_offsets: List[str] - time_shift: Optional[timedelta] + time_shift: Optional[str] time_range: Optional[str] to_dttm: Optional[datetime] @@ -156,7 +155,7 @@ def __init__( # pylint: disable=too-many-locals self.series_limit = series_limit self.series_limit_metric = series_limit_metric self.time_range = time_range - self.time_shift = parse_human_timedelta(time_shift) + self.time_shift = time_shift self.from_dttm = kwargs.get("from_dttm") self.to_dttm = kwargs.get("to_dttm") self.result_type = kwargs.get("result_type") @@ -336,6 +335,7 @@ def to_dict(self) -> Dict[str, Any]: "series_limit": self.series_limit, "series_limit_metric": self.series_limit_metric, "to_dttm": self.to_dttm, + "time_shift": self.time_shift, } return query_object_dict diff --git a/superset/common/query_object_factory.py b/superset/common/query_object_factory.py index e9f5122975b52..88cc7ca1b461b 100644 --- a/superset/common/query_object_factory.py +++ b/superset/common/query_object_factory.py @@ -16,13 +16,12 @@ # under the License. from __future__ import annotations -from datetime import datetime -from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING +from typing import Any, Dict, Optional, TYPE_CHECKING from superset.common.chart_data import ChartDataResultType from superset.common.query_object import QueryObject +from superset.common.utils.time_range_utils import get_since_until_from_time_range from superset.utils.core import apply_max_row_limit, DatasourceDict, DatasourceType -from superset.utils.date_parser import get_since_until if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker @@ -62,7 +61,9 @@ def create( # pylint: disable=too-many-arguments processed_extras = self._process_extras(extras) result_type = kwargs.setdefault("result_type", parent_result_type) row_limit = self._process_row_limit(row_limit, result_type) - from_dttm, to_dttm = self._get_dttms(time_range, time_shift, processed_extras) + from_dttm, to_dttm = get_since_until_from_time_range( + time_range, time_shift, processed_extras + ) kwargs["from_dttm"] = from_dttm kwargs["to_dttm"] = to_dttm return QueryObject( @@ -98,23 +99,6 @@ def _process_row_limit( ) return apply_max_row_limit(row_limit or default_row_limit) - def _get_dttms( - self, - time_range: Optional[str], - time_shift: Optional[str], - extras: Dict[str, Any], - ) -> Tuple[Optional[datetime], Optional[datetime]]: - return get_since_until( - relative_start=extras.get( - "relative_start", self._config["DEFAULT_RELATIVE_START_TIME"] - ), - relative_end=extras.get( - "relative_end", self._config["DEFAULT_RELATIVE_END_TIME"] - ), - time_range=time_range, - time_shift=time_shift, - ) - # light version of the view.utils.core # import view.utils require application context # Todo: move it and the view.utils.core to utils package diff --git a/superset/common/utils/time_range_utils.py b/superset/common/utils/time_range_utils.py new file mode 100644 index 0000000000000..fa6a5244b244b --- /dev/null +++ b/superset/common/utils/time_range_utils.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime +from typing import Any, cast, Dict, Optional, Tuple + +from superset import app +from superset.common.query_object import QueryObject +from superset.utils.core import FilterOperator, get_xaxis_label +from superset.utils.date_parser import get_since_until + + +def get_since_until_from_time_range( + time_range: Optional[str] = None, + time_shift: Optional[str] = None, + extras: Optional[Dict[str, Any]] = None, +) -> Tuple[Optional[datetime], Optional[datetime]]: + return get_since_until( + relative_start=(extras or {}).get( + "relative_start", app.config["DEFAULT_RELATIVE_START_TIME"] + ), + relative_end=(extras or {}).get( + "relative_end", app.config["DEFAULT_RELATIVE_END_TIME"] + ), + time_range=time_range, + time_shift=time_shift, + ) + + +# pylint: disable=invalid-name +def get_since_until_from_query_object( + query_object: QueryObject, +) -> Tuple[Optional[datetime], Optional[datetime]]: + """ + this function will return since and until by tuple if + 1) the time_range is in the query object. + 2) the xaxis column is in the columns field + and its corresponding `temporal_range` filter is in the adhoc filters. + :param query_object: a valid query object + :return: since and until by tuple + """ + if query_object.time_range: + return get_since_until_from_time_range( + time_range=query_object.time_range, + time_shift=query_object.time_shift, + extras=query_object.extras, + ) + + time_range = None + for flt in query_object.filter: + if ( + flt.get("op") == FilterOperator.TEMPORAL_RANGE.value + and flt.get("col") == get_xaxis_label(query_object.columns) + and isinstance(flt.get("val"), str) + ): + time_range = cast(str, flt.get("val")) + + return get_since_until_from_time_range( + time_range=time_range, + time_shift=query_object.time_shift, + extras=query_object.extras, + ) diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index 4855dd1af3cca..89c640b7a6eab 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -80,6 +80,7 @@ from superset.advanced_data_type.types import AdvancedDataTypeResponse from superset.columns.models import Column as NewColumn, UNKOWN_TYPE from superset.common.db_query_status import QueryStatus +from superset.common.utils.time_range_utils import get_since_until_from_time_range from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric from superset.connectors.sqla.utils import ( find_cached_objects_in_session, @@ -334,10 +335,11 @@ def datasource(self) -> RelationshipProperty: def get_time_filter( self, - start_dttm: DateTime, - end_dttm: DateTime, + start_dttm: Optional[DateTime] = None, + end_dttm: Optional[DateTime] = None, + label: Optional[str] = "__time", ) -> ColumnElement: - col = self.get_sqla_col(label="__time") + col = self.get_sqla_col(label=label) l = [] if start_dttm: l.append(col >= self.table.text(self.dttm_sql_literal(start_dttm))) @@ -1267,6 +1269,7 @@ def get_sqla_query( # pylint: disable=too-many-arguments,too-many-locals,too-ma row_offset: Optional[int] = None, timeseries_limit: Optional[int] = None, timeseries_limit_metric: Optional[Metric] = None, + time_shift: Optional[str] = None, ) -> SqlaQuery: """Querying any sqla table from this common interface""" if granularity not in self.dttm_cols and granularity is not None: @@ -1656,6 +1659,23 @@ def get_sqla_query( # pylint: disable=too-many-arguments,too-many-locals,too-ma where_clause_and.append(sqla_col.like(eq)) elif op == utils.FilterOperator.ILIKE.value: where_clause_and.append(sqla_col.ilike(eq)) + elif ( + op == utils.FilterOperator.TEMPORAL_RANGE.value + and isinstance(eq, str) + and col_obj is not None + ): + _since, _until = get_since_until_from_time_range( + time_range=eq, + time_shift=time_shift, + extras=extras, + ) + where_clause_and.append( + col_obj.get_time_filter( + start_dttm=_since, + end_dttm=_until, + label=sqla_col.key, + ) + ) else: raise QueryObjectValidationError( _("Invalid filter operation type: %(op)s", op=op) diff --git a/superset/constants.py b/superset/constants.py index dbd34767b705f..7d759acf6741c 100644 --- a/superset/constants.py +++ b/superset/constants.py @@ -32,6 +32,8 @@ PASSWORD_MASK = "X" * 10 +NO_TIME_RANGE = "No filter" + class RouteMethod: # pylint: disable=too-few-public-methods """ diff --git a/superset/db_engine_specs/sqlite.py b/superset/db_engine_specs/sqlite.py index 8c583060d293b..85442aa877363 100644 --- a/superset/db_engine_specs/sqlite.py +++ b/superset/db_engine_specs/sqlite.py @@ -77,7 +77,11 @@ def convert_dttm( cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() - if tt in (utils.TemporalType.TEXT, utils.TemporalType.DATETIME): + if tt in ( + utils.TemporalType.TEXT, + utils.TemporalType.DATETIME, + utils.TemporalType.TIMESTAMP, + ): return f"""'{dttm.isoformat(sep=" ", timespec="seconds")}'""" return None diff --git a/superset/models/helpers.py b/superset/models/helpers.py index da526b559c7f1..cb314de80275c 100644 --- a/superset/models/helpers.py +++ b/superset/models/helpers.py @@ -1322,7 +1322,7 @@ def get_sqla_col(self, col: Dict[str, Any]) -> Column: col = sa.column(label, type_=col_type) return self.make_sqla_column_compatible(col, label) - def get_sqla_query( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements + def get_sqla_query( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements,unused-argument self, apply_fetch_values_predicate: bool = False, columns: Optional[List[Column]] = None, @@ -1348,6 +1348,7 @@ def get_sqla_query( # pylint: disable=too-many-arguments,too-many-locals,too-ma row_offset: Optional[int] = None, timeseries_limit: Optional[int] = None, timeseries_limit_metric: Optional[Metric] = None, + time_shift: Optional[str] = None, ) -> SqlaQuery: """Querying any sqla table from this common interface""" if granularity not in self.dttm_cols and granularity is not None: diff --git a/superset/utils/core.py b/superset/utils/core.py index 4a088be5dd649..c1c310747ca97 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -96,6 +96,7 @@ EXTRA_FORM_DATA_APPEND_KEYS, EXTRA_FORM_DATA_OVERRIDE_EXTRA_KEYS, EXTRA_FORM_DATA_OVERRIDE_REGULAR_MAPPINGS, + NO_TIME_RANGE, ) from superset.errors import ErrorLevel, SupersetErrorType from superset.exceptions import ( @@ -115,6 +116,7 @@ Metric, ) from superset.utils.database import get_example_database +from superset.utils.date_parser import parse_human_timedelta from superset.utils.dates import datetime_to_epoch, EPOCH from superset.utils.hashing import md5_sha_from_dict, md5_sha_from_str @@ -131,8 +133,6 @@ DTTM_ALIAS = "__timestamp" -NO_TIME_RANGE = "No filter" - TIME_COMPARISON = "__" JS_MAX_INTEGER = 9007199254740991 # Largest int Java Script can handle 2^53-1 @@ -256,6 +256,7 @@ class FilterOperator(str, Enum): REGEX = "REGEX" IS_TRUE = "IS TRUE" IS_FALSE = "IS FALSE" + TEMPORAL_RANGE = "TEMPORAL_RANGE" class FilterStringOperators(str, Enum): @@ -1284,6 +1285,11 @@ def get_base_axis_labels(columns: Optional[List[Column]]) -> Tuple[str, ...]: return tuple(get_column_name(col) for col in axis_cols) +def get_xaxis_label(columns: Optional[List[Column]]) -> Optional[str]: + labels = get_base_axis_labels(columns) + return labels[0] if labels else None + + def get_column_name( column: Column, verbose_map: Optional[Dict[str, Any]] = None ) -> str: @@ -1855,7 +1861,7 @@ class DateColumn: col_label: str timestamp_format: Optional[str] = None offset: Optional[int] = None - time_shift: Optional[timedelta] = None + time_shift: Optional[str] = None def __hash__(self) -> int: return hash(self.col_label) @@ -1868,7 +1874,7 @@ def get_legacy_time_column( cls, timestamp_format: Optional[str], offset: Optional[int], - time_shift: Optional[timedelta], + time_shift: Optional[str], ) -> DateColumn: return cls( timestamp_format=timestamp_format, @@ -1907,7 +1913,7 @@ def normalize_dttm_col( if _col.offset: df[_col.col_label] += timedelta(hours=_col.offset) if _col.time_shift is not None: - df[_col.col_label] += _col.time_shift + df[_col.col_label] += parse_human_timedelta(_col.time_shift) def parse_boolean_string(bool_str: Optional[str]) -> bool: diff --git a/superset/utils/date_parser.py b/superset/utils/date_parser.py index cc0693770bfa7..f0a525570a049 100644 --- a/superset/utils/date_parser.py +++ b/superset/utils/date_parser.py @@ -45,7 +45,7 @@ TimeRangeAmbiguousError, TimeRangeParseFailError, ) -from superset.utils.core import NO_TIME_RANGE +from superset.constants import NO_TIME_RANGE from superset.utils.memoized import memoized ParserElement.enablePackrat() diff --git a/superset/viz.py b/superset/viz.py index 43e71b533c61c..b91fdc3aaf0ff 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -307,7 +307,7 @@ def get_df(self, query_obj: Optional[QueryObjectDict] = None) -> pd.DataFrame: DateColumn.get_legacy_time_column( timestamp_format=timestamp_format, offset=self.datasource.offset, - time_shift=self.time_shift, + time_shift=self.form_data.get("time_shift"), ) ] ), diff --git a/tests/integration_tests/query_context_tests.py b/tests/integration_tests/query_context_tests.py index 2306b0a1e20e5..23bec876f72d2 100644 --- a/tests/integration_tests/query_context_tests.py +++ b/tests/integration_tests/query_context_tests.py @@ -1018,3 +1018,85 @@ def test_time_grain_and_time_offset_on_legacy_query(app_context, physical_datase } ) ) + + +def test_time_offset_with_temporal_range_filter(app_context, physical_dataset): + qc = QueryContextFactory().create( + datasource={ + "type": physical_dataset.type, + "id": physical_dataset.id, + }, + queries=[ + { + "columns": [ + { + "label": "col6", + "sqlExpression": "col6", + "columnType": "BASE_AXIS", + "timeGrain": "P3M", + } + ], + "metrics": [ + { + "label": "SUM(col1)", + "expressionType": "SQL", + "sqlExpression": "SUM(col1)", + } + ], + "time_offsets": ["3 month ago"], + "filters": [ + { + "col": "col6", + "op": "TEMPORAL_RANGE", + "val": "2002-01 : 2003-01", + } + ], + } + ], + result_type=ChartDataResultType.FULL, + force=True, + ) + query_payload = qc.get_df_payload(qc.queries[0]) + df = query_payload["df"] + """ + col6 SUM(col1) SUM(col1)__3 month ago +0 2002-01-01 3 NaN +1 2002-04-01 12 3.0 +2 2002-07-01 21 12.0 +3 2002-10-01 9 21.0 + """ + assert df["SUM(col1)"].to_list() == [3, 12, 21, 9] + # df["SUM(col1)__3 month ago"].dtype is object so have to convert to float first + assert df["SUM(col1)__3 month ago"].astype("float").astype("Int64").to_list() == [ + pd.NA, + 3, + 12, + 21, + ] + + sqls = query_payload["query"].split(";") + """ + SELECT DATE_TRUNC('quarter', col6) AS col6, + SUM(col1) AS "SUM(col1)" + FROM physical_dataset + WHERE col6 >= TO_TIMESTAMP('2002-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US') + AND col6 < TO_TIMESTAMP('2003-01-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US') + GROUP BY DATE_TRUNC('quarter', col6) + LIMIT 10000; + + SELECT DATE_TRUNC('quarter', col6) AS col6, + SUM(col1) AS "SUM(col1)" + FROM physical_dataset + WHERE col6 >= TO_TIMESTAMP('2001-10-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US') + AND col6 < TO_TIMESTAMP('2002-10-01 00:00:00.000000', 'YYYY-MM-DD HH24:MI:SS.US') + GROUP BY DATE_TRUNC('quarter', col6) + LIMIT 10000; + """ + assert ( + re.search(r"WHERE col6 >= .*2002-01-01", sqls[0]) + and re.search(r"AND col6 < .*2003-01-01", sqls[0]) + ) is not None + assert ( + re.search(r"WHERE col6 >= .*2001-10-01", sqls[1]) + and re.search(r"AND col6 < .*2002-10-01", sqls[1]) + ) is not None diff --git a/tests/integration_tests/sqla_models_tests.py b/tests/integration_tests/sqla_models_tests.py index b3ec031fc80de..3088bdfb02c55 100644 --- a/tests/integration_tests/sqla_models_tests.py +++ b/tests/integration_tests/sqla_models_tests.py @@ -23,7 +23,6 @@ import numpy as np import pandas as pd -import sqlalchemy as sa from flask import Flask from pytest_mock import MockFixture from sqlalchemy.sql import text @@ -41,7 +40,6 @@ FilterOperator, GenericDataType, TemporalType, - backend, ) from superset.utils.database import get_example_database from tests.integration_tests.fixtures.birth_names_dashboard import ( @@ -71,6 +69,7 @@ class FilterTestCase(NamedTuple): + column: str operator: str value: Union[float, int, List[Any], str] expected: Union[str, List[str]] @@ -271,19 +270,22 @@ def test_adhoc_metrics_and_calc_columns(self): @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") def test_where_operators(self): filters: Tuple[FilterTestCase, ...] = ( - FilterTestCase(FilterOperator.IS_NULL, "", "IS NULL"), - FilterTestCase(FilterOperator.IS_NOT_NULL, "", "IS NOT NULL"), + FilterTestCase("num", FilterOperator.IS_NULL, "", "IS NULL"), + FilterTestCase("num", FilterOperator.IS_NOT_NULL, "", "IS NOT NULL"), # Some db backends translate true/false to 1/0 - FilterTestCase(FilterOperator.IS_TRUE, "", ["IS 1", "IS true"]), - FilterTestCase(FilterOperator.IS_FALSE, "", ["IS 0", "IS false"]), - FilterTestCase(FilterOperator.GREATER_THAN, 0, "> 0"), - FilterTestCase(FilterOperator.GREATER_THAN_OR_EQUALS, 0, ">= 0"), - FilterTestCase(FilterOperator.LESS_THAN, 0, "< 0"), - FilterTestCase(FilterOperator.LESS_THAN_OR_EQUALS, 0, "<= 0"), - FilterTestCase(FilterOperator.EQUALS, 0, "= 0"), - FilterTestCase(FilterOperator.NOT_EQUALS, 0, "!= 0"), - FilterTestCase(FilterOperator.IN, ["1", "2"], "IN (1, 2)"), - FilterTestCase(FilterOperator.NOT_IN, ["1", "2"], "NOT IN (1, 2)"), + FilterTestCase("num", FilterOperator.IS_TRUE, "", ["IS 1", "IS true"]), + FilterTestCase("num", FilterOperator.IS_FALSE, "", ["IS 0", "IS false"]), + FilterTestCase("num", FilterOperator.GREATER_THAN, 0, "> 0"), + FilterTestCase("num", FilterOperator.GREATER_THAN_OR_EQUALS, 0, ">= 0"), + FilterTestCase("num", FilterOperator.LESS_THAN, 0, "< 0"), + FilterTestCase("num", FilterOperator.LESS_THAN_OR_EQUALS, 0, "<= 0"), + FilterTestCase("num", FilterOperator.EQUALS, 0, "= 0"), + FilterTestCase("num", FilterOperator.NOT_EQUALS, 0, "!= 0"), + FilterTestCase("num", FilterOperator.IN, ["1", "2"], "IN (1, 2)"), + FilterTestCase("num", FilterOperator.NOT_IN, ["1", "2"], "NOT IN (1, 2)"), + FilterTestCase( + "ds", FilterOperator.TEMPORAL_RANGE, "2020 : 2021", "2020-01-01" + ), ) table = self.get_table(name="birth_names") for filter_ in filters: @@ -295,7 +297,11 @@ def test_where_operators(self): "metrics": ["count"], "is_timeseries": False, "filter": [ - {"col": "num", "op": filter_.operator, "val": filter_.value} + { + "col": filter_.column, + "op": filter_.operator, + "val": filter_.value, + } ], "extras": {}, } @@ -835,3 +841,26 @@ def _convert_dttm( assert str(normalized) == str(result) else: assert normalized == result + + +def test__temporal_range_operator_in_adhoc_filter(app_context, physical_dataset): + result = physical_dataset.query( + { + "columns": ["col1", "col2"], + "filter": [ + { + "col": "col5", + "val": "2000-01-05 : 2000-01-06", + "op": FilterOperator.TEMPORAL_RANGE.value, + }, + { + "col": "col6", + "val": "2002-05-11 : 2002-05-12", + "op": FilterOperator.TEMPORAL_RANGE.value, + }, + ], + "is_timeseries": False, + } + ) + df = pd.DataFrame(index=[0], data={"col1": 4, "col2": "e"}) + assert df.equals(result.df) diff --git a/tests/integration_tests/utils_tests.py b/tests/integration_tests/utils_tests.py index 7df9cd82f96f0..9c3bac1c5b0ea 100644 --- a/tests/integration_tests/utils_tests.py +++ b/tests/integration_tests/utils_tests.py @@ -39,6 +39,7 @@ import tests.integration_tests.test_app from superset import app, db, security_manager +from superset.constants import NO_TIME_RANGE from superset.exceptions import CertificateException, SupersetException from superset.models.core import Database, Log from superset.models.dashboard import Dashboard @@ -62,7 +63,6 @@ merge_extra_filters, merge_extra_form_data, merge_request_params, - NO_TIME_RANGE, normalize_dttm_col, parse_ssl_cert, parse_js_uri_path_item, @@ -1060,7 +1060,7 @@ def normalize_col( df: pd.DataFrame, timestamp_format: Optional[str], offset: int, - time_shift: Optional[timedelta], + time_shift: Optional[str], ) -> pd.DataFrame: df = df.copy() normalize_dttm_col( @@ -1091,9 +1091,9 @@ def normalize_col( ) # test offset and timedelta - assert normalize_col(df, None, 1, timedelta(minutes=30))[DTTM_ALIAS][ - 0 - ] == pd.Timestamp(2021, 2, 15, 20, 30, 0, 0) + assert normalize_col(df, None, 1, "30 minutes")[DTTM_ALIAS][0] == pd.Timestamp( + 2021, 2, 15, 20, 30, 0, 0 + ) # test numeric epoch_s format df = pd.DataFrame([{"__timestamp": ts.timestamp(), "a": 1}]) diff --git a/tests/unit_tests/common/test_time_range_utils.py b/tests/unit_tests/common/test_time_range_utils.py new file mode 100644 index 0000000000000..bde1bd4befc0a --- /dev/null +++ b/tests/unit_tests/common/test_time_range_utils.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import datetime +from unittest import mock + +import pytest + +from superset.common.utils.time_range_utils import ( + get_since_until_from_query_object, + get_since_until_from_time_range, +) + + +def test__get_since_until_from_time_range(): + assert get_since_until_from_time_range(time_range="2001 : 2002") == ( + datetime(2001, 1, 1), + datetime(2002, 1, 1), + ) + assert get_since_until_from_time_range( + time_range="2001 : 2002", time_shift="8 hours ago" + ) == ( + datetime(2000, 12, 31, 16, 0, 0), + datetime(2001, 12, 31, 16, 0, 0), + ) + with mock.patch( + "superset.utils.date_parser.EvalDateTruncFunc.eval", + return_value=datetime(2000, 1, 1, 0, 0, 0), + ): + assert ( + get_since_until_from_time_range( + time_range="Last year", + extras={ + "relative_end": "2100", + }, + ) + )[1] == datetime(2100, 1, 1, 0, 0) + with mock.patch( + "superset.utils.date_parser.EvalDateTruncFunc.eval", + return_value=datetime(2000, 1, 1, 0, 0, 0), + ): + assert ( + get_since_until_from_time_range( + time_range="Next year", + extras={ + "relative_start": "2000", + }, + ) + )[0] == datetime(2000, 1, 1, 0, 0) + + +@pytest.mark.query_object( + { + "time_range": "2001 : 2002", + "time_shift": "8 hours ago", + } +) +def test__since_until_from_time_range(dummy_query_object): + assert get_since_until_from_query_object(dummy_query_object) == ( + datetime(2000, 12, 31, 16, 0, 0), + datetime(2001, 12, 31, 16, 0, 0), + ) + + +@pytest.mark.query_object( + { + "filters": [{"col": "dttm", "op": "TEMPORAL_RANGE", "val": "2001 : 2002"}], + "columns": [ + { + "columnType": "BASE_AXIS", + "label": "dttm", + "sqlExpression": "dttm", + } + ], + } +) +def test__since_until_from_adhoc_filters(dummy_query_object): + assert get_since_until_from_query_object(dummy_query_object) == ( + datetime(2001, 1, 1, 0, 0, 0), + datetime(2002, 1, 1, 0, 0, 0), + ) diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 5ad6af888a17f..6740a8b6e280b 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -18,6 +18,7 @@ import importlib import os +import unittest.mock from typing import Any, Callable, Iterator import pytest @@ -29,6 +30,8 @@ from superset import security_manager from superset.app import SupersetApp +from superset.common.chart_data import ChartDataResultType +from superset.common.query_object_factory import QueryObjectFactory from superset.extensions import appbuilder from superset.initialization import SupersetAppInitializer @@ -136,3 +139,27 @@ def full_api_access(mocker: MockFixture) -> Iterator[None]: mocker.patch.object(security_manager, "can_access_all_databases", return_value=True) yield + + +@pytest.fixture +def dummy_query_object(request, app_context): + query_obj_marker = request.node.get_closest_marker("query_object") + result_type_marker = request.node.get_closest_marker("result_type") + + if query_obj_marker is None: + query_object = {} + else: + query_object = query_obj_marker.args[0] + + if result_type_marker is None: + result_type = ChartDataResultType.FULL + else: + result_type = result_type_marker.args[0] + + yield QueryObjectFactory( + app_configurations={ + "ROW_LIMIT": 100, + }, + _datasource_dao=unittest.mock.Mock(), + session_maker=unittest.mock.Mock(), + ).create(parent_result_type=result_type, **query_object)