feat: advanced analytics for timeseries in echart viz (apache#1170)

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/index.ts

@@ -20,6 +20,7 @@ import * as sectionsModule from './sections';
 
 export * from './utils';
 export * from './constants';
+export * from './operators';
 
 // can't do `export * as sections from './sections'`, babel-transformer will fail
 export const sections = sectionsModule;

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/index.ts

@@ -0,0 +1,25 @@
+/* eslint-disable camelcase */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+export { rollingWindowOperator } from './rollingWindowOperator';
+export { timeCompareOperator } from './timeCompareOperator';
+export { timeComparePivotOperator } from './timeComparePivotOperator';
+export { sortOperator } from './sortOperator';
+export { pivotOperator } from './pivotOperator';
+export * from './utils';

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts

@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+import { ensureIsArray, getMetricLabel, PostProcessingPivot } from '@superset-ui/core';
+import { PostProcessingFactory } from './types';
+import { TIME_COLUMN, isValidTimeCompare } from './utils';
+import { timeComparePivotOperator } from './timeComparePivotOperator';
+
+export const pivotOperator: PostProcessingFactory<PostProcessingPivot | undefined> = (
+  formData,
+  queryObject,
+) => {
+  const metricLabels = ensureIsArray(queryObject.metrics).map(getMetricLabel);
+  if (queryObject.is_timeseries && metricLabels.length) {
+    if (isValidTimeCompare(formData, queryObject)) {
+      return timeComparePivotOperator(formData, queryObject);
+    }
+
+    return {
+      operation: 'pivot',
+      options: {
+        index: [TIME_COLUMN],
+        columns: queryObject.columns || [],
+        // Create 'dummy' mean aggregates to assign cell values in pivot table
+        // use the 'mean' aggregates to avoid drop NaN. PR: https://github.com/apache-superset/superset-ui/pull/1231
+        aggregates: Object.fromEntries(metricLabels.map(metric => [metric, { operator: 'mean' }])),
+        drop_missing_columns: false,
+      },
+    };
+  }
+
+  return undefined;
+};

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/rollingWindowOperator.ts

@@ -0,0 +1,80 @@
+/* eslint-disable camelcase */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+import {
+  ensureIsInt,
+  ensureIsArray,
+  RollingType,
+  PostProcessingRolling,
+  PostProcessingCum,
+  ComparisionType,
+} from '@superset-ui/core';
+import { getMetricOffsetsMap, isValidTimeCompare, TIME_COMPARISON_SEPARATOR } from './utils';
+import { PostProcessingFactory } from './types';
+
+export const rollingWindowOperator: PostProcessingFactory<
+  PostProcessingRolling | PostProcessingCum | undefined
+> = (formData, queryObject) => {
+  let columns: (string | undefined)[];
+  if (isValidTimeCompare(formData, queryObject)) {
+    const metricsMap = getMetricOffsetsMap(formData, queryObject);
+    const comparisonType = formData.comparison_type;
+    if (comparisonType === ComparisionType.Values) {
+      // time compare type: actual values
+      columns = [...Array.from(metricsMap.values()), ...Array.from(metricsMap.keys())];
+    } else {
+      // time compare type: absolute / percentage / ratio
+      columns = Array.from(metricsMap.entries()).map(([offset, metric]) =>
+        [comparisonType, metric, offset].join(TIME_COMPARISON_SEPARATOR),
+      );
+    }
+  } else {
+    columns = ensureIsArray(queryObject.metrics).map(metric => {
+      if (typeof metric === 'string') {
+        return metric;
+      }
+      return metric.label;
+    });
+  }
+  const columnsMap = Object.fromEntries(columns.map(col => [col, col]));
+
+  if (formData.rolling_type === RollingType.Cumsum) {
+    return {
+      operation: 'cum',
+      options: {
+        operator: 'sum',
+        columns: columnsMap,
+      },
+    };
+  }
+
+  if ([RollingType.Sum, RollingType.Mean, RollingType.Std].includes(formData.rolling_type)) {
+    return {
+      operation: 'rolling',
+      options: {
+        rolling_type: formData.rolling_type,
+        window: ensureIsInt(formData.rolling_periods, 1),
+        min_periods: ensureIsInt(formData.min_periods, 0),
+        columns: columnsMap,
+      },
+    };
+  }
+
+  return undefined;
+};

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/sortOperator.ts

@@ -0,0 +1,39 @@
+/* eslint-disable camelcase */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+import { PostProcessingSort, RollingType } from '@superset-ui/core';
+import { PostProcessingFactory } from './types';
+import { TIME_COLUMN } from './utils';
+
+export const sortOperator: PostProcessingFactory<PostProcessingSort | undefined> = (
+  formData,
+  queryObject,
+) => {
+  if (queryObject.is_timeseries && Object.values(RollingType).includes(formData.rolling_type)) {
+    return {
+      operation: 'sort',
+      options: {
+        columns: {
+          [TIME_COLUMN]: true,
+        },
+      },
+    };
+  }
+  return undefined;
+};

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/timeCompareOperator.ts

@@ -0,0 +1,44 @@
+/* eslint-disable camelcase */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+import { ComparisionType, PostProcessingCompare } from '@superset-ui/core';
+import { getMetricOffsetsMap, isValidTimeCompare } from './utils';
+import { PostProcessingFactory } from './types';
+
+export const timeCompareOperator: PostProcessingFactory<PostProcessingCompare | undefined> = (
+  formData,
+  queryObject,
+) => {
+  const comparisonType = formData.comparison_type;
+  const metricOffsetMap = getMetricOffsetsMap(formData, queryObject);
+
+  if (isValidTimeCompare(formData, queryObject) && comparisonType !== ComparisionType.Values) {
+    return {
+      operation: 'compare',
+      options: {
+        source_columns: Array.from(metricOffsetMap.values()),
+        compare_columns: Array.from(metricOffsetMap.keys()),
+        compare_type: comparisonType,
+        drop_original_columns: true,
+      },
+    };
+  }
+
+  return undefined;
+};

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts

@@ -0,0 +1,58 @@
+/* eslint-disable camelcase */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+import { ComparisionType, PostProcessingPivot, NumpyFunction } from '@superset-ui/core';
+import { getMetricOffsetsMap, isValidTimeCompare, TIME_COMPARISON_SEPARATOR } from './utils';
+import { PostProcessingFactory } from './types';
+
+export const timeComparePivotOperator: PostProcessingFactory<PostProcessingPivot | undefined> = (
+  formData,
+  queryObject,
+) => {
+  const comparisonType = formData.comparison_type;
+  const metricOffsetMap = getMetricOffsetsMap(formData, queryObject);
+
+  if (isValidTimeCompare(formData, queryObject)) {
+    const valuesAgg = Object.fromEntries(
+      [...metricOffsetMap.values(), ...metricOffsetMap.keys()].map(metric => [
+        metric,
+        // use the 'mean' aggregates to avoid drop NaN
+        { operator: 'mean' as NumpyFunction },
+      ]),
+    );
+    const changeAgg = Object.fromEntries(
+      [...metricOffsetMap.entries()]
+        .map(([offset, metric]) => [comparisonType, metric, offset].join(TIME_COMPARISON_SEPARATOR))
+        // use the 'mean' aggregates to avoid drop NaN
+        .map(metric => [metric, { operator: 'mean' as NumpyFunction }]),
+    );
+
+    return {
+      operation: 'pivot',
+      options: {
+        index: ['__timestamp'],
+        columns: queryObject.columns || [],
+        aggregates: comparisonType === ComparisionType.Values ? valuesAgg : changeAgg,
+        drop_missing_columns: false,
+      },
+    };
+  }
+
+  return undefined;
+};

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/types.ts

@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+import { QueryFormData, QueryObject } from '@superset-ui/core';
+
+export interface PostProcessingFactory<T> {
+  (formData: QueryFormData, queryObject: QueryObject): T;
+}

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-chart-controls/src/operators/utils/constants.ts

@@ -0,0 +1,21 @@
+/* eslint-disable camelcase */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitationsxw
+ * under the License.
+ */
+export const TIME_COMPARISON_SEPARATOR = '__';
+export const TIME_COLUMN = '__timestamp';