Refactor to support DataSketches Quantile、Theta、Frequency Sketches

Signed-off-by: chenminghua8 <cmptmn@126.com>

fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java

@@ -1624,98 +1624,6 @@ private void registerBuiltinDsFunction() {
             addBuiltin(AggregateFunction.createBuiltin(DS_HLL_COUNT_DISTINCT,
                     Lists.newArrayList(t, Type.INT, Type.VARCHAR), Type.BIGINT, Type.VARBINARY,
                     true, false, true));
-            // ds_theta(col)
-            addBuiltin(AggregateFunction.createBuiltin(DS_THETA,
-                    Lists.newArrayList(t), Type.BIGINT, Type.VARBINARY,
-                    true, false, true));
-        }
-
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.DOUBLE), Type.ARRAY_DOUBLE, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.FLOAT), Type.ARRAY_FLOAT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.INT), Type.ARRAY_INT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.BIGINT), Type.ARRAY_BIGINT, Type.VARBINARY,
-                false, false, true));
-
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.DOUBLE, Type.DOUBLE), Type.ARRAY_DOUBLE, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.FLOAT, Type.DOUBLE), Type.ARRAY_FLOAT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.INT, Type.DOUBLE), Type.ARRAY_INT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.BIGINT, Type.DOUBLE), Type.ARRAY_BIGINT, Type.VARBINARY,
-                false, false, true));
-
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.DOUBLE, Type.DOUBLE, Type.INT), Type.ARRAY_DOUBLE, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.FLOAT, Type.DOUBLE, Type.INT), Type.ARRAY_FLOAT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.INT, Type.DOUBLE, Type.INT), Type.ARRAY_INT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.BIGINT, Type.DOUBLE, Type.INT), Type.ARRAY_BIGINT, Type.VARBINARY,
-                false, false, true));
-
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.DOUBLE, Type.ARRAY_DOUBLE), Type.ARRAY_DOUBLE, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.FLOAT, Type.ARRAY_DOUBLE), Type.ARRAY_FLOAT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.INT, Type.ARRAY_DOUBLE), Type.ARRAY_INT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.BIGINT, Type.ARRAY_DOUBLE), Type.ARRAY_BIGINT, Type.VARBINARY,
-                false, false, true));
-
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.DOUBLE, Type.ARRAY_DOUBLE, Type.INT), Type.ARRAY_DOUBLE, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.FLOAT, Type.ARRAY_DOUBLE, Type.INT), Type.ARRAY_FLOAT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.INT, Type.ARRAY_DOUBLE, Type.INT), Type.ARRAY_INT, Type.VARBINARY,
-                false, false, true));
-        addBuiltin(AggregateFunction.createBuiltin(DS_QUANTILE,
-                Lists.newArrayList(Type.BIGINT, Type.ARRAY_DOUBLE, Type.INT), Type.ARRAY_BIGINT, Type.VARBINARY,
-                false, false, true));
-
-        ImmutableList<Type> DS_FREQUENT_SUPPORTED_TYPES =
-                ImmutableList.<Type>builder()
-                        .addAll(Type.FLOAT_TYPES)
-                        .addAll(Type.INTEGER_TYPES)
-                        .addAll(Type.STRING_TYPES)
-                        .addAll(Type.DATE_TYPES)
-                        .build();
-        for (Type type : DS_FREQUENT_SUPPORTED_TYPES) {
-            ArrayType retType = DS_FREQUENT_RET_TYPE_BUILDER.apply(type);
-            addBuiltin(AggregateFunction.createBuiltin(DS_FREQUENT,
-                    Lists.newArrayList(type), retType, Type.VARBINARY,
-                    false, true, true));
-            addBuiltin(AggregateFunction.createBuiltin(DS_FREQUENT,
-                    Lists.newArrayList(type, Type.BIGINT), retType, Type.VARBINARY,
-                    false, true, true));
-            addBuiltin(AggregateFunction.createBuiltin(DS_FREQUENT,
-                    Lists.newArrayList(type, Type.BIGINT, Type.INT), retType, Type.VARBINARY,
-                    false, true, true));
-            addBuiltin(AggregateFunction.createBuiltin(DS_FREQUENT,
-                    Lists.newArrayList(type, Type.BIGINT, Type.INT, Type.INT), retType, Type.VARBINARY,
-                    false, true, true));
         }
     }
 

fe/fe-core/src/main/java/com/starrocks/sql/analyzer/FunctionAnalyzer.java

@@ -342,10 +342,7 @@ private static void analyzeBuiltinAggFunction(FunctionName fnName,
                 || fnName.getFunction().equals(FunctionSet.MAX)
                 || fnName.getFunction().equals(FunctionSet.NDV)
                 || fnName.getFunction().equals(FunctionSet.APPROX_COUNT_DISTINCT)
-                || fnName.getFunction().equals(FunctionSet.DS_HLL_COUNT_DISTINCT)
-                || fnName.getFunction().equals(FunctionSet.DS_QUANTILE)
-                || fnName.getFunction().equals(FunctionSet.DS_FREQUENT)
-                || fnName.getFunction().equals(FunctionSet.DS_THETA))
+                || fnName.getFunction().equals(FunctionSet.DS_HLL_COUNT_DISTINCT))
                 && !arg.getType().canApplyToNumeric()) {
             throw new SemanticException(Type.NOT_SUPPORT_AGG_ERROR_MSG);
         }
@@ -547,94 +544,7 @@ private static void analyzeBuiltinAggFunction(FunctionName fnName,
                 }
             }
         }
-
-        // ds_quantile
-        if (fnName.getFunction().equals(FunctionSet.DS_QUANTILE)) {
-            int argSize = functionCallExpr.getChildren().size();
-            if (argSize > 3 || argSize < 1) {
-                throw new SemanticException(fnName + " requires one/two/three parameters: ds_quantile(col, ranks, k)");
-            }
-            if (!functionCallExpr.getChild(0).getType().isNumericType()) {
-                throw new SemanticException(
-                        "ds_quantile requires the first parameter's type is numeric type");
-            }
-            if (argSize >= 2) {
-                if (functionCallExpr.getChild(1) instanceof ArrayExpr) {
-                    ArrayExpr ranksArrExpr = (ArrayExpr) functionCallExpr.getChild(1);
-                    int ranksSize = ranksArrExpr.getChildren().size();
-                    for (int i = 0; i < ranksSize; i++) {
-                        if (!(ranksArrExpr.getChild(i) instanceof DecimalLiteral)) {
-                            throw new SemanticException(fnName + " 's second parameter's data type is wrong.");
-                        }
-                        double rank = ((LiteralExpr) ranksArrExpr.getChild(i)).getDoubleValue();
-                        if (rank < 0 || rank > 1) {
-                            throw new SemanticException(
-                                    fnName + " rank should be between 0 and 1.");
-                        }
-                    }
-                } else if ((functionCallExpr.getChild(1) instanceof DecimalLiteral)) {
-                    double rank = ((LiteralExpr) functionCallExpr.getChild(1)).getDoubleValue();
-                    if (rank < 0 || rank > 1) {
-                        throw new SemanticException(
-                                fnName + " rank should be between 0 and 1.");
-                    }
-                } else {
-                    throw new SemanticException(fnName + " 's second parameter's data type is wrong.");
-                }
-            }
-            if (argSize == 3) {
-                if (!(functionCallExpr.getChild(2) instanceof IntLiteral)) {
-                    throw new SemanticException(fnName + " 's third parameter's data type is wrong.");
-                }
-                long k = ((LiteralExpr) functionCallExpr.getChild(2)).getLongValue();
-                if (k < 2 || k > 32768) {
-                    throw new SemanticException(
-                            fnName + " third parameter'value should be between 2 and 32768.");
-                }
-            }
-        }
-
-        // ds_frequent
-        if (fnName.getFunction().equals(FunctionSet.DS_FREQUENT)) {
-            int argSize = functionCallExpr.getChildren().size();
-            if (argSize > 4) {
-                throw new SemanticException(fnName + " requires one/two/three/four parameters: ds_frequent(" +
-                        "col, counter_num, lg_max_map_size, lg_start_map_size)");
-            }
-            if (argSize >= 2) {
-                if (!(functionCallExpr.getChild(1) instanceof IntLiteral)) {
-                    throw new SemanticException(fnName + " 's second parameter's data type is wrong.");
-                }
-                long counterNum = ((LiteralExpr) functionCallExpr.getChild(1)).getLongValue();
-                if (counterNum < 1) {
-                    throw new SemanticException(
-                            fnName + " second parameter'value must be greater than 1.");
-                }
-            }
-
-            if (argSize >= 3) {
-                if (!(functionCallExpr.getChild(2) instanceof IntLiteral)) {
-                    throw new SemanticException(fnName + " 's third parameter's data type is wrong.");
-                }
-                long lgMaxMapSize = ((LiteralExpr) functionCallExpr.getChild(2)).getLongValue();
-                if (lgMaxMapSize < 3 || lgMaxMapSize > 21) {
-                    throw new SemanticException(
-                            fnName + " third parameter'value should be between 3 and 21.");
-                }
-            }
-
-            if (argSize == 4) {
-                if (!(functionCallExpr.getChild(3) instanceof IntLiteral)) {
-                    throw new SemanticException(fnName + " 's third parameter's data type is wrong.");
-                }
-                long lgStartMapSize = ((LiteralExpr) functionCallExpr.getChild(3)).getLongValue();
-                if (lgStartMapSize < 3 || lgStartMapSize > 21) {
-                    throw new SemanticException(
-                            fnName + " fourth parameter'value should be between 3 and 21.");
-                }
-            }
-        }
-
+
         if (fnName.getFunction().equals(FunctionSet.COVAR_POP) || fnName.getFunction().equals(FunctionSet.COVAR_SAMP) ||
                 fnName.getFunction().equals(FunctionSet.CORR)) {
             if (functionCallExpr.getChildren().size() != 2) {

fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeAggregateTest.java

@@ -307,44 +307,4 @@ public void testWindowFunnelFunction() {
         analyzeSuccess("SELECT window_funnel(1, ta, 0, [ta='a', ta='b']) FROM tall");
         analyzeSuccess("SELECT window_funnel(1, ta, 0, [true, true, false]) FROM tall");
     }
-
-    @Test
-    public void testDsHll() {
-        analyzeSuccess("select ds_hll(v1) from t0");
-        analyzeSuccess("select ds_hll(v1, 4) from t0");
-        analyzeSuccess("select ds_hll(v1, 10, 'HLL_6') from t0");
-    }
-
-    @Test
-    public void testDsTheta() {
-        analyzeSuccess("select ds_theta(v1) from t0");
-    }
-
-    @Test
-    public void testDsQuantile() {
-        analyzeSuccess("select ds_quantile(v1) from t0");
-        analyzeSuccess("select ds_quantile(v1, 0.5) from t0");
-        analyzeSuccess("select ds_quantile(v1, 0.5, 21) from t0");
-        analyzeFail("select ds_quantile(v1, 2) from t0");
-        analyzeFail("select ds_quantile(v1, '0.5', 21) from t0");
-        analyzeFail("select ds_quantile(v1, 0.5, 32769) from t0");
-        analyzeFail("select ds_quantile(v1, ['0.5', '0.6'], 21) from t0");
-        analyzeFail("select ds_quantile(v1, [2.0, 3.0], 21) from t0");
-        analyzeFail("select ds_quantile(v1, 0.5, 21, 21) from t0");
-        analyzeFail("select ds_quantile(v1, 0.5, '21') from t0");
-    }
-    @Test
-    public void testDsFrequent() {
-        analyzeSuccess("select ds_frequent(v1) from t0");
-        analyzeSuccess("select ds_frequent(v1, 10) from t0");
-        analyzeSuccess("select ds_frequent(v1, 10, 20) from t0");
-        analyzeSuccess("select ds_frequent(v1, 10, 20, 16) from t0");
-        analyzeFail("select ds_frequent(v1, '10') from t0");
-        analyzeFail("select ds_frequent(v1, 0) from t0");
-        analyzeFail("select ds_frequent(v1, 10, '20') from t0");
-        analyzeFail("select ds_frequent(v1, 10, 200) from t0");
-        analyzeFail("select ds_frequent(v1, 10, 20, '16') from t0");
-        analyzeFail("select ds_frequent(v1, 10, 20, 200) from t0");
-        analyzeFail("select ds_frequent(v1, 10, 20, 16, 16) from t0");
-    }
 }