From 82b393314c674d62bd9050c4ce1dec75369d9453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Mon, 25 Sep 2023 12:45:11 +0800 Subject: [PATCH] [enhancement](Nereids) check unimplemented func in nereids (#24766) --- .../doris/catalog/FunctionRegistry.java | 8 + .../org/apache/doris/catalog/FunctionSet.java | 7 + .../trees/expressions/FunctionTest.java | 161 ++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/FunctionTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java index 012f15faeafef5..57fcbff6a49053 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import javax.annotation.concurrent.ThreadSafe; @@ -75,6 +76,13 @@ public FunctionBuilder findFunctionBuilder(String name, Object argument) { return findFunctionBuilder(null, name, ImmutableList.of(argument)); } + public Optional> tryGetBuiltinBuilders(String name) { + List builders = name2InternalBuiltinBuilders.get(name); + return name2InternalBuiltinBuilders.get(name) == null + ? Optional.empty() + : Optional.of(ImmutableList.copyOf(builders)); + } + // currently we only find function by name and arity and args' types. public FunctionBuilder findFunctionBuilder(String dbName, String name, List arguments) { int arity = arguments.size(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index e1c62cc739d2b7..9390969d7a5f01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1677,6 +1677,13 @@ public List getBulitinFunctions() { return builtinFunctions; } + public List getAllFunctions() { + List functions = Lists.newArrayList(); + vectorizedFunctions.forEach((k, v) -> functions.addAll(v)); + tableFunctions.forEach((k, v) -> functions.addAll(v)); + return functions; + } + public static final String EXPLODE_SPLIT = "explode_split"; public static final String EXPLODE_BITMAP = "explode_bitmap"; public static final String EXPLODE_JSON_ARRAY_INT = "explode_json_array_int"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/FunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/FunctionTest.java new file mode 100644 index 00000000000000..6a5289d018b54c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/FunctionTest.java @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.analysis.LambdaFunctionCallExpr; +import org.apache.doris.catalog.BuiltinWindowFunctions; +import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.FunctionRegistry; +import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.functions.BuiltinFunctionBuilder; +import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +class FunctionTest { + + @Disabled + @Test + void testCatalog() { + FunctionRegistry functionRegistry = new FunctionRegistry(); + Set>> catalogFunc = getAllCatalogFunc(); + Set unBounded = new HashSet<>(); + Set notUnBoundedFunc = new HashSet<>(); + Set unParsed = new HashSet<>(); + Set unmatchedArg = new HashSet<>(); + NereidsParser parser = new NereidsParser(); + for (Pair> func : catalogFunc) { + try { + String funcCall = construncFuncCall(func); + Expression unboundedFunc = parser.parseExpression(funcCall); + if (unboundedFunc instanceof UnboundFunction) { + Optional> builders = functionRegistry.tryGetBuiltinBuilders(func.first); + if (builders.isPresent() && builders.get().stream().allMatch(b -> b instanceof BuiltinFunctionBuilder)) { + try { + functionRegistry.findFunctionBuilder(((UnboundFunction) unboundedFunc).getName(), unboundedFunc.getArguments()); + } catch (Exception e) { + if (BuiltinWindowFunctions.INSTANCE.windowFunctions.stream() + .anyMatch(w -> w.names.contains(func.first))) { + // nereids' window function is different with legacy's + continue; + } + unmatchedArg.add(String.format("%s %s", func.first, func.second)); + } + continue; + } + unBounded.add(func.first); + } else { + notUnBoundedFunc.add(func.first); + } + } catch (Exception exception) { + unParsed.add(func.first); + } + } + System.out.println(unmatchedArg.stream().sorted().map(s -> s + "\n").collect(Collectors.toList())); + System.out.println(unBounded.stream().sorted().map(s -> s + "\n").collect(Collectors.toList())); + System.out.println(notUnBoundedFunc); + System.out.println(unParsed); + } + + private List constructArg(List argTypes) { + return argTypes.stream().map( + typeStr -> "null" + ).collect(Collectors.toList()); + } + + private String construncFuncCall(Pair> func) { + List args = constructArg(func.second); + ImmutableSet simpleFunc = ImmutableSet.builder() + .add("add") + .add("subtract") + .add("divide") + .add("multiply") + .add("eq") + .add("le") + .add("lt") + .add("gt") + .add("ne") + .add("ge") + .add("is_null_pred") + .add("is_not_null_pred") + .add("eq_for_null") + .add("mod") + .add("int_divide") + .add("bitand") + .add("bitor") + .add("bitxor") + .add("bitnot") + .add("and") + .add("or") + .add("not") + .add("n") + .build(); + ImmutableSet inPredicate = ImmutableSet.builder() + .add("in_set_lookup") + .add("not_in_set_lookup") + .add("in_iterate") + .add("not_in_iterate") + .build(); + if (func.first.contains("match_")) { + return String.format("'a' %s 'abcdefg'", func.first); + } else if (simpleFunc.contains(func.first)) { + return "1 + 1"; + } else if (func.first.contains("like")) { + return String.format("'a' %s '%%a%%'", func.first); + } else if (func.first.contains("regexp")) { + return String.format("'a' %s '^a'", func.first); + } else if (inPredicate.contains(func.first)) { + return String.format("continue"); + } else if (LambdaFunctionCallExpr.LAMBDA_FUNCTION_SET.contains(func.first)) { + return String.format("%s(null)", func.first); + } + return String.format("%s%s", func.first, + args.stream().collect(Collectors.joining(", ", "(", ")"))); + } + + private Set>> getAllCatalogFunc() { + FunctionSet functionSet = new FunctionSet<>(); + functionSet.init(); + return functionSet.getAllFunctions().stream() + .filter(f -> filterUnvaliedFuncName(f.functionName())) + .map(f -> Pair.of(f.getFunctionName().toString(), + Arrays.stream(f.getArgs()).map(Type::toSql).collect(Collectors.toList()))) + .collect(Collectors.toSet()); + } + + private boolean filterUnvaliedFuncName(String name) { + if (name.contains("castto")) { + // like casttostring + return false; + } + return true; + } +}