Skip to content

Remove SqlKind from FunctionCall #13293

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jackie-Jiang merged 1 commit into from
Jun 2, 2024
Merged

Remove SqlKind from FunctionCall #13293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import java.math.BigDecimal;
import java.util.ArrayList;
Expand Down Expand Up @@ -408,18 +409,16 @@ public static String canonicalizeFunctionName(String functionName) {
private static final Map<String, String> CANONICAL_NAME_TO_SPECIAL_KEY_MAP;

static {
CANONICAL_NAME_TO_SPECIAL_KEY_MAP = new HashMap<>();
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
for (FilterKind filterKind : FilterKind.values()) {
CANONICAL_NAME_TO_SPECIAL_KEY_MAP.put(canonicalizeFunctionName(filterKind.name()), filterKind.name());
builder.put(canonicalizeFunctionName(filterKind.name()), filterKind.name());
}
CANONICAL_NAME_TO_SPECIAL_KEY_MAP.put("stdistance", "st_distance");
CANONICAL_NAME_TO_SPECIAL_KEY_MAP = builder.build();
}

/**
* Converts the function name into its canonical form, but preserving the special keys.
* - Keep FilterKind.name() as is because we need to read the FilterKind via FilterKind.valueOf().
* - Keep ST_Distance as is because we use exact match when applying geo-spatial index up to release 0.10.0.
* TODO: Remove the ST_Distance special handling after releasing 0.11.0.
*/
public static String canonicalizeFunctionNamePreservingSpecialKey(String functionName) {
String canonicalName = canonicalizeFunctionName(functionName);
Expand Down
9 changes: 4 additions & 5 deletions pinot-common/src/main/proto/expressions.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,10 @@ message Literal {
}

message FunctionCall {
int32 sqlKind = 1;
ColumnDataType dataType = 2;
string functionName = 3;
repeated RexExpression functionOperands = 4;
bool isDistinct = 5;
ColumnDataType dataType = 1;
string functionName = 2;
repeated RexExpression functionOperands = 3;
bool isDistinct = 4;
}

message RexExpression {
Expand Down
220 changes: 58 additions & 162 deletions ...query-planner/src/main/java/org/apache/pinot/query/parser/CalciteRexExpressionParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,28 +19,16 @@
package org.apache.pinot.query.parser;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.calcite.rel.RelFieldCollation.Direction;
import org.apache.calcite.rel.RelFieldCollation.NullDirection;
import org.apache.calcite.sql.SqlKind;
import org.apache.pinot.common.request.Expression;
import org.apache.pinot.common.request.ExpressionType;
import org.apache.pinot.common.request.Function;
import org.apache.pinot.common.request.PinotQuery;
import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.query.planner.logical.RexExpression;
import org.apache.pinot.query.planner.plannode.SortNode;
import org.apache.pinot.segment.spi.AggregationFunctionType;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.sql.FilterKind;
import org.apache.pinot.sql.parsers.ParserUtils;
import org.apache.pinot.sql.parsers.SqlCompilationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


/**
Expand All @@ -54,69 +42,45 @@ public class CalciteRexExpressionParser {
private CalciteRexExpressionParser() {
}

private static final Logger LOGGER = LoggerFactory.getLogger(CalciteRexExpressionParser.class);
private static final Map<String, String> CANONICAL_NAME_TO_SPECIAL_KEY_MAP;
private static final String ARRAY_TO_MV_FUNCTION_NAME = "arraytomv";

static {
CANONICAL_NAME_TO_SPECIAL_KEY_MAP = new HashMap<>();
// adding filter kind special handling
for (FilterKind filterKind : FilterKind.values()) {
CANONICAL_NAME_TO_SPECIAL_KEY_MAP.put(RequestUtils.canonicalizeFunctionName(filterKind.name()),
filterKind.name());
}
// adding SqlKind.OTHERS and SqlKind.OTHER_FUNCTIONS that have canonical names.
CANONICAL_NAME_TO_SPECIAL_KEY_MAP.put("||", "concat");
}
// The following function names are canonical names.
private static final String AND = "AND";
private static final String OR = "OR";
private static final String FILTER = "filter";
private static final String ASC = "asc";
private static final String DESC = "desc";
private static final String NULLS_FIRST = "nullsfirst";
private static final String NULLS_LAST = "nullslast";
private static final String COUNT = "count";
private static final String ARRAY_TO_MV = "arraytomv";

// --------------------------------------------------------------------------
// Relational conversion Utils
// --------------------------------------------------------------------------

public static List<Expression> convertProjectList(List<RexExpression> projectList, PinotQuery pinotQuery) {
List<Expression> selectExpr = new ArrayList<>();
final Iterator<RexExpression> iterator = projectList.iterator();
while (iterator.hasNext()) {
final RexExpression next = iterator.next();
selectExpr.add(toExpression(next, pinotQuery));
public static List<Expression> convertRexNodes(List<RexExpression> rexNodes, PinotQuery pinotQuery) {
List<Expression> expressions = new ArrayList<>(rexNodes.size());
for (RexExpression rexNode : rexNodes) {
expressions.add(toExpression(rexNode, pinotQuery));
}
return selectExpr;
return expressions;
}

public static List<Expression> convertAggregateList(List<Expression> groupSetList, List<RexExpression> aggCallList,
public static List<Expression> convertAggregateList(List<Expression> groupByList, List<RexExpression> aggCallList,
List<Integer> filterArgIndices, PinotQuery pinotQuery) {
List<Expression> selectExpr = new ArrayList<>(groupSetList);

for (int idx = 0; idx < aggCallList.size(); idx++) {
final RexExpression aggCall = aggCallList.get(idx);
int filterArgIdx = filterArgIndices.get(idx);
int numAggCalls = aggCallList.size();
List<Expression> expressions = new ArrayList<>(groupByList.size() + numAggCalls);
expressions.addAll(groupByList);
for (int i = 0; i < numAggCalls; i++) {
Expression aggFunction = toExpression(aggCallList.get(i), pinotQuery);
int filterArgIdx = filterArgIndices.get(i);
if (filterArgIdx == -1) {
selectExpr.add(toExpression(aggCall, pinotQuery));
expressions.add(aggFunction);
} else {
selectExpr.add(toExpression(new RexExpression.FunctionCall(SqlKind.FILTER, aggCall.getDataType(), "FILTER",
Arrays.asList(aggCall, new RexExpression.InputRef(filterArgIdx))), pinotQuery));
expressions.add(
RequestUtils.getFunctionExpression(FILTER, aggFunction, pinotQuery.getSelectList().get(filterArgIdx)));
}
}

return selectExpr;
}

public static List<Expression> convertGroupByList(List<RexExpression> rexNodeList, PinotQuery pinotQuery) {
List<Expression> groupByExpr = new ArrayList<>();

final Iterator<RexExpression> iterator = rexNodeList.iterator();
while (iterator.hasNext()) {
final RexExpression next = iterator.next();
groupByExpr.add(toExpression(next, pinotQuery));
}

return groupByExpr;
}

private static List<Expression> convertDistinctSelectList(RexExpression.FunctionCall rexCall, PinotQuery pinotQuery) {
List<Expression> selectExpr = new ArrayList<>();
selectExpr.add(convertDistinctAndSelectListToFunctionExpression(rexCall, pinotQuery));
return selectExpr;
return expressions;
}

public static List<Expression> convertOrderByList(SortNode node, PinotQuery pinotQuery) {
Expand All @@ -134,65 +98,28 @@ public static List<Expression> convertOrderByList(SortNode node, PinotQuery pino

private static Expression convertOrderBy(RexExpression rexNode, Direction direction, NullDirection nullDirection,
PinotQuery pinotQuery) {
Expression expression = toExpression(rexNode, pinotQuery);
if (direction == Direction.ASCENDING) {
Expression expression = getFunctionExpression("asc");
expression.getFunctionCall().addToOperands(toExpression(rexNode, pinotQuery));
Expression asc = RequestUtils.getFunctionExpression(ASC, expression);
// NOTE: Add explicit NULL direction only if it is not the default behavior (default behavior treats NULL as the
// largest value)
if (nullDirection == NullDirection.FIRST) {
Expression nullFirstExpression = getFunctionExpression("nullsfirst");
nullFirstExpression.getFunctionCall().addToOperands(expression);
return nullFirstExpression;
} else {
return expression;
}
return nullDirection == NullDirection.FIRST ? RequestUtils.getFunctionExpression(NULLS_FIRST, asc) : asc;
} else {
Expression expression = getFunctionExpression("desc");
expression.getFunctionCall().addToOperands(toExpression(rexNode, pinotQuery));
Expression desc = RequestUtils.getFunctionExpression(DESC, expression);
// NOTE: Add explicit NULL direction only if it is not the default behavior (default behavior treats NULL as the
// largest value)
if (nullDirection == NullDirection.LAST) {
Expression nullLastExpression = getFunctionExpression("nullslast");
nullLastExpression.getFunctionCall().addToOperands(expression);
return nullLastExpression;
} else {
return expression;
}
}
}

private static Expression convertDistinctAndSelectListToFunctionExpression(RexExpression.FunctionCall rexCall,
PinotQuery pinotQuery) {
Expression functionExpression = getFunctionExpression("distinct");
for (RexExpression node : rexCall.getFunctionOperands()) {
Expression columnExpression = toExpression(node, pinotQuery);
if (columnExpression.getType() == ExpressionType.IDENTIFIER && columnExpression.getIdentifier().getName()
.equals("*")) {
throw new SqlCompilationException(
"Syntax error: Pinot currently does not support DISTINCT with *. Please specify each column name after "
+ "DISTINCT keyword");
} else if (columnExpression.getType() == ExpressionType.FUNCTION) {
Function functionCall = columnExpression.getFunctionCall();
String function = functionCall.getOperator();
if (AggregationFunctionType.isAggregationFunction(function)) {
throw new SqlCompilationException(
"Syntax error: Use of DISTINCT with aggregation functions is not supported");
}
}
functionExpression.getFunctionCall().addToOperands(columnExpression);
return nullDirection == NullDirection.LAST ? RequestUtils.getFunctionExpression(NULLS_LAST, desc) : desc;
}
return functionExpression;
}

public static Expression toExpression(RexExpression rexNode, PinotQuery pinotQuery) {
LOGGER.debug("Current processing RexNode: {}, node.getKind(): {}", rexNode, rexNode.getKind());
switch (rexNode.getKind()) {
case INPUT_REF:
return inputRefToIdentifier((RexExpression.InputRef) rexNode, pinotQuery);
case LITERAL:
return compileLiteralExpression(((RexExpression.Literal) rexNode).getValue());
default:
return compileFunctionExpression((RexExpression.FunctionCall) rexNode, pinotQuery);
if (rexNode instanceof RexExpression.InputRef) {
return inputRefToIdentifier((RexExpression.InputRef) rexNode, pinotQuery);
} else if (rexNode instanceof RexExpression.Literal) {
return compileLiteralExpression(((RexExpression.Literal) rexNode).getValue());
} else {
assert rexNode instanceof RexExpression.FunctionCall;
return compileFunctionExpression((RexExpression.FunctionCall) rexNode, pinotQuery);
}
}

Expand All @@ -213,41 +140,24 @@ private static Expression inputRefToIdentifier(RexExpression.InputRef inputRef,
}

private static Expression compileFunctionExpression(RexExpression.FunctionCall rexCall, PinotQuery pinotQuery) {
SqlKind functionKind = rexCall.getKind();
String functionName;
switch (functionKind) {
case AND:
return compileAndExpression(rexCall, pinotQuery);
case OR:
return compileOrExpression(rexCall, pinotQuery);
case OTHER:
case OTHER_FUNCTION:
functionName = canonicalizeFunctionName(rexCall.getFunctionName());
// Special handle for leaf stage multi-value columns, as the default behavior for filter and group by is not
// sql standard, so need to use `array_to_mv` to convert the array to v1 multi-value column for behavior
// consistency meanwhile not violating the sql standard.
if (ARRAY_TO_MV_FUNCTION_NAME.equals(functionName)) {
return toExpression(rexCall.getFunctionOperands().get(0), pinotQuery);
}
break;
default:
functionName = canonicalizeFunctionName(functionKind.name());
break;
String functionName = rexCall.getFunctionName();
if (functionName.equals(AND)) {
return compileAndExpression(rexCall, pinotQuery);
}
if (functionName.equals(OR)) {
return compileOrExpression(rexCall, pinotQuery);
}
String canonicalName = RequestUtils.canonicalizeFunctionNamePreservingSpecialKey(functionName);
List<RexExpression> childNodes = rexCall.getFunctionOperands();
List<Expression> operands = new ArrayList<>(childNodes.size());
for (RexExpression childNode : childNodes) {
operands.add(toExpression(childNode, pinotQuery));
if (canonicalName.equals(COUNT) && childNodes.isEmpty()) {
return RequestUtils.getFunctionExpression(COUNT, RequestUtils.getIdentifierExpression("*"));
}
// for COUNT, add a star (*) identifier to operand list b/c V1 doesn't handle empty operand functions.
if (functionKind == SqlKind.COUNT && operands.isEmpty()) {
operands.add(RequestUtils.getIdentifierExpression("*"));
} else {
ParserUtils.validateFunction(functionName, operands);
if (canonicalName.equals(ARRAY_TO_MV)) {
return toExpression(childNodes.get(0), pinotQuery);
}
Expression functionExpression = getFunctionExpression(functionName);
functionExpression.getFunctionCall().setOperands(operands);
return functionExpression;
List<Expression> operands = convertRexNodes(childNodes, pinotQuery);
ParserUtils.validateFunction(canonicalName, operands);
return RequestUtils.getFunctionExpression(canonicalName, operands);
}

/**
Expand All @@ -256,16 +166,15 @@ private static Expression compileFunctionExpression(RexExpression.FunctionCall r
private static Expression compileAndExpression(RexExpression.FunctionCall andNode, PinotQuery pinotQuery) {
List<Expression> operands = new ArrayList<>();
for (RexExpression childNode : andNode.getFunctionOperands()) {
if (childNode.getKind() == SqlKind.AND) {
if (childNode instanceof RexExpression.FunctionCall && ((RexExpression.FunctionCall) childNode).getFunctionName()
.equals(AND)) {
Expression childAndExpression = compileAndExpression((RexExpression.FunctionCall) childNode, pinotQuery);
operands.addAll(childAndExpression.getFunctionCall().getOperands());
} else {
operands.add(toExpression(childNode, pinotQuery));
}
}
Expression andExpression = getFunctionExpression(SqlKind.AND.name());
andExpression.getFunctionCall().setOperands(operands);
return andExpression;
return RequestUtils.getFunctionExpression(AND, operands);
}

/**
Expand All @@ -274,27 +183,14 @@ private static Expression compileAndExpression(RexExpression.FunctionCall andNod
private static Expression compileOrExpression(RexExpression.FunctionCall orNode, PinotQuery pinotQuery) {
List<Expression> operands = new ArrayList<>();
for (RexExpression childNode : orNode.getFunctionOperands()) {
if (childNode.getKind() == SqlKind.OR) {
if (childNode instanceof RexExpression.FunctionCall && ((RexExpression.FunctionCall) childNode).getFunctionName()
.equals(OR)) {
Expression childAndExpression = compileOrExpression((RexExpression.FunctionCall) childNode, pinotQuery);
operands.addAll(childAndExpression.getFunctionCall().getOperands());
} else {
operands.add(toExpression(childNode, pinotQuery));
}
}
Expression andExpression = getFunctionExpression(SqlKind.OR.name());
andExpression.getFunctionCall().setOperands(operands);
return andExpression;
}

private static Expression getFunctionExpression(String canonicalName) {
Expression expression = new Expression(ExpressionType.FUNCTION);
Function function = new Function(canonicalName);
expression.setFunctionCall(function);
return expression;
}

private static String canonicalizeFunctionName(String functionName) {
String canonicalizeName = RequestUtils.canonicalizeFunctionName(functionName);
return CANONICAL_NAME_TO_SPECIAL_KEY_MAP.getOrDefault(canonicalizeName, canonicalizeName);
return RequestUtils.getFunctionExpression(OR, operands);
}
}
5 changes: 2 additions & 3 deletions ...-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ private static PlanNode convertLogicalAggregate(PinotLogicalAggregate node, int

private static PlanNode convertLogicalProject(LogicalProject node, int currentStageId) {
return new ProjectNode(currentStageId, toDataSchema(node.getRowType()),
node.getProjects().stream().map(RexExpressionUtils::fromRexNode).collect(Collectors.toList()));
RexExpressionUtils.fromRexNodes(node.getProjects()));
}

private static PlanNode convertLogicalFilter(LogicalFilter node, int currentStageId) {
Expand All @@ -196,8 +196,7 @@ private static PlanNode convertLogicalJoin(LogicalJoin node, int currentStageId)
// Parse out all equality JOIN conditions
JoinInfo joinInfo = node.analyzeCondition();
JoinNode.JoinKeys joinKeys = new JoinNode.JoinKeys(joinInfo.leftKeys, joinInfo.rightKeys);
List<RexExpression> joinClause =
joinInfo.nonEquiConditions.stream().map(RexExpressionUtils::fromRexNode).collect(Collectors.toList());
List<RexExpression> joinClause = RexExpressionUtils.fromRexNodes(joinInfo.nonEquiConditions);
return new JoinNode(currentStageId, toDataSchema(node.getRowType()), toDataSchema(node.getLeft().getRowType()),
toDataSchema(node.getRight().getRowType()), joinType, joinKeys, joinClause, node.getHints());
}
Expand Down
Loading