Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
c26ff17
Support type checker For PPL functions
qianheng-aws Apr 18, 2025
e639dcf
Implement type checkers for UDF
yuancu Apr 25, 2025
a7f252f
Implement composite type checker
yuancu Apr 25, 2025
a887e2d
Merge origin/main into type-checker
yuancu May 14, 2025
e1e4deb
Define type checkers for UDFs
yuancu May 14, 2025
fe0385c
Display expected signatures when type checking fails & support type c…
yuancu May 16, 2025
f3f94ce
Add ITs for UDF type checking
yuancu May 16, 2025
5733337
Merge remote-tracking branch 'origin/main' into type-checker
yuancu May 16, 2025
f059c44
Fix: correct CONV's operand types
yuancu May 16, 2025
d288f38
Fix test parse IT
yuancu May 16, 2025
a11381b
Remove unused code in UserDefinedFunctionUtils
yuancu May 16, 2025
4c66bbb
Refactor: improve ITs, javadocs, type checkers
yuancu May 22, 2025
138b67b
Improve javadoc for PPLTypeChecker
yuancu May 22, 2025
783cdb1
Return false at validation for mismatched operand count with expectat…
yuancu May 22, 2025
8bba4b7
Merge remote-tracking branch 'origin/main' into type-checker
yuancu May 22, 2025
f6107b1
Add type checker for cidrmatch
yuancu May 22, 2025
882cb88
Correct cidrmatch type checker
yuancu May 22, 2025
d34857f
Fix compile error in CalcitePPLAbstractTest.java
yuancu May 22, 2025
77d49a1
Experiment: support type checking for built-in operators by accessing…
yuancu May 22, 2025
1bf526d
Check composition type for only calcite's built-in operators
yuancu May 23, 2025
714a39d
Support parameter validation for comparabale operators
yuancu May 26, 2025
2376ca6
Support parameter validation for coalesce (by reusing comparable type…
yuancu May 26, 2025
281f755
Support parameter validation for substring
yuancu May 26, 2025
45f1566
Support parameter validation for if
yuancu May 26, 2025
638c3d6
Support parameter validation for nullif, isempty, isblank
yuancu May 26, 2025
df71074
Merge remote-tracking branch 'origin/main' into type-checker
yuancu May 26, 2025
139d858
Align types in type check error information to actual types in PPL
yuancu May 27, 2025
85e0b31
Display PPL types when failing to resolve a function
yuancu May 28, 2025
63c94a4
Merge origin/main into type-checker
yuancu Jun 4, 2025
18921bd
Create type checker for geoip function
yuancu Jun 4, 2025
e3a491d
Merge origin/main into type-checker
yuancu Jun 5, 2025
d9669ab
Define type checker for grok and item operators
yuancu Jun 5, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ public static ExprType convertSqlTypeNameToExprType(SqlTypeName sqlTypeName) {
case ARRAY -> ARRAY;
case MAP -> STRUCT;
case GEOMETRY -> GEO_POINT;
case NULL -> UNDEFINED;
case NULL, ANY -> UNDEFINED;
default -> UNKNOWN;
};
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
*
* * Copyright OpenSearch Contributors
* * SPDX-License-Identifier: Apache-2.0
*
*/

package org.opensearch.sql.calcite.utils;

import org.apache.calcite.sql.type.CompositeOperandTypeChecker;
import org.apache.calcite.sql.type.FamilyOperandTypeChecker;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.opensearch.sql.expression.function.UDFOperandMetadata;

/**
* This class contains common operand types for PPL functions. They are created by either wrapping a
* {@link FamilyOperandTypeChecker} or a {@link CompositeOperandTypeChecker} with a {@link
* UDFOperandMetadata}.
*/
public class PPLOperandTypes {
// This class is not meant to be instantiated.
private PPLOperandTypes() {}

public static final UDFOperandMetadata NONE = UDFOperandMetadata.wrap(OperandTypes.family());
public static final UDFOperandMetadata OPTIONAL_INTEGER =
UDFOperandMetadata.wrap(
(CompositeOperandTypeChecker) OperandTypes.INTEGER.or(OperandTypes.family()));
public static final UDFOperandMetadata STRING =
UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.STRING);
public static final UDFOperandMetadata INTEGER =
UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.INTEGER);
public static final UDFOperandMetadata NUMERIC =
UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.NUMERIC);
public static final UDFOperandMetadata INTEGER_INTEGER =
UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.INTEGER_INTEGER);
public static final UDFOperandMetadata STRING_STRING =
UDFOperandMetadata.wrap(OperandTypes.STRING_STRING);
public static final UDFOperandMetadata NUMERIC_NUMERIC =
UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC);
public static final UDFOperandMetadata NUMERIC_NUMERIC_NUMERIC =
UDFOperandMetadata.wrap(
OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC));

public static final UDFOperandMetadata DATETIME_OR_STRING =
UDFOperandMetadata.wrap(
(CompositeOperandTypeChecker) OperandTypes.DATETIME.or(OperandTypes.STRING));
public static final UDFOperandMetadata DATETIME_DATETIME =
UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME));
public static final UDFOperandMetadata DATETIME_OR_STRING_DATETIME_OR_STRING =
UDFOperandMetadata.wrap(
(CompositeOperandTypeChecker)
OperandTypes.STRING_STRING
.or(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME))
.or(OperandTypes.family(SqlTypeFamily.DATETIME, SqlTypeFamily.STRING))
.or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.DATETIME)));
public static final UDFOperandMetadata TIME_OR_TIMESTAMP_OR_STRING =
UDFOperandMetadata.wrap(
(CompositeOperandTypeChecker)
OperandTypes.STRING.or(OperandTypes.TIME).or(OperandTypes.TIMESTAMP));
public static final UDFOperandMetadata DATE_OR_TIMESTAMP_OR_STRING =
UDFOperandMetadata.wrap(
(CompositeOperandTypeChecker) OperandTypes.DATE_OR_TIMESTAMP.or(OperandTypes.STRING));
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.util.Collections;
import java.util.List;
import java.util.TimeZone;
import javax.annotation.Nullable;
import org.apache.calcite.DataContext;
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
import org.apache.calcite.adapter.enumerable.NullPolicy;
Expand All @@ -34,12 +35,14 @@
import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.util.Optionality;
import org.opensearch.sql.calcite.type.AbstractExprRelDataType;
import org.opensearch.sql.calcite.udf.UserDefinedAggFunction;
import org.opensearch.sql.data.model.ExprValueUtils;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.executor.QueryType;
import org.opensearch.sql.expression.function.FunctionProperties;
import org.opensearch.sql.expression.function.ImplementorUDF;
import org.opensearch.sql.expression.function.UDFOperandMetadata;

public class UserDefinedFunctionUtils {

Expand Down Expand Up @@ -88,7 +91,20 @@ static SqlReturnTypeInference getReturnTypeInferenceForArray() {
};
}

// TODO: pass the function properties directly to the UDF instead of string
public static SqlTypeName convertRelDataTypeToSqlTypeName(RelDataType type) {
if (type instanceof AbstractExprRelDataType<?> exprType) {
return switch (exprType.getUdt()) {
case EXPR_DATE -> SqlTypeName.DATE;
case EXPR_TIME -> SqlTypeName.TIME;
case EXPR_TIMESTAMP -> SqlTypeName.TIMESTAMP;
case EXPR_IP -> SqlTypeName.VARCHAR;
case EXPR_BINARY -> SqlTypeName.VARBINARY;
default -> type.getSqlTypeName();
};
}
return type.getSqlTypeName();
}

public static FunctionProperties restoreFunctionProperties(DataContext dataContext) {
long currentTimeInNanos = DataContext.Variable.UTC_TIMESTAMP.get(dataContext);
Instant instant =
Expand Down Expand Up @@ -148,13 +164,15 @@ public static List<Expression> convertToExprValues(
* @param methodName the name of the method
* @param returnTypeInference the return type inference of the UDF
* @param nullPolicy the null policy of the UDF
* @param operandMetadata type checker
* @return an adapted ImplementorUDF with the expr method, which is a UserDefinedFunctionBuilder
*/
public static ImplementorUDF adaptExprMethodToUDF(
java.lang.reflect.Type type,
String methodName,
SqlReturnTypeInference returnTypeInference,
NullPolicy nullPolicy) {
NullPolicy nullPolicy,
@Nullable UDFOperandMetadata operandMetadata) {
NotNullImplementor implementor =
(translator, call, translatedOperands) -> {
List<Expression> operands =
Expand All @@ -168,6 +186,11 @@ public static ImplementorUDF adaptExprMethodToUDF(
public SqlReturnTypeInference getReturnTypeInference() {
return returnTypeInference;
}

@Override
public UDFOperandMetadata getOperandMetadata() {
return operandMetadata;
}
};
}

Expand All @@ -185,7 +208,8 @@ public static ImplementorUDF adaptExprMethodWithPropertiesToUDF(
java.lang.reflect.Type type,
String methodName,
SqlReturnTypeInference returnTypeInference,
NullPolicy nullPolicy) {
NullPolicy nullPolicy,
UDFOperandMetadata operandMetadata) {
NotNullImplementor implementor =
(translator, call, translatedOperands) -> {
List<Expression> operands =
Expand All @@ -200,6 +224,11 @@ public static ImplementorUDF adaptExprMethodWithPropertiesToUDF(
public SqlReturnTypeInference getReturnTypeInference() {
return returnTypeInference;
}

@Override
public UDFOperandMetadata getOperandMetadata() {
return operandMetadata;
}
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,8 @@ public class CalciteUnsupportedException extends QueryEngineException {
public CalciteUnsupportedException(String message) {
super(message);
}

public CalciteUnsupportedException(String message, Throwable cause) {
super(message, cause);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@

/** Exception for Expression Evaluation. */
public class ExpressionEvaluationException extends QueryEngineException {

public ExpressionEvaluationException(String message) {
super(message);
}

public ExpressionEvaluationException(String message, Throwable cause) {
super(message, cause);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ public void executeWithCalcite(
} else {
if (t instanceof Error) {
// Calcite may throw AssertError during query execution.
listener.onFailure(new CalciteUnsupportedException(t.getMessage()));
listener.onFailure(new CalciteUnsupportedException(t.getMessage(), t));
} else {
listener.onFailure((Exception) t);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,18 @@

package org.opensearch.sql.expression.function;

import static org.opensearch.sql.expression.function.PPLFuncImpTable.FunctionImp.ANY_TYPE;

import java.util.List;
import org.apache.calcite.rel.type.RelDataType;

/** Function signature is composed by function name and arguments list. */
public record CalciteFuncSignature(FunctionName functionName, List<RelDataType> funcArgTypes) {
public record CalciteFuncSignature(FunctionName functionName, PPLTypeChecker typeChecker) {

public boolean match(FunctionName functionName, List<RelDataType> paramTypeList) {
if (funcArgTypes == null) return true;
if (!functionName.equals(this.functionName()) || paramTypeList.size() != funcArgTypes.size()) {
return false;
}
for (int i = 0; i < paramTypeList.size(); i++) {
RelDataType paramType = paramTypeList.get(i);
RelDataType funcType = funcArgTypes.get(i);
if (ANY_TYPE != funcType && paramType.getFamily() != funcType.getFamily()) {
return false;
}
}
return true;
if (!functionName.equals(this.functionName())) return false;
// For complex type checkers (e.g., OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED),
// the typeChecker will be null because only simple family-based type checks are currently
// supported.
if (typeChecker == null) return true;
return typeChecker.checkOperandTypes(paramTypeList);
}
}
Loading
Loading