Skip to content

Commit

Permalink
[CALCITE-6730] Add CONVERT function(enabled in Oracle library)
Browse files Browse the repository at this point in the history
  • Loading branch information
ILuffZhe committed Dec 30, 2024
1 parent 4c70fc7 commit 901aadb
Show file tree
Hide file tree
Showing 16 changed files with 345 additions and 15 deletions.
15 changes: 11 additions & 4 deletions core/src/main/codegen/templates/Parser.jj
Original file line number Diff line number Diff line change
Expand Up @@ -6319,10 +6319,17 @@ SqlNode BuiltinFunctionCall() :
}
|
<COMMA> e = SimpleIdentifier() { args.add(e); }
<COMMA> e = SimpleIdentifier() { args.add(e); }
<RPAREN> {
return SqlStdOperatorTable.CONVERT.createCall(s.end(this), args);
}
(
<COMMA> e = SimpleIdentifier() { args.add(e); }
<RPAREN> {
SqlOperator op = SqlStdOperatorTable.getConvertFuncByConformance(this.conformance);
return op.createCall(s.end(this), args);
}
|
<RPAREN> {
return SqlLibraryOperators.CONVERT_ORACLE.createCall(s.end(this), args);
}
)
)
|
// MSSql CONVERT(type, val [,style])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@
import static org.apache.calcite.sql.fun.SqlLibraryOperators.CONCAT_WS_POSTGRESQL;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.CONCAT_WS_SPARK;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.CONTAINS_SUBSTR;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.CONVERT_ORACLE;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.COSD;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.COSH;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.COTH;
Expand Down Expand Up @@ -740,6 +741,7 @@ void populate1() {
defineMethod(CONCAT_WS_SPARK,
BuiltInMethod.MULTI_TYPE_STRING_ARRAY_CONCAT_WITH_SEPARATOR.method,
NullPolicy.ARG0);
defineMethod(CONVERT_ORACLE, BuiltInMethod.CONVERT_ORACLE.method, NullPolicy.ARG0);
defineMethod(OVERLAY, BuiltInMethod.OVERLAY.method, NullPolicy.STRICT);
defineMethod(POSITION, BuiltInMethod.POSITION.method, NullPolicy.STRICT);
defineMethod(ASCII, BuiltInMethod.ASCII.method, NullPolicy.STRICT);
Expand Down
23 changes: 23 additions & 0 deletions core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
Original file line number Diff line number Diff line change
Expand Up @@ -1681,6 +1681,29 @@ public static String translateWithCharset(String s, String transcodingName) {
}
}

/** Oracle's {@code CONVERT(charValue, destCharsetName[, srcCharsetName])} function,
* return null if s is null or empty. */
public static String convertOracle(String s, String... args) {
final Charset src;
final Charset dest;
if (args.length == 1) {
// srcCharsetName is not specified
src = Charset.defaultCharset();
dest = SqlUtil.getCharset(args[0]);
} else {
dest = SqlUtil.getCharset(args[0]);
src = SqlUtil.getCharset(args[1]);
}
byte[] bytes = s.getBytes(src);
final CharsetDecoder decoder = dest.newDecoder();
final ByteBuffer buffer = ByteBuffer.wrap(bytes);
try {
return decoder.decode(buffer).toString();
} catch (CharacterCodingException ex) {
throw RESOURCE.charsetEncoding(s, dest.name()).ex();
}
}

/** State for {@code PARSE_URL}. */
@Deterministic
public static class ParseUrlFunction {
Expand Down
7 changes: 5 additions & 2 deletions core/src/main/java/org/apache/calcite/sql/SqlKind.java
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ public enum SqlKind {
/** {@code CONVERT} function. */
CONVERT,

/** Oracle's {@code CONVERT} function. */
CONVERT_ORACLE,

/** {@code TRANSLATE} function. */
TRANSLATE,

Expand Down Expand Up @@ -1455,8 +1458,8 @@ public enum SqlKind {
public static final Set<SqlKind> EXPRESSION =
EnumSet.complementOf(
concat(
EnumSet.of(AS, ARGUMENT_ASSIGNMENT, CONVERT, TRANSLATE, DEFAULT,
RUNNING, FINAL, LAST, FIRST, PREV, NEXT,
EnumSet.of(AS, ARGUMENT_ASSIGNMENT, CONVERT, CONVERT_ORACLE, TRANSLATE,
DEFAULT, RUNNING, FINAL, LAST, FIRST, PREV, NEXT,
FILTER, WITHIN_GROUP, IGNORE_NULLS, RESPECT_NULLS, SEPARATOR,
DESCENDING, CUBE, ROLLUP, GROUPING_SETS, EXTEND, LATERAL,
SELECT, JOIN, OTHER_FUNCTION, POSITION, CAST, TRIM, FLOOR, CEIL,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,10 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding,
public static final SqlFunction SUBSTR_ORACLE =
SUBSTR.withKind(SqlKind.SUBSTR_ORACLE);

@LibraryOperator(libraries = {ORACLE})
public static final SqlFunction CONVERT_ORACLE =
new SqlOracleConvertFunction("CONVERT");

/** PostgreSQL's "SUBSTR(string, position [, substringLength ])" function. */
@LibraryOperator(libraries = {POSTGRESQL})
public static final SqlFunction SUBSTR_POSTGRESQL =
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.calcite.sql.fun;

import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexCallBinding;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlCallBinding;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlOperandCountRange;
import org.apache.calcite.sql.SqlOperatorBinding;
import org.apache.calcite.sql.SqlUtil;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlOperandCountRanges;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;

import java.nio.charset.Charset;
import java.util.List;

import static org.apache.calcite.sql.type.NonNullableAccessors.getCollation;

import static java.util.Objects.requireNonNull;

/** Oracle's "CONVERT(charValue, destCharsetName[, srcCharsetName])" function.
*
* <p>It has a slight different semantics to standard SQL's
* {@link SqlStdOperatorTable#CONVERT} function on operands' order, and default
* charset will be used if the {@code srcCharsetName} is not specified.
*
* <p>Returns null if {@code charValue} is null or empty. */
public class SqlOracleConvertFunction extends SqlConvertFunction {
//~ Constructors -----------------------------------------------------------

public SqlOracleConvertFunction(String name) {
super(name, SqlKind.CONVERT_ORACLE, ReturnTypes.ARG0, null, null,
SqlFunctionCategory.STRING);
}

//~ Methods ----------------------------------------------------------------

@Override public void validateCall(SqlCall call, SqlValidator validator,
SqlValidatorScope scope, SqlValidatorScope operandScope) {
final List<SqlNode> operands = call.getOperandList();
operands.get(0).validateExpr(validator, scope);
// validate if the Charsets are legal.
assert operands.get(1) instanceof SqlIdentifier;
final String src_charset = operands.get(1).toString();
SqlUtil.getCharset(src_charset);
if (operands.size() == 3) {
assert operands.get(2) instanceof SqlIdentifier;
final String dest_charset = operands.get(2).toString();
SqlUtil.getCharset(dest_charset);
}
super.validateQuantifier(validator, call);
}

@Override public RelDataType inferReturnType(
SqlOperatorBinding opBinding) {
final RelDataType ret = opBinding.getOperandType(0);
if (SqlTypeUtil.isNull(ret)) {
return ret;
}
final String descCharsetName;
if (opBinding instanceof SqlCallBinding) {
descCharsetName = ((SqlCallBinding) opBinding).getCall().operand(1).toString();
} else {
descCharsetName = ((RexCallBinding) opBinding).getStringLiteralOperand(1);
}
assert descCharsetName != null;
Charset descCharset = SqlUtil.getCharset(descCharsetName);
return opBinding
.getTypeFactory().createTypeWithCharsetAndCollation(ret, descCharset, getCollation(ret));
}

@Override public RelDataType deriveType(SqlValidator validator,
SqlValidatorScope scope, SqlCall call) {
RelDataType nodeType =
validator.deriveType(scope, call.operand(0));
requireNonNull(nodeType, "nodeType");
RelDataType ret = validateOperands(validator, scope, call);
if (SqlTypeUtil.isNull(ret)) {
return ret;
}
Charset descCharset = SqlUtil.getCharset(call.operand(1).toString());
return validator.getTypeFactory()
.createTypeWithCharsetAndCollation(ret, descCharset, getCollation(ret));
}

@Override public String getSignatureTemplate(final int operandsCount) {
switch (operandsCount) {
case 2:
return "{0}({1}, {2})";
case 3:
return "{0}({1}, {2}, {3})";
default:
throw new IllegalStateException("operandsCount should be 2 or 3, got "
+ operandsCount);
}
}

@Override public SqlOperandCountRange getOperandCountRange() {
return SqlOperandCountRanges.between(2, 3);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2832,4 +2832,15 @@ public static SqlOperator floorCeil(boolean floor, SqlConformance conformance) {
return floor ? SqlStdOperatorTable.FLOOR : SqlStdOperatorTable.CEIL;
}
}

/** Returns the operator for standard {@code CONVERT} and Oracle's {@code CONVERT}
* with the given library. */
public static SqlOperator getConvertFuncByConformance(SqlConformance conformance) {
if (SqlConformanceEnum.ORACLE_10 == conformance
|| SqlConformanceEnum.ORACLE_12 == conformance) {
return SqlLibraryOperators.CONVERT_ORACLE;
} else {
return SqlStdOperatorTable.CONVERT;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4287,8 +4287,10 @@ private void checkRollUp(@Nullable SqlNode grandParent, @Nullable SqlNode parent
// can be another SqlCall, or an SqlIdentifier.
checkRollUp(grandParent, parent, stripDot, scope, contextClause);
} else if (stripDot.getKind() == SqlKind.CONVERT
|| stripDot.getKind() == SqlKind.TRANSLATE) {
// only need to check operand[0] for CONVERT or TRANSLATE
|| stripDot.getKind() == SqlKind.TRANSLATE
|| stripDot.getKind() == SqlKind.CONVERT_ORACLE) {
// only need to check operand[0] for
// CONVERT, TRANSLATE or CONVERT_ORACLE
SqlNode child = ((SqlCall) stripDot).getOperandList().get(0);
checkRollUp(parent, current, child, scope, contextClause);
} else if (stripDot.getKind() == SqlKind.LAMBDA) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,7 @@ private StandardConvertletTable() {
(cx, call) -> cx.convertExpression(call.operand(0)));

registerOp(SqlStdOperatorTable.CONVERT, this::convertCharset);
registerOp(SqlLibraryOperators.CONVERT_ORACLE, this::convertCharset);
registerOp(SqlStdOperatorTable.TRANSLATE, this::translateCharset);
// "SQRT(x)" is equivalent to "POWER(x, .5)"
registerOp(SqlStdOperatorTable.SQRT,
Expand Down Expand Up @@ -855,15 +856,38 @@ protected RexNode convertFloorCeil(SqlRexContext cx, SqlCall call) {
protected RexNode convertCharset(
@UnknownInitialization StandardConvertletTable this,
SqlRexContext cx, SqlCall call) {
final RexBuilder rexBuilder = cx.getRexBuilder();
final SqlParserPos pos = call.getParserPosition();
final SqlNode expr = call.operand(0);
final String srcCharset = call.operand(1).toString();
final String destCharset = call.operand(2).toString();
final RexBuilder rexBuilder = cx.getRexBuilder();
return rexBuilder.makeCall(pos, SqlStdOperatorTable.CONVERT,
cx.convertExpression(expr),
rexBuilder.makeLiteral(srcCharset),
rexBuilder.makeLiteral(destCharset));
final SqlOperator op = call.getOperator();
final String srcCharset;
final String destCharset;
switch (op.getKind()) {
case CONVERT:
srcCharset = call.operand(1).toString();
destCharset = call.operand(2).toString();
return rexBuilder.makeCall(pos, op,
cx.convertExpression(expr),
rexBuilder.makeLiteral(srcCharset),
rexBuilder.makeLiteral(destCharset));
case CONVERT_ORACLE:
destCharset = call.operand(1).toString();
switch (call.operandCount()) {
case 2:
// when srcCharsetName is not specified
return rexBuilder.makeCall(pos, op,
cx.convertExpression(expr),
rexBuilder.makeLiteral(destCharset));
default:
srcCharset = call.operand(2).toString();
return rexBuilder.makeCall(pos, op,
cx.convertExpression(expr),
rexBuilder.makeLiteral(destCharset),
rexBuilder.makeLiteral(srcCharset));
}
default:
throw Util.unexpected(op.getKind());
}
}

protected RexNode translateCharset(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ public enum BuiltInMethod {
TO_CODE_POINTS(SqlFunctions.class, "toCodePoints", String.class),
CONVERT(SqlFunctions.class, "convertWithCharset", String.class, String.class,
String.class),
CONVERT_ORACLE(SqlFunctions.class, "convertOracle", String.class, String[].class),
EXP(SqlFunctions.class, "exp", double.class),
MOD(SqlFunctions.class, "mod", long.class, long.class),
POWER(SqlFunctions.class, "power", double.class, double.class),
Expand Down
42 changes: 42 additions & 0 deletions core/src/test/java/org/apache/calcite/test/JdbcTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.parser.impl.SqlParserImpl;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.validate.SqlConformanceEnum;
import org.apache.calcite.sql2rel.SqlToRelConverter.Config;
import org.apache.calcite.test.schemata.catchall.CatchallSchema;
import org.apache.calcite.test.schemata.foodmart.FoodmartSchema;
Expand Down Expand Up @@ -7352,6 +7353,47 @@ private void checkGetTimestamp(Connection con) throws SQLException {
.throws_("No match found for function signature NVL(<NUMERIC>, <NUMERIC>)");
}

/** Test case for
* <a href="https://issues.apache.org/jira/browse/CALCITE-6730">[CALCITE-6730]
* Add CONVERT function(enabled in Oracle library)</a>. */
@Test void testConvertOracle() {
CalciteAssert.AssertThat withOracle10 =
CalciteAssert.hr()
.with(SqlConformanceEnum.ORACLE_10);
testConvertOracleInternal(withOracle10);

CalciteAssert.AssertThat withOracle12
= withOracle10.with(SqlConformanceEnum.ORACLE_12);
testConvertOracleInternal(withOracle12);
}

private void testConvertOracleInternal(CalciteAssert.AssertThat with) {
with.query("select \"name\", \"empid\" from \"hr\".\"emps\"\n"
+ "where convert(\"name\", GBK)=_GBK'Eric'")
.returns("name=Eric; empid=200\n");
with.query("select \"name\", \"empid\" from \"hr\".\"emps\"\n"
+ "where _BIG5'Eric'=convert(\"name\", LATIN1)")
.throws_("Cannot apply operation '=' to strings with "
+ "different charsets 'Big5' and 'ISO-8859-1'");
// use LATIN1 as dest charset, not BIG5
with.query("select \"name\", \"empid\" from \"hr\".\"emps\"\n"
+ "where _BIG5'Eric'=convert(\"name\", LATIN1, BIG5)")
.throws_("Cannot apply operation '=' to strings with "
+ "different charsets 'Big5' and 'ISO-8859-1'");

// check cast
with.query("select \"name\", \"empid\" from \"hr\".\"emps\"\n"
+ "where cast(convert(\"name\", LATIN1, UTF8) as varchar)='Eric'")
.returns("name=Eric; empid=200\n");
// the result of convert(\"name\", GBK) has GBK charset
// while CHAR(5) has ISO-8859-1 charset, which is not allowed to cast
with.query("select \"name\", \"empid\" from \"hr\".\"emps\"\n"
+ "where cast(convert(\"name\", GBK) as varchar)='Eric'")
.throws_(
"cannot convert value of type "
+ "JavaType(class java.lang.String CHARACTER SET \"GBK\") to type VARCHAR NOT NULL");
}

@Test void testIf() {
CalciteAssert.that(CalciteAssert.Config.REGULAR)
.with(CalciteConnectionProperty.FUN, "bigquery")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import static org.apache.calcite.runtime.SqlFunctions.concatMultiWithNull;
import static org.apache.calcite.runtime.SqlFunctions.concatMultiWithSeparator;
import static org.apache.calcite.runtime.SqlFunctions.concatWithNull;
import static org.apache.calcite.runtime.SqlFunctions.convertOracle;
import static org.apache.calcite.runtime.SqlFunctions.fromBase64;
import static org.apache.calcite.runtime.SqlFunctions.greater;
import static org.apache.calcite.runtime.SqlFunctions.initcap;
Expand Down Expand Up @@ -319,6 +320,11 @@ static <E> List<E> list() {
assertThat(concatMultiObjectWithSeparator("abc", null, null), is(""));
}

@Test void testConvertOracle() {
assertThat(convertOracle("a", "UTF8", "LATIN1"), is("a"));
assertThat(convertOracle("a", "UTF8"), is("a"));
}

@Test void testPosixRegex() {
final SqlFunctions.PosixRegexFunction f =
new SqlFunctions.PosixRegexFunction();
Expand Down
Loading

0 comments on commit 901aadb

Please sign in to comment.