Skip to content

Commit

Permalink
KYLIN-3700 Quote sql identities when creating flat table
Browse files Browse the repository at this point in the history
  • Loading branch information
@ZhengshuaiPENG @shaofengshi
ZhengshuaiPENG authored and shaofengshi committed Nov 28, 2018
1 parent 97ee98b commit 4158d7b
Show file tree
Hide file tree
Showing 22 changed files with 921 additions and 165 deletions.
5 changes: 3 additions & 2 deletions core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ public String build() {
case CLI:
buf.append("hive -e \"");
for (String statement : statements) {
buf.append(statement).append("\n");
//in bash need escape " and ` by using \
buf.append(statement.replaceAll("`", "\\\\`")).append("\n");
}
buf.append("\"");
buf.append(parseProps());
Expand All @@ -79,7 +80,7 @@ public String build() {
try {
tmpHqlPath = "/tmp/" + UUID.randomUUID().toString() + ".hql";
for (String statement : statements) {
hql.append(statement);
hql.append(statement.replaceAll("`", "\\\\`"));
hql.append("\n");
}
String createFileCmd = String.format(Locale.ROOT, CREATE_HQL_TMP_FILE_TEMPLATE, tmpHqlPath, hql);
Expand Down
10 changes: 5 additions & 5 deletions core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ public void testHiveCLI() {
hivePropsOverwrite.put("hive.execution.engine", "tez");
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
hiveCmdBuilder.addStatement("DROP TABLE `test`;");
hiveCmdBuilder.addStatement("SHOW\n TABLES;");
hiveCmdBuilder.setHiveConfProps(hiveProps);
hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
assertEquals(
"hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez",
"hive -e \"USE default;\nDROP TABLE \\`test\\`;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez",
hiveCmdBuilder.build());
}

Expand All @@ -80,7 +80,7 @@ public void testBeeline() throws IOException {

HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
hiveCmdBuilder.addStatement("DROP TABLE `test`;");
hiveCmdBuilder.addStatement("SHOW TABLES;");

String cmd = hiveCmdBuilder.build();
Expand All @@ -91,7 +91,7 @@ public void testBeeline() throws IOException {
Pair<Integer, String> execute = cliCommandExecutor.execute(createFileCmd);
String hqlStatement = FileUtils.readFileToString(new File(hqlFile), Charset.defaultCharset());
assertEquals(
"USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW TABLES;" + lineSeparator,
"USE default;" + lineSeparator + "DROP TABLE `test`;" + lineSeparator + "SHOW TABLES;" + lineSeparator,
hqlStatement);
assertBeelineCmd(cmd);
FileUtils.forceDelete(new File(hqlFile));
Expand All @@ -105,7 +105,7 @@ public void testSparkSqlForTableOps() throws IOException {

HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
hiveCmdBuilder.addStatement("DROP TABLE `test`;");
hiveCmdBuilder.addStatement("SHOW TABLES;");
String cmd = hiveCmdBuilder.build();
assertBeelineCmd(cmd);
Expand Down
45 changes: 33 additions & 12 deletions core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;

import static org.apache.kylin.job.util.FlatTableSqlQuoteUtils.quote;
import static org.apache.kylin.job.util.FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr;

import com.google.common.collect.Lists;

/**
Expand Down Expand Up @@ -124,7 +127,7 @@ public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc)
}
}

return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc)
return "INSERT OVERWRITE TABLE " + quote(flatDesc.getTableName()) + " " + generateSelectDataStatement(flatDesc)
+ ";\n";
}

Expand All @@ -146,24 +149,29 @@ public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc,
sql.append(",");
}
String colTotalName = String.format(Locale.ROOT, "%s.%s", col.getTableRef().getTableName(), col.getName());
String quotedColTotalName = String.format(Locale.ROOT, "%s.%s",
quote(col.getTableRef().getTableName()),
quote(col.getName()));
if (skipAsList.contains(colTotalName)) {
sql.append(col.getExpressionInSourceDB() + sep);
sql.append(getQuotedColExpressionInSourceDB(flatDesc, col)).append(sep);
} else {
sql.append(col.getExpressionInSourceDB() + " as " + colName(col, true) + sep);
sql.append(getQuotedColExpressionInSourceDB(flatDesc, col)).append(" as ")
.append(quote(colName(col))).append(sep);
}
}
appendJoinStatement(flatDesc, sql, singleLine);
appendWhereStatement(flatDesc, sql, singleLine);
return sql.toString();
}

public static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine) {
static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine) {
final String sep = singleLine ? " " : "\n";
Set<TableRef> dimTableCache = new HashSet<>();

DataModelDesc model = flatDesc.getDataModel();
TableRef rootTable = model.getRootFactTable();
sql.append("FROM " + rootTable.getTableIdentity() + " as " + rootTable.getAlias() + " " + sep);
sql.append(" FROM ").append(flatDesc.getDataModel().getRootFactTable().getTableIdentityQuoted("`"))
.append(" as ").append(quote(rootTable.getAlias())).append(sep);

for (JoinTableDesc lookupDesc : model.getJoinTables()) {
JoinDesc join = lookupDesc.getJoin();
Expand All @@ -177,13 +185,15 @@ public static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuil
}
String joinType = join.getType().toUpperCase(Locale.ROOT);

sql.append(joinType + " JOIN " + dimTable.getTableIdentity() + " as " + dimTable.getAlias() + sep);
sql.append(joinType).append(" JOIN ").append(dimTable.getTableIdentityQuoted("`"))
.append(" as ").append(quote(dimTable.getAlias())).append(sep);
sql.append("ON ");
for (int i = 0; i < pk.length; i++) {
if (i > 0) {
sql.append(" AND ");
}
sql.append(fk[i].getExpressionInSourceDB() + " = " + pk[i].getExpressionInSourceDB());
sql.append(getQuotedColExpressionInSourceDB(flatDesc, fk[i])).append(" = ")
.append(getQuotedColExpressionInSourceDB(flatDesc, pk[i]));
}
sql.append(sep);

Expand Down Expand Up @@ -218,18 +228,20 @@ private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBu

DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc,
model.getFilterCondition(), "`");
whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
}

if (flatDesc.getSegment() != null) {
PartitionDesc partDesc = model.getPartitionDesc();
if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
SegmentRange segRange = flatDesc.getSegRange();

if (segRange != null && !segRange.isInfinite()) {
whereBuilder.append(" AND (");
whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
flatDesc.getSegment(), segRange));
String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange), "`");
whereBuilder.append(quotedPartitionCond);
whereBuilder.append(")" + sep);
}
}
Expand Down Expand Up @@ -265,7 +277,7 @@ private static String getHiveDataType(String javaDataType) {
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) {
final String tableName = flatDesc.getTableName();
StringBuilder sql = new StringBuilder();
sql.append("INSERT OVERWRITE TABLE " + tableName + " SELECT * FROM " + tableName);
sql.append("INSERT OVERWRITE TABLE " + quote(tableName) + " SELECT * FROM " + quote(tableName));

if (flatDesc.getClusterBy() != null) {
appendClusterStatement(sql, flatDesc.getClusterBy());
Expand All @@ -291,4 +303,13 @@ public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc
return sql.toString();
}

public static String getQuotedColExpressionInSourceDB(IJoinedFlatTableDesc flatDesc, TblColRef col) {
if (!col.getColumnDesc().isComputedColumn()) {
return quote(col.getTableAlias()) + "."
+ quote(col.getName());
} else {
String computeExpr = col.getColumnDesc().getComputedColumnExpr();
return quoteIdentifierInSqlExpr(flatDesc, computeExpr, "`");
}
}
}
Loading

0 comments on commit 4158d7b

Please sign in to comment.