Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fix](Join) Fix the bug of outer join function under vectorization #9954

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix inline view
  • Loading branch information
EmmyMiao87 committed Jun 27, 2022
commit a03239c6df5c4a2fa3c0a1623d22776daf66f4af
6 changes: 3 additions & 3 deletions be/src/vec/exec/join/vhash_join_node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1198,15 +1198,15 @@ Status HashJoinNode::_extract_probe_join_column(Block& block, NullMap& null_map,

Status HashJoinNode::_process_build_block(RuntimeState* state, Block& block, uint8_t offset) {
SCOPED_TIMER(_build_table_timer);
if (_join_op == TJoinOp::LEFT_OUTER_JOIN || _join_op == TJoinOp::FULL_OUTER_JOIN) {
_convert_block_to_null(block);
}
size_t rows = block.rows();
if (UNLIKELY(rows == 0)) {
return Status::OK();
}
COUNTER_UPDATE(_build_rows_counter, rows);

if (_join_op == TJoinOp::LEFT_OUTER_JOIN || _join_op == TJoinOp::FULL_OUTER_JOIN) {
_convert_block_to_null(block);
}
ColumnRawPtrs raw_ptrs(_build_expr_ctxs.size());

NullMap null_map_val(rows);
Expand Down
14 changes: 12 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
Original file line number Diff line number Diff line change
Expand Up @@ -253,8 +253,9 @@ private static class GlobalState {
private final Map<TupleId, List<ExprId>> eqJoinConjuncts = Maps.newHashMap();

// set of conjuncts that have been assigned to some PlanNode
private Set<ExprId> assignedConjuncts =
Collections.newSetFromMap(new IdentityHashMap<ExprId, Boolean>());
private Set<ExprId> assignedConjuncts = Collections.newSetFromMap(new IdentityHashMap<ExprId, Boolean>());

private Set<TupleId> inlineViewTupleIds = Sets.newHashSet();

// map from outer-joined tuple id, ie, one that is nullable in this select block,
// to the last Join clause (represented by its rhs table ref) that outer-joined it
Expand Down Expand Up @@ -790,6 +791,7 @@ public SlotDescriptor registerColumnRef(TableName tblName, String colName) throw
}
result = globalState.descTbl.addSlotDescriptor(d);
result.setColumn(col);
// TODO: need to remove this outer join'
result.setIsNullable(col.isAllowNull() || isOuterJoined(d.getId()));

slotRefMap.put(key, result);
Expand Down Expand Up @@ -889,6 +891,10 @@ public SlotDescriptor copySlotDescriptor(SlotDescriptor srcSlotDesc, TupleDescri
return result;
}

public void registerInlineViewTupleId(TupleId tupleId) {
globalState.inlineViewTupleIds.add(tupleId);
}

/**
* Register conjuncts that are outer joined by a full outer join. For a given
* predicate, we record the last full outer join that outer-joined any of its
Expand Down Expand Up @@ -2172,6 +2178,10 @@ public boolean isOuterJoined(TupleId tid) {
return globalState.outerJoinedTupleIds.containsKey(tid);
}

public boolean isInlineView(TupleId tid) {
return globalState.inlineViewTupleIds.contains(tid);
}

public boolean containSubquery() {
return globalState.containsSubquery;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ public void removeByRhsExpr(Expr rhsExpr) {
}
}

public void updateLhsExprs(List<Expr> lhsExprList) {
lhs = lhsExprList;
}

/**
* Return a map which is equivalent to applying f followed by g,
* i.e., g(f()).
Expand Down
20 changes: 20 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
Original file line number Diff line number Diff line change
Expand Up @@ -154,13 +154,33 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
tblRef.analyze(analyzer);
leftTblRef = tblRef;
}
// Fix the problem of column nullable attribute error caused by inline view + outer join
changeTblRefToNullable(analyzer);

// TODO: remove when query from hive table is supported
checkFromHiveTable(analyzer);

analyzed = true;
}

// set null-side inlinve view column
// For example: select * from (select a as k1 from t) tmp right join b on tmp.k1=b.k1
// The columns from tmp should be nullable.
// The table ref tmp will be used by HashJoinNode.computeOutputTuple()
private void changeTblRefToNullable(Analyzer analyzer) {
for (TableRef tableRef : tablerefs) {
if (!(tableRef instanceof InlineViewRef)) {
continue;
}
InlineViewRef inlineViewRef = (InlineViewRef) tableRef;
if (analyzer.isOuterJoined(inlineViewRef.getId())) {
for (SlotDescriptor slotDescriptor : inlineViewRef.getDesc().getSlots()) {
slotDescriptor.setIsNullable(true);
}
}
}
}

public FromClause clone() {
ArrayList<TableRef> clone = Lists.newArrayList();
for (TableRef tblRef : tablerefs) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ public TupleDescriptor createTupleDescriptor(Analyzer analyzer) throws AnalysisE
TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor();
result.setIsMaterialized(false);
result.setTable(inlineView);
analyzer.registerInlineViewTupleId(result.getId());
return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@
import org.apache.doris.analysis.TableRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.analysis.TupleIsNullPredicate;
import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.CheckedMath;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.Pair;
Expand Down Expand Up @@ -385,7 +385,7 @@ public void init(Analyzer analyzer) throws UserException {
}
}

private void computeOutputTuple(Analyzer analyzer) throws AnalysisException {
private void computeOutputTuple(Analyzer analyzer) throws UserException {
// 1. create new tuple
vOutputTupleDesc = analyzer.getDescTbl().createTupleDescriptor();
boolean copyLeft = false;
Expand Down Expand Up @@ -427,6 +427,8 @@ private void computeOutputTuple(Analyzer analyzer) throws AnalysisException {
break;
}
ExprSubstitutionMap srcTblRefToOutputTupleSmap = new ExprSubstitutionMap();
int leftNullableNumber = 0;
int rightNullableNumber = 0;
if (copyLeft) {
for (TupleDescriptor leftTupleDesc : analyzer.getDescTbl().getTupleDesc(getChild(0).getOutputTblRefIds())) {
for (SlotDescriptor leftSlotDesc : leftTupleDesc.getSlots()) {
Expand All @@ -437,6 +439,7 @@ private void computeOutputTuple(Analyzer analyzer) throws AnalysisException {
analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, leftSlotDesc);
if (leftNullable) {
outputSlotDesc.setIsNullable(true);
leftNullableNumber++;
}
srcTblRefToOutputTupleSmap.put(new SlotRef(leftSlotDesc), new SlotRef(outputSlotDesc));
}
Expand All @@ -453,6 +456,7 @@ private void computeOutputTuple(Analyzer analyzer) throws AnalysisException {
analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, rightSlotDesc);
if (rightNullable) {
outputSlotDesc.setIsNullable(true);
rightNullableNumber++;
}
srcTblRefToOutputTupleSmap.put(new SlotRef(rightSlotDesc), new SlotRef(outputSlotDesc));
}
Expand All @@ -471,7 +475,37 @@ private void computeOutputTuple(Analyzer analyzer) throws AnalysisException {
}
}
vOutputTupleDesc.computeStatAndMemLayout();
// 3. change the outputSmap
// 3. add tupleisnull in null-side
Preconditions.checkState(srcTblRefToOutputTupleSmap.getLhs().size() == vSrcToOutputSMap.getLhs().size());
// Condition1: the left child is null-side
// Condition2: the left child is a inline view
// Then: add tuple is null in left child columns
if (leftNullable && getChild(0).tblRefIds.size() == 1 && analyzer.isInlineView(getChild(0).tblRefIds.get(0))) {
List<Expr> tupleIsNullLhs = TupleIsNullPredicate
.wrapExprs(vSrcToOutputSMap.getLhs().subList(0, leftNullableNumber), getChild(0).getTupleIds(),
analyzer);
tupleIsNullLhs
.addAll(vSrcToOutputSMap.getLhs().subList(leftNullableNumber, vSrcToOutputSMap.getLhs().size()));
vSrcToOutputSMap.updateLhsExprs(tupleIsNullLhs);
}
// Condition1: the right child is null-side
// Condition2: the right child is a inline view
// Then: add tuple is null in right child columns
if (rightNullable && getChild(1).tblRefIds.size() == 1 && analyzer.isInlineView(getChild(1).tblRefIds.get(0))) {
if (rightNullableNumber != 0) {
int rightBeginIndex = vSrcToOutputSMap.size() - rightNullableNumber;
List<Expr> tupleIsNullLhs = TupleIsNullPredicate
.wrapExprs(vSrcToOutputSMap.getLhs().subList(rightBeginIndex, vSrcToOutputSMap.size()),
getChild(1).getTupleIds(), analyzer);
List<Expr> newLhsList = Lists.newArrayList();
if (rightBeginIndex > 0) {
newLhsList.addAll(vSrcToOutputSMap.getLhs().subList(0, rightBeginIndex));
}
newLhsList.addAll(tupleIsNullLhs);
vSrcToOutputSMap.updateLhsExprs(newLhsList);
}
}
// 4. change the outputSmap
outputSmap = ExprSubstitutionMap.combineAndReplace(outputSmap, srcTblRefToOutputTupleSmap);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import org.apache.doris.common.Pair;
import org.apache.doris.common.Reference;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;

import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
Expand Down Expand Up @@ -1379,7 +1380,7 @@ private PlanNode createInlineViewPlan(Analyzer analyzer, InlineViewRef inlineVie
ExprSubstitutionMap outputSmap = ExprSubstitutionMap.compose(
inlineViewRef.getSmap(), rootNode.getOutputSmap(), analyzer);

if (analyzer.isOuterJoined(inlineViewRef.getId())) {
if (analyzer.isOuterJoined(inlineViewRef.getId()) && !VectorizedUtil.isVectorized()) {
rootNode.setWithoutTupleIsNullOutputSmap(outputSmap);
// Exprs against non-matched rows of an outer join should always return NULL.
// Make the rhs exprs of the output smap nullable, if necessary. This expr wrapping
Expand Down