Skip to content

Commit

Permalink
Misc fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jackie-Jiang
Jackie-Jiang committed Jul 27, 2021
1 parent 2bd14e4 commit f7d6bf4
Show file tree
Hide file tree
Showing 8 changed files with 175 additions and 229 deletions.
1 change: 1 addition & 0 deletions pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationGroupByOrderByPlanNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2;


/**
* The <code>AggregationGroupByOrderByPlanNode</code> class provides the execution plan for aggregation group-by order-by query on a
* single segment.
Expand Down
39 changes: 18 additions & 21 deletions pinot-core/src/main/java/org/apache/pinot/core/startree/CompositePredicateEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,43 +18,40 @@
*/
package org.apache.pinot.core.startree;

import java.util.ArrayList;
import java.util.List;
import org.apache.pinot.common.request.context.FilterContext;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;


/**
* Represents a composite predicate.
*
* A composite predicate evaluator represents a set of predicates conjoined by a given relation.
* Consider the given predicate: (d1 > 10 OR d1 < 50). A composite predicate will represent
* two predicates -- d1 > 10 and d1 < 50 and represent that they are related by the operator OR.
* A composite predicate evaluator represents a single predicate evaluator or multiple predicate evaluators conjoined
* with OR.
* Consider the given predicate: (d1 > 10 OR d1 < 50). A composite predicate will represent two predicates -- (d1 > 10)
* and (d1 < 50) and represent that they are related by the operator OR.
*/
public class CompositePredicateEvaluator {
// Since top level predicates are implicitly ANDed together, we only expect OR or PREDICATE type here
private final FilterContext.Type _filterContextType;
private final List<PredicateEvaluator> _predicateEvaluators;

public CompositePredicateEvaluator(FilterContext.Type filterContextType) {
assert filterContextType == FilterContext.Type.OR || filterContextType == FilterContext.Type.PREDICATE;

_filterContextType = filterContextType;

_predicateEvaluators = new ArrayList<>();
}

public void addPredicateEvaluator(PredicateEvaluator predicateEvaluator) {
assert predicateEvaluator != null;

_predicateEvaluators.add(predicateEvaluator);
public CompositePredicateEvaluator(List<PredicateEvaluator> predicateEvaluators) {
assert !predicateEvaluators.isEmpty();
_predicateEvaluators = predicateEvaluators;
}

public List<PredicateEvaluator> getPredicateEvaluators() {
return _predicateEvaluators;
}

public FilterContext.Type getFilterContextType() {
return _filterContextType;
/**
* Applies a dictionary id to the composite predicate evaluator. Returns {@code true} if the dictionary id matches any
* predicate evaluator, {@code false} otherwise.
*/
public boolean apply(int dictId) {
for (PredicateEvaluator predicateEvaluator : _predicateEvaluators) {
if (predicateEvaluator.applySV(dictId)) {
return true;
}
}
return false;
}
}
161 changes: 77 additions & 84 deletions pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.Queue;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.FilterContext;
import org.apache.pinot.common.request.context.predicate.Predicate;
Expand Down Expand Up @@ -85,9 +86,10 @@ public static AggregationFunctionColumnPair[] extractAggregationFunctionPairs(
* Extracts a map from the column to a list of {@link PredicateEvaluator}s for it. Returns {@code null} if the filter
* cannot be solved by the star-tree.
*
* A predicate can be simple (d1 > 10) or composite (d1 > 10 AND d2 < 50) or multi levelled (d1 > 50 AND (d2 > 10 OR d2 < 35)
* This method represents a list of CompositePredicates per dimension. For each dimension, all CompositePredicates in the
* list are implicitly ANDed together. Any OR predicates are nested within a CompositePredicate.
* A predicate can be simple (d1 > 10) or composite (d1 > 10 AND d2 < 50) or multi levelled
* (d1 > 50 AND (d2 > 10 OR d2 < 35)).
* This method represents a list of CompositePredicates per dimension. For each dimension, all CompositePredicates in
* the list are implicitly ANDed together. Any OR predicates are nested within a CompositePredicate.
*/
@Nullable
public static Map<String, List<CompositePredicateEvaluator>> extractPredicateEvaluatorsMap(IndexSegment indexSegment,
Expand All @@ -106,41 +108,26 @@ public static Map<String, List<CompositePredicateEvaluator>> extractPredicateEva
queue.addAll(filterNode.getChildren());
break;
case OR:
String column = isOrClauseValidForStarTree(filterNode);

if (column != null) {
CompositePredicateEvaluator compositePredicateEvaluator = new CompositePredicateEvaluator(
FilterContext.Type.OR);

filterNode.getChildren().forEach(k -> compositePredicateEvaluator
.addPredicateEvaluator(getPredicateEvaluatorForPredicate(indexSegment, k)));

predicateEvaluatorsMap.computeIfAbsent(column, k -> new ArrayList<>()).add(compositePredicateEvaluator);
} else {
Pair<String, List<PredicateEvaluator>> pair = isOrClauseValidForStarTree(indexSegment, filterNode);
if (pair == null) {
return null;
}
List<PredicateEvaluator> predicateEvaluators = pair.getRight();
// NOTE: Empty list means always true
if (!predicateEvaluators.isEmpty()) {
predicateEvaluatorsMap.computeIfAbsent(pair.getLeft(), k -> new ArrayList<>())
.add(new CompositePredicateEvaluator(predicateEvaluators));
}
break;
case PREDICATE:
Predicate predicate = filterNode.getPredicate();
ExpressionContext lhs = predicate.getLhs();
if (lhs.getType() != ExpressionContext.Type.IDENTIFIER) {
// Star-tree does not support non-identifier expression
return null;
}
column = lhs.getIdentifier();

PredicateEvaluator predicateEvaluator = getPredicateEvaluatorForPredicate(indexSegment, filterNode);

PredicateEvaluator predicateEvaluator = getPredicateEvaluatorForPredicate(indexSegment, predicate);
if (predicateEvaluator == null) {
return null;
}

if (predicateEvaluator != null && !predicateEvaluator.isAlwaysTrue()) {
CompositePredicateEvaluator compositePredicateEvaluator = new CompositePredicateEvaluator(
FilterContext.Type.PREDICATE);

compositePredicateEvaluator.addPredicateEvaluator(predicateEvaluator);
predicateEvaluatorsMap.computeIfAbsent(column, k -> new ArrayList<>()).add(compositePredicateEvaluator);
if (!predicateEvaluator.isAlwaysTrue()) {
predicateEvaluatorsMap.computeIfAbsent(predicate.getLhs().getIdentifier(), k -> new ArrayList<>())
.add(new CompositePredicateEvaluator(Collections.singletonList(predicateEvaluator)));
}
break;
default:
Expand Down Expand Up @@ -186,66 +173,78 @@ public static boolean isFitForStarTree(StarTreeV2Metadata starTreeV2Metadata,
}

/**
* Evaluates a given filter to ascertain if the OR clause is valid for StarTree processing.
*
* Evaluates whether the given OR clause is valid for StarTree processing.
* StarTree supports OR predicates on a single dimension only (d1 < 10 OR d1 > 50).
*
* @return The single literal on which the predicate is based if true, null otherwise
* @return The pair of single identifier and predicate evaluators applied to it if true; {@code null} if the OR clause
* cannot be solved with star-tree; empty predicate evaluator list if the OR clause always evaluates to true.
*/
private static String isOrClauseValidForStarTree(FilterContext filterContext) {
assert filterContext != null;

Set<String> seenIdentifiers = new HashSet<>();

if (!isOrClauseValidForStarTreeInternal(filterContext, seenIdentifiers)) {
return null;
@Nullable
private static Pair<String, List<PredicateEvaluator>> isOrClauseValidForStarTree(IndexSegment indexSegment,
FilterContext filter) {
assert filter.getType() == FilterContext.Type.OR;

List<Predicate> predicates = new ArrayList<>();
extractOrClausePredicates(filter, predicates);

String identifier = null;
List<PredicateEvaluator> predicateEvaluators = new ArrayList<>();
for (Predicate predicate : predicates) {
PredicateEvaluator predicateEvaluator = getPredicateEvaluatorForPredicate(indexSegment, predicate);
if (predicateEvaluator == null) {
// The predicate cannot be solved with star-tree
return null;
}
if (predicateEvaluator.isAlwaysTrue()) {
// Use empty predicate evaluators to represent always true
return Pair.of(null, Collections.emptyList());
}
if (!predicateEvaluator.isAlwaysFalse()) {
String predicateIdentifier = predicate.getLhs().getIdentifier();
if (identifier == null) {
identifier = predicateIdentifier;
} else {
if (!identifier.equals(predicateIdentifier)) {
// The predicates are applied to multiple columns
return null;
}
}
predicateEvaluators.add(predicateEvaluator);
}
}

boolean result = seenIdentifiers.size() == 1;

return result ? seenIdentifiers.iterator().next() : null;
return Pair.of(identifier, predicateEvaluators);
}

/** Internal processor for the above evaluator */
private static boolean isOrClauseValidForStarTreeInternal(FilterContext filterContext, Set<String> seenLiterals) {
assert filterContext != null;

if (filterContext.getType() == FilterContext.Type.OR) {
List<FilterContext> childFilterContexts = filterContext.getChildren();
/**
* Extracts the predicates under the given OR clause, returns {@code false} if there is nested AND under OR clause.
*/
private static boolean extractOrClausePredicates(FilterContext filter, List<Predicate> predicates) {
assert filter.getType() == FilterContext.Type.OR;

for (FilterContext childFilterContext : childFilterContexts) {
if (!isOrClauseValidForStarTreeInternal(childFilterContext, seenLiterals)) {
for (FilterContext child : filter.getChildren()) {
switch (child.getType()) {
case AND:
return false;
}
}
} else if (filterContext.getType() == FilterContext.Type.PREDICATE) {
String literalValue = validateExpressionAndExtractIdentifier(filterContext.getPredicate());

if (literalValue != null) {
seenLiterals.add(literalValue);
} else {
return false;
case OR:
if (!extractOrClausePredicates(child, predicates)) {
return false;
}
case PREDICATE:
predicates.add(child.getPredicate());
break;
default:
throw new IllegalStateException();
}
}

return true;
}

/** Checks if the given predicate has an expression which is of type identifier and returns it */
private static String validateExpressionAndExtractIdentifier(Predicate predicate) {
assert predicate != null;

ExpressionContext expressionContext = predicate.getLhs();

if (expressionContext.getType() == ExpressionContext.Type.IDENTIFIER) {
return expressionContext.getIdentifier();
}

return null;
}

private static PredicateEvaluator getPredicateEvaluatorForPredicate(IndexSegment indexSegment, FilterContext filterNode) {
Predicate predicate = filterNode.getPredicate();
/**
* Returns the predicate evaluator for the given predicate, or {@code null} if the predicate cannot be solved with
* star-tree.
*/
@Nullable
private static PredicateEvaluator getPredicateEvaluatorForPredicate(IndexSegment indexSegment, Predicate predicate) {
ExpressionContext lhs = predicate.getLhs();
if (lhs.getType() != ExpressionContext.Type.IDENTIFIER) {
// Star-tree does not support non-identifier expression
Expand All @@ -268,13 +267,7 @@ private static PredicateEvaluator getPredicateEvaluatorForPredicate(IndexSegment
case IS_NOT_NULL:
return null;
}
PredicateEvaluator predicateEvaluator = PredicateEvaluatorProvider
return PredicateEvaluatorProvider
.getPredicateEvaluator(predicate, dictionary, dataSource.getDataSourceMetadata().getDataType());
if (predicateEvaluator.isAlwaysFalse()) {
// Do not use star-tree if there is no matching record
return null;
}

return predicateEvaluator;
}
}
Loading

0 comments on commit f7d6bf4

Please sign in to comment.