Skip to content

Pushdown some unary operators. #957

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 16, 2024
Merged

Pushdown some unary operators. #957

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
47be4aa
Pushdown some unary operators.
Tony-Romanov Jan 11, 2024
7a6a3c4
Cosmetic fix.
Tony-Romanov Jan 12, 2024
bdd2383
Separate option for unary operators.
Tony-Romanov Jan 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,15 @@
"VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "KqpOlapXor"}
},
{
"Name": "TKqpOlapFilterUnaryOp",
"Base": "TExprBase",
"Match": {"Type": "Tuple", "Name": "KqpOlapFilterUnaryOp"},
"Children": [
{"Index": 0, "Name": "Operator", "Type": "TCoAtom"},
{"Index": 1, "Name": "Arg", "Type": "TExprBase"}
]
},
{
"Name": "TKqpOlapFilterBinaryOp",
"Base": "TExprBase",
Expand Down
13 changes: 11 additions & 2 deletions ydb/core/kqp/host/kqp_type_ann.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -864,10 +864,19 @@ bool ValidateOlapFilterConditions(const TExprNode* node, const TStructExprType*
}
}
return res;
} else if (TKqpOlapFilterBinaryOp::Match(node)) {
if (!EnsureArgsCount(*node, 3, ctx)) {
} else if (TKqpOlapFilterUnaryOp::Match(node)) {
const auto op = node->Child(TKqpOlapFilterUnaryOp::idx_Operator);
if (!EnsureAtom(*op, ctx)) {
return false;
}
if (!op->IsAtom({"minus", "abs", "not", "size", "exists", "empty"})) {
ctx.AddError(TIssue(ctx.GetPosition(node->Pos()),
TStringBuilder() << "Unexpected OLAP unary operation: " << op->Content()
));
return false;
}
return ValidateOlapFilterConditions(node->Child(TKqpOlapFilterUnaryOp::idx_Arg), itemType, ctx);
} else if (TKqpOlapFilterBinaryOp::Match(node)) {
const auto op = node->Child(TKqpOlapFilterBinaryOp::idx_Operator);
if (!EnsureAtom(*op, ctx)) {
return false;
Expand Down
162 changes: 94 additions & 68 deletions ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ struct TPushdownSettings : public NPushdown::TSettings {
Enable(EFlag::LikeOperator, NSsa::RuntimeVersion >= 2U);
Enable(EFlag::LikeOperatorOnlyForUtf8, NSsa::RuntimeVersion < 3U);
Enable(EFlag::JsonQueryOperators | EFlag::JsonExistsOperator, NSsa::RuntimeVersion >= 3U);
Enable(EFlag::ArithmeticalExpressions, NSsa::RuntimeVersion >= 4U);
Enable(EFlag::ArithmeticalExpressions | EFlag::UnaryOperators, NSsa::RuntimeVersion >= 4U);
Enable(EFlag::LogicalXorOperator
| EFlag::ParameterExpression
| EFlag::CastExpression
Expand Down Expand Up @@ -100,8 +100,43 @@ bool IsFalseLiteral(TExprBase node) {
return node.Maybe<TCoBool>() && !FromString<bool>(node.Cast<TCoBool>().Literal().Value());
}

std::optional<std::pair<TExprBase, TExprBase>> ExtractBinaryFunctionParameters(const TExprBase& op, TExprContext& ctx, TPositionHandle pos);
std::vector<std::pair<TExprBase, TExprBase>> ExtractComparisonParameters(const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos);
std::vector<TExprBase> ConvertComparisonNode(const TExprBase& nodeIn, TExprContext& ctx, TPositionHandle pos);

std::optional<std::pair<TExprBase, TExprBase>> ExtractBinaryFunctionParameters(const TExprBase& op, TExprContext& ctx, TPositionHandle pos)
{
const auto left = ConvertComparisonNode(TExprBase(op.Ref().HeadPtr()), ctx, pos);
if (left.size() != 1U) {
return std::nullopt;
}

const auto right = ConvertComparisonNode(TExprBase(op.Ref().TailPtr()), ctx, pos);
if (right.size() != 1U) {
return std::nullopt;
}

return std::make_pair(left.front(), right.front());
}

std::vector<std::pair<TExprBase, TExprBase>> ExtractComparisonParameters(const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos)
{
std::vector<std::pair<TExprBase, TExprBase>> out;
auto left = ConvertComparisonNode(predicate.Left(), ctx, pos);

if (left.empty()) {
return out;
}

auto right = ConvertComparisonNode(predicate.Right(), ctx, pos);
if (left.size() != right.size()) {
return out;
}

for (ui32 i = 0; i < left.size(); ++i) {
out.emplace_back(std::move(std::make_pair(left[i], right[i])));
}

return out;
}

TMaybeNode<TExprBase> ComparisonPushdown(const std::vector<std::pair<TExprBase, TExprBase>>& parameters, const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos);

Expand All @@ -117,9 +152,13 @@ TMaybeNode<TExprBase> YqlCoalescePushdown(const TCoCoalesce& coalesce, TExprCont
return NullNode;
}

TVector<TExprBase> ConvertComparisonNode(const TExprBase& nodeIn, TExprContext& ctx, TPositionHandle pos)
bool IsGoodTypeForPushdown(const TTypeAnnotationNode& type) {
return NUdf::EDataTypeFeatures::IntegralType & NUdf::GetDataTypeInfo(RemoveOptionality(type).Cast<TDataExprType>()->GetSlot()).Features;
}

std::vector<TExprBase> ConvertComparisonNode(const TExprBase& nodeIn, TExprContext& ctx, TPositionHandle pos)
{
TVector<TExprBase> out;
std::vector<TExprBase> out;
const auto convertNode = [&ctx, &pos](const TExprBase& node) -> TMaybeNode<TExprBase> {
if (node.Maybe<TCoNull>()) {
return node;
Expand Down Expand Up @@ -162,20 +201,35 @@ TVector<TExprBase> ConvertComparisonNode(const TExprBase& nodeIn, TExprContext&
return builder.Done();
}

if (const auto maybeArithmetic = node.Maybe<TCoBinaryArithmetic>()) {
if (const auto arithmetic = maybeArithmetic.Cast(); !arithmetic.Maybe<TCoAggrAdd>()) {
if (const auto params = ExtractBinaryFunctionParameters(arithmetic, ctx, pos)) {
return Build<TKqpOlapFilterBinaryOp>(ctx, pos)
.Operator().Value(arithmetic.Ref().Content(), TNodeFlags::Default).Build()
.Left(params->first)
.Right(params->second)
.Done();
if constexpr (NKikimr::NSsa::RuntimeVersion >= 4U) {
if (const auto maybeArithmetic = node.Maybe<TCoBinaryArithmetic>()) {
if (const auto arithmetic = maybeArithmetic.Cast(); IsGoodTypeForPushdown(*arithmetic.Ref().GetTypeAnn()) && !arithmetic.Maybe<TCoAggrAdd>()) {
if (const auto params = ExtractBinaryFunctionParameters(arithmetic, ctx, pos)) {
return Build<TKqpOlapFilterBinaryOp>(ctx, pos)
.Operator().Value(arithmetic.Ref().Content(), TNodeFlags::Default).Build()
.Left(params->first)
.Right(params->second)
.Done();
}
}
}
}

if (const auto maybeCoalesce = node.Maybe<TCoCoalesce>()) {
return YqlCoalescePushdown(maybeCoalesce.Cast(), ctx);
if (const auto maybeArithmetic = node.Maybe<TCoUnaryArithmetic>()) {
if (const auto arithmetic = maybeArithmetic.Cast(); IsGoodTypeForPushdown(*arithmetic.Ref().GetTypeAnn())) {
if (const auto params = ConvertComparisonNode(arithmetic.Arg(), ctx, pos); 1U == params.size()) {
TString oper(arithmetic.Ref().Content());
YQL_ENSURE(oper.to_lower());
return Build<TKqpOlapFilterUnaryOp>(ctx, pos)
.Operator().Value(oper, TNodeFlags::Default).Build()
.Arg(params.front())
.Done();
}
}
}

if (const auto maybeCoalesce = node.Maybe<TCoCoalesce>()) {
return YqlCoalescePushdown(maybeCoalesce.Cast(), ctx);
}
}

if (const auto maybeCompare = node.Maybe<TCoCompare>()) {
Expand Down Expand Up @@ -219,42 +273,6 @@ TVector<TExprBase> ConvertComparisonNode(const TExprBase& nodeIn, TExprContext&
return out;
}

std::optional<std::pair<TExprBase, TExprBase>> ExtractBinaryFunctionParameters(const TExprBase& op, TExprContext& ctx, TPositionHandle pos)
{
const auto left = ConvertComparisonNode(TExprBase(op.Ref().HeadPtr()), ctx, pos);
if (left.size() != 1U) {
return std::nullopt;
}

const auto right = ConvertComparisonNode(TExprBase(op.Ref().TailPtr()), ctx, pos);
if (right.size() != 1U) {
return std::nullopt;
}

return std::make_pair(left.front(), right.front());
}

std::vector<std::pair<TExprBase, TExprBase>> ExtractComparisonParameters(const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos)
{
std::vector<std::pair<TExprBase, TExprBase>> out;
auto left = ConvertComparisonNode(predicate.Left(), ctx, pos);

if (left.empty()) {
return out;
}

auto right = ConvertComparisonNode(predicate.Right(), ctx, pos);
if (left.size() != right.size()) {
return out;
}

for (ui32 i = 0; i < left.size(); ++i) {
out.emplace_back(std::move(std::make_pair(left[i], right[i])));
}

return out;
}

TExprBase BuildOneElementComparison(const std::pair<TExprBase, TExprBase>& parameter, const TCoCompare& predicate,
TExprContext& ctx, TPositionHandle pos, bool forceStrictComparison)
{
Expand Down Expand Up @@ -425,13 +443,20 @@ TMaybeNode<TCoAtomList> BuildColumnsFromLambda(const TCoLambda& lambda, TExprCon
}
#endif

template<bool Empty>
TMaybeNode<TExprBase> ExistsPushdown(const TCoExists& exists, TExprContext& ctx, TPositionHandle pos)
{
auto columnName = exists.Optional().Cast<TCoMember>().Name();

return Build<TKqpOlapFilterExists>(ctx, pos)
.Column(columnName)
.Done();
const auto columnName = exists.Optional().Cast<TCoMember>().Name();
if constexpr (NSsa::RuntimeVersion >= 4U) {
return Build<TKqpOlapFilterUnaryOp>(ctx, pos)
.Operator().Value(Empty ? "empty" : "exists", TNodeFlags::Default).Build()
.Arg(columnName)
.Done();
} else {
return Build<TKqpOlapFilterExists>(ctx, pos)
.Column(columnName)
.Done();
}
}

TMaybeNode<TExprBase> JsonExistsPushdown(const TCoJsonExists& jsonExists, TExprContext& ctx, TPositionHandle pos)
Expand Down Expand Up @@ -505,32 +530,33 @@ TMaybeNode<TExprBase> CoalescePushdown(const TCoCoalesce& coalesce, TExprContext

TFilterOpsLevels PredicatePushdown(const TExprBase& predicate, TExprContext& ctx, TPositionHandle pos)
{
auto maybeCoalesce = predicate.Maybe<TCoCoalesce>();
if (maybeCoalesce.IsValid()) {
if (const auto maybeCoalesce = predicate.Maybe<TCoCoalesce>()) {
auto coalescePred = CoalescePushdown(maybeCoalesce.Cast(), ctx, pos);
return TFilterOpsLevels(coalescePred);
}

auto maybeExists = predicate.Maybe<TCoExists>();
if (maybeExists.IsValid()) {
auto existsPred = ExistsPushdown(maybeExists.Cast(), ctx, pos);
if (const auto maybeExists = predicate.Maybe<TCoExists>()) {
auto existsPred = ExistsPushdown<false>(maybeExists.Cast(), ctx, pos);
return TFilterOpsLevels(existsPred);
}

auto maybeJsonExists = predicate.Maybe<TCoJsonExists>();
if (maybeJsonExists.IsValid()) {
if (const auto maybeJsonExists = predicate.Maybe<TCoJsonExists>()) {
auto jsonExistsPred = JsonExistsPushdown(maybeJsonExists.Cast(), ctx, pos);
return TFilterOpsLevels(jsonExistsPred);
}

auto maybePredicate = predicate.Maybe<TCoCompare>();
if (maybePredicate.IsValid()) {
if (const auto maybePredicate = predicate.Maybe<TCoCompare>()) {
auto pred = SimplePredicatePushdown(maybePredicate.Cast(), ctx, pos);
return TFilterOpsLevels(pred);
}

if (predicate.Maybe<TCoNot>()) {
auto notNode = predicate.Cast<TCoNot>();
if (const auto maybeNot = predicate.Maybe<TCoNot>()) {
const auto notNode = maybeNot.Cast();
if constexpr (NSsa::RuntimeVersion >= 4U) {
if (const auto maybeExists = notNode.Value().Maybe<TCoExists>()) {
return TFilterOpsLevels(ExistsPushdown<true>(maybeExists.Cast(), ctx, pos));
}
}
auto pushedFilters = PredicatePushdown(notNode.Value(), ctx, pos);
pushedFilters.WrapToNotOp(ctx, pos);
return pushedFilters;
Expand Down
Loading