Skip to content

[CBO] user warning added if cbo didn't work #11950

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Nov 29, 2024
Merged

[CBO] user warning added if cbo didn't work #11950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
224330a
[CBO] user warning added if cbo didn't work
Nov 25, 2024
b39d5f9
[CBO] user warning added if cbo hadn't worked
Nov 25, 2024
a04d541
fix
Nov 25, 2024
156b603
fix
Nov 25, 2024
652e80a
fix
Nov 25, 2024
07eb425
fix
Nov 25, 2024
628fe06
fix
pashandor789 Nov 26, 2024
0519745
fix
pashandor789 Nov 26, 2024
fe3c9fb
Merge branch 'main' into WARNCBOADDEDD
pashandor789 Nov 26, 2024
fc24ae0
fix
pashandor789 Nov 26, 2024
192cf78
fix
pashandor789 Nov 27, 2024
7859dce
fix
pashandor789 Nov 28, 2024
9147fb4
fix
pashandor789 Nov 28, 2024
d155e5f
fix
pashandor789 Nov 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ydb/core/kqp/opt/logical/kqp_opt_log.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,10 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
}

TMaybeNode<TExprBase> OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) {
auto maxDPccpDPTableSize = Config->MaxDPccpDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPccpDPTableSize);
auto maxDPhypDPTableSize = Config->MaxDPHypDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPHypDPTableSize);
auto optLevel = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel);
auto providerCtx = TKqpProviderContext(KqpCtx, optLevel);
auto opt = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(providerCtx, maxDPccpDPTableSize));
auto opt = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(providerCtx, maxDPhypDPTableSize, ctx));
TExprBase output = DqOptimizeEquiJoinWithCosts(node, ctx, TypesCtx, optLevel,
*opt, [](auto& rels, auto label, auto node, auto stat) {
rels.emplace_back(std::make_shared<TKqpRelOptimizerNode>(TString(label), *stat, node));
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/kqp/provider/yql_kikimr_settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ TKikimrConfiguration::TKikimrConfiguration() {
REGISTER_SETTING(*this, EnableSpillingNodes)
.Parser([](const TString& v) { return ParseEnableSpillingNodes(v); });

REGISTER_SETTING(*this, MaxDPccpDPTableSize);
REGISTER_SETTING(*this, MaxDPHypDPTableSize);

REGISTER_SETTING(*this, MaxTasksPerStage);
REGISTER_SETTING(*this, MaxSequentialReadsInFlight);
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/kqp/provider/yql_kikimr_settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ struct TKikimrSettings {
NCommon::TConfSetting<bool, false> OptUseFinalizeByKey;
NCommon::TConfSetting<ui32, false> CostBasedOptimizationLevel;

NCommon::TConfSetting<ui32, false> MaxDPccpDPTableSize;
NCommon::TConfSetting<ui32, false> MaxDPHypDPTableSize;


NCommon::TConfSetting<ui32, false> MaxTasksPerStage;
Expand Down
14 changes: 7 additions & 7 deletions ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3566,7 +3566,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings)
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT(result.GetIssues().Empty());
// UNIT_ASSERT(result.GetIssues().Empty());
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
"[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]]]");

Expand Down Expand Up @@ -3610,7 +3610,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings)
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT(result.GetIssues().Empty());
// UNIT_ASSERT(result.GetIssues().Empty());
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
"[[[\"Table1Primary4\"];[4]];[[\"Table1Primary3\"];[3]]]");

Expand Down Expand Up @@ -3655,7 +3655,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings)
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT(result.GetIssues().Empty());
// UNIT_ASSERT(result.GetIssues().Empty());
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
"[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]];[[\"Table1Primary55\"]]]");

Expand Down Expand Up @@ -3699,7 +3699,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings)
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT(result.GetIssues().Empty());
// UNIT_ASSERT(result.GetIssues().Empty());
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
"[[[\"Table1Primary55\"];[55]];[[\"Table1Primary4\"];[4]];[[\"Table1Primary3\"];[3]]]");

Expand Down Expand Up @@ -3804,7 +3804,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings)
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT(result.GetIssues().Empty());
// UNIT_ASSERT(result.GetIssues().Empty());
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
"[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]]]");

Expand Down Expand Up @@ -3856,7 +3856,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings)
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT(result.GetIssues().Empty());
// UNIT_ASSERT(result.GetIssues().Empty());
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
"[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]];[[\"Table1Primary55\"];#]]");

Expand Down Expand Up @@ -4397,7 +4397,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings).ExtractValueSync();

UNIT_ASSERT_C(result2.IsSuccess(), result2.GetIssues().ToString());
UNIT_ASSERT(result2.GetIssues().Empty());
// UNIT_ASSERT(result2.GetIssues().Empty());

UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result2.GetResultSet(0)), "[[[\"Payload1\"]]]");

Expand Down
11 changes: 7 additions & 4 deletions ydb/library/yql/dq/opt/dq_cbo_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,14 @@ Y_UNIT_TEST_SUITE(DQCBO) {

Y_UNIT_TEST(Empty) {
TBaseProviderContext pctx;
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000));
TExprContext dummyCtx;
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000, dummyCtx));
}

Y_UNIT_TEST(JoinSearch2Rels) {
TBaseProviderContext pctx;
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000));
TExprContext dummyCtx;
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000, dummyCtx));

auto rel1 = std::make_shared<TRelOptimizerNode>(
"a",
Expand Down Expand Up @@ -80,7 +82,8 @@ Type: ManyManyJoin, Nrows: 2e+10, Ncols: 2, ByteSize: 0, Cost: 2.00112e+10, Sel:

Y_UNIT_TEST(JoinSearch3Rels) {
TBaseProviderContext pctx;
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000));
TExprContext dummyCtx;
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000, dummyCtx));

auto rel1 = std::make_shared<TRelOptimizerNode>("a",
TOptimizerStatistics(BaseTable, 100000, 1, 0, 1000000));
Expand Down Expand Up @@ -243,7 +246,7 @@ Y_UNIT_TEST(DqOptimizeEquiJoinWithCostsNative) {
TExprContext ctx;
TBaseProviderContext pctx;
std::function<IOptimizerNew*()> optFactory = [&]() {
return MakeNativeOptimizerNew(pctx, 100000);
return MakeNativeOptimizerNew(pctx, 100000, ctx);
};
_DqOptimizeEquiJoinWithCosts(optFactory, ctx);
}
Expand Down
3 changes: 2 additions & 1 deletion ydb/library/yql/dq/opt/dq_opt_hypergraph_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@ struct TTestContext : public TBaseProviderContext {
template <typename TProviderContext = TTestContext>
std::shared_ptr<IBaseOptimizerNode> Enumerate(const std::shared_ptr<IBaseOptimizerNode>& root, const TOptimizerHints& hints = {}) {
auto ctx = TProviderContext();
TExprContext dummyCtx;
auto optimizer =
std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(ctx, std::numeric_limits<ui32>::max()));
std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(ctx, std::numeric_limits<ui32>::max(), dummyCtx));

Y_ENSURE(root->Kind == EOptimizerNodeKind::JoinNodeType);
auto res = optimizer->JoinSearch(std::static_pointer_cast<TJoinOptimizerNode>(root), hints);
Expand Down
4 changes: 2 additions & 2 deletions ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ namespace NYql::NDq {
namespace {
class TDqOptimizerFactory : public IOptimizerFactory {
public:
virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext&, const TNativeSettings& settings) const override {
return IOptimizerNew::TPtr(MakeNativeOptimizerNew(pctx, settings.MaxDPhypDPTableSize));
virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ectx, const TNativeSettings& settings) const override {
return IOptimizerNew::TPtr(MakeNativeOptimizerNew(pctx, settings.MaxDPhypDPTableSize, ectx));
}

virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerPG(IProviderContext& pctx, TExprContext& ctx, const TPGSettings& settings) const override {
Expand Down
26 changes: 20 additions & 6 deletions ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,10 @@ void ComputeStatistics(const std::shared_ptr<TJoinOptimizerNode>& join, IProvide

class TOptimizerNativeNew: public IOptimizerNew {
public:
TOptimizerNativeNew(IProviderContext& ctx, ui32 maxDPhypDPTableSize)
TOptimizerNativeNew(IProviderContext& ctx, ui32 maxDPhypDPTableSize, TExprContext& exprCtx)
: IOptimizerNew(ctx)
, MaxDPhypTableSize_(maxDPhypDPTableSize)
, MaxDPHypTableSize_(maxDPhypDPTableSize)
, ExprCtx(exprCtx)
{}

std::shared_ptr<TJoinOptimizerNode> JoinSearch(
Expand Down Expand Up @@ -272,8 +273,15 @@ class TOptimizerNativeNew: public IOptimizerNew {
TJoinHypergraph<TNodeSet> hypergraph = MakeJoinHypergraph<TNodeSet>(joinTree, hints);
TDPHypSolver<TNodeSet> solver(hypergraph, this->Pctx);

if (solver.CountCC(MaxDPhypTableSize_) >= MaxDPhypTableSize_) {
if (solver.CountCC(MaxDPHypTableSize_) >= MaxDPHypTableSize_) {
YQL_CLOG(TRACE, CoreDq) << "Maximum DPhyp threshold exceeded";
ExprCtx.AddWarning(
YqlIssue(
{}, TIssuesIds::DQ_OPTIMIZE_ERROR,
"Cost Based Optimizer could not be applied to this query: "
"Enumeration is too large, use PRAGMA MaxDPHypDPTableSize='4294967295' to disable the limitation"
)
);
ComputeStatistics(joinTree, this->Pctx);
return joinTree;
}
Expand Down Expand Up @@ -304,11 +312,12 @@ class TOptimizerNativeNew: public IOptimizerNew {
}

private:
ui32 MaxDPhypTableSize_;
ui32 MaxDPHypTableSize_;
TExprContext& ExprCtx;
};

IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize) {
return new TOptimizerNativeNew(ctx, maxDPhypDPTableSize);
IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& pctx, const ui32 maxDPhypDPTableSize, TExprContext& ectx) {
return new TOptimizerNativeNew(pctx, maxDPhypDPTableSize, ectx);
}

TExprBase DqOptimizeEquiJoinWithCosts(
Expand Down Expand Up @@ -357,6 +366,11 @@ TExprBase DqOptimizeEquiJoinWithCosts(
// of the EquiJoin and n-1 argument are the parameters to EquiJoin

if (!DqCollectJoinRelationsWithStats(rels, typesCtx, equiJoin, providerCollect)){
ctx.AddWarning(
YqlIssue({}, TIssuesIds::DQ_OPTIMIZE_ERROR,
"Cost Based Optimizer could not be applied to this query: couldn't load statistics"
)
);
return node;
}

Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/dq/opt/dq_opt_join_cost_based.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,6 @@ NYql::NNodes::TExprBase DqOptimizeEquiJoinWithCosts(
const TOptimizerHints& hints = {}
);

IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPccpDPTableSize);
IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPHypDPTableSize, TExprContext& ectx);

} // namespace NYql::NDq
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/dq/common/yql_dq_settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ struct TDqSettings {
static constexpr ETaskRunnerStats TaskRunnerStats = ETaskRunnerStats::Basic;
static constexpr ESpillingEngine SpillingEngine = ESpillingEngine::Disable;
static constexpr ui32 CostBasedOptimizationLevel = 4;
static constexpr ui32 MaxDPccpDPTableSize = 40000U;
static constexpr ui32 MaxDPHypDPTableSize = 40000U;
static constexpr ui64 MaxAttachmentsSize = 2_GB;
static constexpr bool SplitStageOnDqReplicate = true;
static constexpr ui64 EnableSpillingNodes = 0;
Expand Down
Loading