Merge f410524 into 65aa640

Author: pashandor789

library/cpp/disjoint_sets/disjoint_sets.h

@@ -16,6 +16,8 @@ class TDisjointSets {
     size_t NumberOfSets;
 
 public:
+    TDisjointSets() = default;
+    
     TDisjointSets(size_t setCount)
         : Parents(setCount)
         , Ranks(setCount, 0)

ydb/library/yql/core/yql_cost_function.h

@@ -46,7 +46,7 @@ struct TJoinColumn {
         return RelName == other.RelName && AttributeName == other.AttributeName;
     }
 
-    struct HashFunction
+    struct THashFunction
     {
         size_t operator()(const TJoinColumn& c) const
         {

ydb/library/yql/dq/opt/dq_opt_conflict_rules_collector.h

@@ -68,7 +68,7 @@ class TConflictRulesCollector {
                 ConflictRules_.emplace_back(
                     SubtreeNodes_[child->LeftArg],
                     SubtreeNodes_[child->RightArg]
-                );     
+                );
             }
         };
 

ydb/library/yql/dq/opt/dq_opt_dphyp_solver.h

@@ -33,7 +33,7 @@ namespace NYql::NDq {
  * This class is templated by std::bitset with the largest number of joins we can process
  * or std::bitset<64>, which has a more efficient implementation of enumerating subsets of set.
  */
-template <typename TNodeSet>
+template <typename TNodeSet, bool ProcessCycles>
 class TDPHypSolver {
 public:
     TDPHypSolver(
@@ -58,7 +58,7 @@ class TDPHypSolver {
 
     void EnumerateCmpRec(const TNodeSet& s1, const TNodeSet& s2, const TNodeSet& x);
 
-    void EmitCsgCmp(const TNodeSet& s1, const TNodeSet& s2, const typename TJoinHypergraph<TNodeSet>::TEdge* csgCmpEdge);
+    void EmitCsgCmp(const TNodeSet& s1, const TNodeSet& s2, const typename TJoinHypergraph<TNodeSet>::TEdge* csgCmpEdge, const typename TJoinHypergraph<TNodeSet>::TEdge* reversedCsgCmpEdge);
 
 private:
     // Create an exclusion set that contains all the nodes of the graph that are smaller or equal to
@@ -113,7 +113,7 @@ class TDPHypSolver {
 /*
  * Count the number of items in the DP table of DPHyp
  */
-template <typename TNodeSet> ui32 TDPHypSolver<TNodeSet>::CountCC(ui32 budget) {
+template <typename TNodeSet, bool ProcessCycles> ui32 TDPHypSolver<TNodeSet, ProcessCycles>::CountCC(ui32 budget) {
     TNodeSet allNodes;
     allNodes.set();
     ui32 cost = 0;
@@ -135,7 +135,7 @@ template <typename TNodeSet> ui32 TDPHypSolver<TNodeSet>::CountCC(ui32 budget) {
 /**
  * Recursively count the nuber of items in the DP table of DPccp
 */
-template <typename TNodeSet> ui32 TDPHypSolver<TNodeSet>::CountCCRec(const TNodeSet& s, const TNodeSet& x, ui32 cost, ui32 budget) {
+template <typename TNodeSet, bool ProcessCycles> ui32 TDPHypSolver<TNodeSet, ProcessCycles>::CountCCRec(const TNodeSet& s, const TNodeSet& x, ui32 cost, ui32 budget) {
     TNodeSet neighs = Neighs(s, x);
 
     if (neighs == TNodeSet{}) {
@@ -161,7 +161,7 @@ template <typename TNodeSet> ui32 TDPHypSolver<TNodeSet>::CountCCRec(const TNode
     return cost;
 }
 
-template<typename TNodeSet> TNodeSet TDPHypSolver<TNodeSet>::Neighs(TNodeSet s, TNodeSet x) {
+template<typename TNodeSet, bool ProcessCycles> TNodeSet TDPHypSolver<TNodeSet, ProcessCycles>::Neighs(TNodeSet s, TNodeSet x) {
     TNodeSet neighs{};
 
     auto& nodes = Graph_.GetNodes();
@@ -190,11 +190,16 @@ template<typename TNodeSet> TNodeSet TDPHypSolver<TNodeSet>::Neighs(TNodeSet s,
 }
 
 template<>
-inline std::bitset<64> TDPHypSolver<std::bitset<64>>::NextBitset(const std::bitset<64>& prev, const std::bitset<64>& final) {
+inline std::bitset<64> TDPHypSolver<std::bitset<64>, false>::NextBitset(const std::bitset<64>& prev, const std::bitset<64>& final) {
     return std::bitset<64>((prev | ~final).to_ulong() + 1) & final;
 }
 
-template<typename TNodeSet> TNodeSet TDPHypSolver<TNodeSet>::NextBitset(const TNodeSet& prev, const TNodeSet& final) {
+template<>
+inline std::bitset<64> TDPHypSolver<std::bitset<64>, true>::NextBitset(const std::bitset<64>& prev, const std::bitset<64>& final) {
+    return std::bitset<64>((prev | ~final).to_ulong() + 1) & final;
+}
+
+template<typename TNodeSet, bool ProcessCycles> TNodeSet TDPHypSolver<TNodeSet, ProcessCycles>::NextBitset(const TNodeSet& prev, const TNodeSet& final) {
     if (prev == final) {
         return final;
     }
@@ -223,7 +228,7 @@ template<typename TNodeSet> TNodeSet TDPHypSolver<TNodeSet>::NextBitset(const TN
     return res;
 }
 
-template<typename TNodeSet> std::shared_ptr<TJoinOptimizerNodeInternal> TDPHypSolver<TNodeSet>::Solve(const TOptimizerHints& hints) {
+template<typename TNodeSet, bool ProcessCycles> std::shared_ptr<TJoinOptimizerNodeInternal> TDPHypSolver<TNodeSet, ProcessCycles>::Solve(const TOptimizerHints& hints) {
     for (auto& h : hints.CardinalityHints->Hints) {
         TNodeSet hintSet = Graph_.GetNodesByRelNames(h.JoinLabels);
         CardHintsTable_[hintSet] = &h;
@@ -270,7 +275,7 @@ template<typename TNodeSet> std::shared_ptr<TJoinOptimizerNodeInternal> TDPHypSo
  * First it emits CSGs that are created by adding neighbors of S to S
  * Then it recurses on the S fused with its neighbors.
  */
-template <typename TNodeSet> void TDPHypSolver<TNodeSet>::EnumerateCsgRec(const TNodeSet& s1, const TNodeSet& x) {
+template <typename TNodeSet, bool ProcessCycles> void TDPHypSolver<TNodeSet, ProcessCycles>::EnumerateCsgRec(const TNodeSet& s1, const TNodeSet& x) {
     TNodeSet neighs =  Neighs(s1, x);
 
     if (neighs == TNodeSet{}) {
@@ -313,7 +318,7 @@ template <typename TNodeSet> void TDPHypSolver<TNodeSet>::EnumerateCsgRec(const
  * First it iterates through neighbors of the initial set S and emits pairs
  * (S,S2), where S2 is the neighbor of S. Then it recursively emits complement pairs
  */
-template <typename TNodeSet> void TDPHypSolver<TNodeSet>::EmitCsg(const TNodeSet& s1) {
+template <typename TNodeSet, bool ProcessCycles> void TDPHypSolver<TNodeSet, ProcessCycles>::EmitCsg(const TNodeSet& s1) {
     TNodeSet x = s1 | MakeBiMin(s1);
     TNodeSet neighs = Neighs(s1, x);
 
@@ -326,8 +331,15 @@ template <typename TNodeSet> void TDPHypSolver<TNodeSet>::EmitCsg(const TNodeSet
             TNodeSet s2{};
             s2[i] = 1;
 
-            if (auto* edge = Graph_.FindEdgeBetween(s1, s2)) {
-                EmitCsgCmp(s1, s2, edge);
+            if constexpr (ProcessCycles) {
+                if (auto edge = Graph_.FindEdgeWithAllConditionsBetween(s1, s2)) {
+                    auto reversedEdge = edge->CreateReversed(-1);
+                    EmitCsgCmp(s1, s2, &edge.value(), &reversedEdge);
+                }
+            } else {
+                if (auto* edge = Graph_.FindEdgeBetween(s1, s2)) {
+                    EmitCsgCmp(s1, s2, edge, &Graph_.GetEdge(edge->ReversedEdgeId));
+                }
             }
 
             EnumerateCmpRec(s1, s2, x | MakeB(neighs, GetLowestSetBit(s2)));
@@ -341,7 +353,7 @@ template <typename TNodeSet> void TDPHypSolver<TNodeSet>::EmitCsg(const TNodeSet
  * that are obtained by adding S2's neighbors to itself
  * Then it recusrses into pairs (S1,S2+next)
  */
-template <typename TNodeSet> void TDPHypSolver<TNodeSet>::EnumerateCmpRec(const TNodeSet& s1, const TNodeSet& s2, const TNodeSet& x) {
+template <typename TNodeSet, bool ProcessCycles> void TDPHypSolver<TNodeSet, ProcessCycles>::EnumerateCmpRec(const TNodeSet& s1, const TNodeSet& s2, const TNodeSet& x) {
     TNodeSet neighs = Neighs(s2, x);
 
     if (neighs == TNodeSet{}) {
@@ -355,8 +367,15 @@ template <typename TNodeSet> void TDPHypSolver<TNodeSet>::EnumerateCmpRec(const
         next = NextBitset(prev, neighs);
 
         if (DpTable_.contains(s2 | next)) {
-            if (auto* edge = Graph_.FindEdgeBetween(s1, s2 | next)) {
-                EmitCsgCmp(s1, s2 | next, edge);
+            if constexpr (ProcessCycles) {
+                if (auto edge = Graph_.FindEdgeWithAllConditionsBetween(s1, s2 | next)) {
+                    auto reversedEdge = edge->CreateReversed(-1);
+                    EmitCsgCmp(s1, s2 | next, &edge.value(), &reversedEdge);
+                }
+            } else {
+                if (auto* edge = Graph_.FindEdgeBetween(s1, s2 | next)) {
+                    EmitCsgCmp(s1, s2 | next, edge, &Graph_.GetEdge(edge->ReversedEdgeId));
+                }
             }
         }
 
@@ -381,7 +400,7 @@ template <typename TNodeSet> void TDPHypSolver<TNodeSet>::EnumerateCmpRec(const
     }
 }
 
-template <typename TNodeSet> TNodeSet TDPHypSolver<TNodeSet>::MakeBiMin(const TNodeSet& s) {
+template <typename TNodeSet, bool ProcessCycles> TNodeSet TDPHypSolver<TNodeSet, ProcessCycles>::MakeBiMin(const TNodeSet& s) {
     TNodeSet res{};
 
     for (size_t i = 0; i < NNodes_; i++) {
@@ -395,7 +414,7 @@ template <typename TNodeSet> TNodeSet TDPHypSolver<TNodeSet>::MakeBiMin(const TN
     return res;
 }
 
-template <typename TNodeSet> TNodeSet TDPHypSolver<TNodeSet>::MakeB(const TNodeSet& s, size_t v) {
+template <typename TNodeSet, bool ProcessCycles> TNodeSet TDPHypSolver<TNodeSet, ProcessCycles>::MakeB(const TNodeSet& s, size_t v) {
     TNodeSet res{};
 
     for (size_t i = 0; i < NNodes_; i++) {
@@ -407,7 +426,7 @@ template <typename TNodeSet> TNodeSet TDPHypSolver<TNodeSet>::MakeB(const TNodeS
     return res;
 }
 
-template <typename TNodeSet> std::shared_ptr<TJoinOptimizerNodeInternal> TDPHypSolver<TNodeSet>::PickBestJoin(
+template <typename TNodeSet, bool ProcessCycles> std::shared_ptr<TJoinOptimizerNodeInternal> TDPHypSolver<TNodeSet, ProcessCycles>::PickBestJoin(
     const std::shared_ptr<IBaseOptimizerNode>& left,
     const std::shared_ptr<IBaseOptimizerNode>& right,
     EJoinKind joinKind,
@@ -434,7 +453,7 @@ template <typename TNodeSet> std::shared_ptr<TJoinOptimizerNodeInternal> TDPHypS
     for (auto joinAlgo : AllJoinAlgos) {
         if (ctx.IsJoinApplicable(left, right, joinConditions, leftJoinKeys, rightJoinKeys, joinAlgo, joinKind)){
             auto cost = ctx.ComputeJoinStats(*left->Stats, *right->Stats, leftJoinKeys, rightJoinKeys, joinAlgo, joinKind, maybeCardHint).Cost;
-            if (cost < bestCost) {
+            if (cost <= bestCost) {
                 bestCost = cost;
                 bestAlgo = joinAlgo;
                 bestJoinIsReversed = false;
@@ -444,7 +463,7 @@ template <typename TNodeSet> std::shared_ptr<TJoinOptimizerNodeInternal> TDPHypS
         if (isCommutative) {
             if (ctx.IsJoinApplicable(right, left, reversedJoinConditions, rightJoinKeys, leftJoinKeys, joinAlgo, joinKind)){
                 auto cost = ctx.ComputeJoinStats(*right->Stats, *left->Stats,  rightJoinKeys, leftJoinKeys, joinAlgo, joinKind, maybeCardHint).Cost;
-                if (cost < bestCost) {
+                if (cost <= bestCost) {
                     bestCost = cost;
                     bestAlgo = joinAlgo;
                     bestJoinIsReversed = true;
@@ -465,19 +484,23 @@ template <typename TNodeSet> std::shared_ptr<TJoinOptimizerNodeInternal> TDPHypS
 /* 
  * Emit a single CSG + CMP pair
  */
-template<typename TNodeSet> void TDPHypSolver<TNodeSet>::EmitCsgCmp(const TNodeSet& s1, const TNodeSet& s2, const typename TJoinHypergraph<TNodeSet>::TEdge* csgCmpEdge) {
+template<typename TNodeSet, bool ProcessCycles> void TDPHypSolver<TNodeSet, ProcessCycles>::EmitCsgCmp(
+    const TNodeSet& s1, 
+    const TNodeSet& s2, 
+    const typename TJoinHypergraph<TNodeSet>::TEdge* csgCmpEdge,
+    const typename TJoinHypergraph<TNodeSet>::TEdge* reversedCsgCmpEdge
+) {
     // Here we actually build the join and choose and compare the
     // new plan to what's in the dpTable, if it there
 
     Y_ENSURE(DpTable_.contains(s1), "DP Table does not contain S1");
     Y_ENSURE(DpTable_.contains(s2), "DP Table does not conaint S2");
 
-    const auto* reversedEdge = &Graph_.GetEdge(csgCmpEdge->ReversedEdgeId);
     auto leftNodes = DpTable_[s1];
     auto rightNodes = DpTable_[s2];
 
     if (csgCmpEdge->IsReversed) {
-        std::swap(csgCmpEdge, reversedEdge);
+        std::swap(csgCmpEdge, reversedCsgCmpEdge);
         std::swap(leftNodes, rightNodes);
     }
 
@@ -494,7 +517,7 @@ template<typename TNodeSet> void TDPHypSolver<TNodeSet>::EmitCsgCmp(const TNodeS
         csgCmpEdge->RightAny,
         csgCmpEdge->IsCommutative,
         csgCmpEdge->JoinConditions,
-        reversedEdge->JoinConditions,
+        reversedCsgCmpEdge->JoinConditions,
         csgCmpEdge->LeftJoinKeys,
         csgCmpEdge->RightJoinKeys,
         Pctx_,
@@ -509,7 +532,7 @@ template<typename TNodeSet> void TDPHypSolver<TNodeSet>::EmitCsgCmp(const TNodeS
     #ifndef NDEBUG
         auto pair = std::make_pair(s1, s2);
         Y_ENSURE (!CheckTable_.contains(pair), "Check table already contains pair S1|S2");
-        CheckTable_[ std::pair<TNodeSet,TNodeSet>(s1, s2) ] = true;
+        CheckTable_[pair] = true;
     #endif
 }