[yt provider] Check table content weight for local execution (#11207)

rvu1024 · web-flow · commit 895022080397 · 2024-11-02T11:23:09.000Z
diff --git a/ydb/library/yql/providers/yt/common/yql_configuration.h b/ydb/library/yql/providers/yt/common/yql_configuration.h
@@ -87,7 +87,7 @@ constexpr ui32 DEFAULT_MAX_INPUT_TABLES = 3000;
 constexpr ui32 DEFAULT_MAX_OUTPUT_TABLES = 100;
 constexpr ui64 DEFAULT_APPLY_STORED_CONSTRAINTS = 0ULL;
 
-constexpr bool DEFAULT_TABLE_CONTENT_LOCAL_EXEC = false;
+constexpr ui64 DEFAULT_TABLE_CONTENT_LOCAL_EXEC = 0;
 
 constexpr ui32 DEFAULT_BATCH_LIST_FOLDER_CONCURRENCY = 5;
 
diff --git a/ydb/library/yql/providers/yt/common/yql_yt_settings.cpp b/ydb/library/yql/providers/yt/common/yql_yt_settings.cpp
@@ -5,8 +5,8 @@
 #include <ydb/library/yql/public/udf/udf_data_type.h>
 
 #include <library/cpp/yson/node/node_io.h>
-
 #include <library/cpp/regex/pcre/regexp.h>
+#include <library/cpp/string_utils/parse_size/parse_size.h>
 
 #include <util/generic/yexception.h>
 #include <util/generic/size_literals.h>
@@ -252,7 +252,17 @@ TYtConfiguration::TYtConfiguration(TTypeAnnotationContext& typeCtx)
     REGISTER_SETTING(*this, TableContentTmpFolder);
     REGISTER_SETTING(*this, TableContentColumnarStatistics);
     REGISTER_SETTING(*this, TableContentUseSkiff);
-    REGISTER_SETTING(*this, TableContentLocalExecution);
+    REGISTER_SETTING(*this, TableContentLocalExecution)
+        .Parser([](const TString& v) {
+            // backward compatible parse from bool
+            bool value = true;
+            if (!v || TryFromString<bool>(v, value)) {
+                return value ? 10_MB : 0_MB;
+            } else {
+                return NSize::ParseSize(v);
+            }
+        })
+        .Upper(5_GB);
     REGISTER_SETTING(*this, DisableJobSplitting);
     REGISTER_SETTING(*this, UseColumnarStatistics)
         .Parser([](const TString& v) {
diff --git a/ydb/library/yql/providers/yt/common/yql_yt_settings.h b/ydb/library/yql/providers/yt/common/yql_yt_settings.h
@@ -250,7 +250,7 @@ struct TYtSettings {
     NCommon::TConfSetting<NSize::TSize, false> TableContentMinAvgChunkSize;
     NCommon::TConfSetting<ui32, false> TableContentMaxInputTables;
     NCommon::TConfSetting<ui32, false> TableContentMaxChunksForNativeDelivery;
-    NCommon::TConfSetting<bool, false> TableContentLocalExecution;
+    NCommon::TConfSetting<NSize::TSize, false> TableContentLocalExecution;
     NCommon::TConfSetting<bool, false> UseTypeV2;
     NCommon::TConfSetting<bool, false> UseNativeYtTypes;
     NCommon::TConfSetting<bool, false> UseNativeDescSort;
diff --git a/ydb/library/yql/providers/yt/gateway/native/yql_yt_transform.cpp b/ydb/library/yql/providers/yt/gateway/native/yql_yt_transform.cpp
@@ -71,11 +71,13 @@ TGatewayTransformer::TGatewayTransformer(const TExecContextBase& execCtx, TYtSet
     }
 }
 
-TCallableVisitFunc TGatewayTransformer::operator()(TInternName name) {
+TCallableVisitFunc TGatewayTransformer::operator()(TInternName internName) {
+    auto name = internName.Str();
+    const bool small = name.SkipPrefix("Small");
     if (name == TYtTableContent::CallableName()) {
 
         *TableContentFlag_ = true;
-        *RemoteExecutionFlag_ = *RemoteExecutionFlag_ || !Settings_->TableContentLocalExecution.Get().GetOrElse(DEFAULT_TABLE_CONTENT_LOCAL_EXEC);
+        *RemoteExecutionFlag_ = *RemoteExecutionFlag_ || !small;
 
         if (EPhase::Content == Phase_ || EPhase::All == Phase_) {
             return [&](NMiniKQL::TCallable& callable, const TTypeEnvironment& env) {
diff --git a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp
@@ -23,7 +23,7 @@ TMaybe<bool> TYtPhysicalOptProposalTransformer::CanFuseLambdas(const TCoLambda&
 
     TExprNode::TPtr updatedBody = innerLambda.Body().Ptr();
     if (maxJobMemoryLimit) {
-        auto status = UpdateTableContentMemoryUsage(innerLambda.Body().Ptr(), updatedBody, State_, ctx);
+        auto status = UpdateTableContentMemoryUsage(innerLambda.Body().Ptr(), updatedBody, State_, ctx, false);
         if (status.Level != TStatus::Ok) {
             return {};
         }
@@ -36,7 +36,7 @@ TMaybe<bool> TYtPhysicalOptProposalTransformer::CanFuseLambdas(const TCoLambda&
 
     updatedBody = outerLambda.Body().Ptr();
     if (maxJobMemoryLimit) {
-        auto status = UpdateTableContentMemoryUsage(outerLambda.Body().Ptr(), updatedBody, State_, ctx);
+        auto status = UpdateTableContentMemoryUsage(outerLambda.Body().Ptr(), updatedBody, State_, ctx, false);
         if (status.Level != TStatus::Ok) {
             return {};
         }
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_datasink_exec.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_datasink_exec.cpp
@@ -227,7 +227,7 @@ class TYtDataSinkExecTransformer : public TExecTransformerBase {
         }
 
         bool hasNonDeterministicFunctions = false;
-        if (const auto status = PeepHoleOptimizeBeforeExec(optimizedNode, optimizedNode, State_, hasNonDeterministicFunctions, ctx); status.Level != TStatus::Ok) {
+        if (const auto status = PeepHoleOptimizeBeforeExec(optimizedNode, optimizedNode, State_, hasNonDeterministicFunctions, ctx, false); status.Level != TStatus::Ok) {
             return SyncStatus(status);
         }
 
@@ -652,7 +652,7 @@ class TYtDataSinkExecTransformer : public TExecTransformerBase {
         }
 
         bool hasNonDeterministicFunctions = false;
-        if (const auto status = PeepHoleOptimizeBeforeExec(optimizedNode, optimizedNode, State_, hasNonDeterministicFunctions, ctx); status.Level != TStatus::Ok) {
+        if (const auto status = PeepHoleOptimizeBeforeExec(optimizedNode, optimizedNode, State_, hasNonDeterministicFunctions, ctx, false); status.Level != TStatus::Ok) {
             return SyncStatus(status);
         }
 
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_datasource_exec.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_datasource_exec.cpp
@@ -135,7 +135,7 @@ class TYtDataSourceExecTransformer : public TExecTransformerBase {
         }
 
         bool hasNonDeterministicFunctions = false;
-        if (const auto status = PeepHoleOptimizeBeforeExec(optimizedInput, optimizedInput, State_, hasNonDeterministicFunctions, ctx); status.Level != IGraphTransformer::TStatus::Ok) {
+        if (const auto status = PeepHoleOptimizeBeforeExec(optimizedInput, optimizedInput, State_, hasNonDeterministicFunctions, ctx, true); status.Level != IGraphTransformer::TStatus::Ok) {
             return SyncStatus(status);
         }
 
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_datasource_type_ann.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_datasource_type_ann.cpp
@@ -846,7 +846,8 @@ class TYtDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {
             return TStatus::Error;
         }
 
-        if (!ValidateSettings(tableContent.Settings().Ref(), EYtSettingType::MemUsage | EYtSettingType::ItemsCount | EYtSettingType::RowFactor | EYtSettingType::Split, ctx)) {
+        const EYtSettingTypes allowed = EYtSettingType::MemUsage | EYtSettingType::ItemsCount | EYtSettingType::RowFactor | EYtSettingType::Split | EYtSettingType::Small;
+        if (!ValidateSettings(tableContent.Settings().Ref(), allowed, ctx)) {
             return TStatus::Error;
         }
 
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp
@@ -2,6 +2,7 @@
 #include "yql_yt_provider_impl.h"
 #include "yql_yt_op_settings.h"
 #include "yql_yt_op_hash.h"
+#include "yql_yt_optimize.h"
 
 #include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/lib/mkql_helpers/mkql_helpers.h>
@@ -712,6 +713,11 @@ std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> CalculateNo
     auto typeTransformer = CreateTypeAnnotationTransformer(callableTransformer, *state->Types);
 
     TExprNode::TPtr optimized;
+    status = UpdateTableContentMemoryUsage(list, optimized, state, ctx, true);
+    if (status.Level == IGraphTransformer::TStatus::Error) {
+        return SyncStatus(status);
+    }
+
     bool hasNonDeterministicFunctions = false;
     status = PeepHoleOptimizeNode(list, optimized, ctx, *state->Types, typeTransformer.Get(), hasNonDeterministicFunctions);
     if (status.Level == IGraphTransformer::TStatus::Error) {
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_horizontal_join.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_horizontal_join.cpp
@@ -665,7 +665,7 @@ TExprNode::TPtr THorizontalJoinOptimizer::HandleList(const TExprNode::TPtr& node
 
             TExprNode::TPtr updatedLambda = map.Mapper().Ptr();
             if (MaxJobMemoryLimit) {
-                auto status = UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx);
+                auto status = UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx, false);
                 if (status.Level != IGraphTransformer::TStatus::Ok) {
                     return {};
                 }
@@ -1346,7 +1346,7 @@ bool TMultiHorizontalJoinOptimizer::HandleGroup(const TVector<TYtMap>& maps, TEx
 
         TExprNode::TPtr updatedLambda = map.Mapper().Ptr();
         if (MaxJobMemoryLimit) {
-            if (UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx).Level != IGraphTransformer::TStatus::Ok) {
+            if (UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx, false).Level != IGraphTransformer::TStatus::Ok) {
                 return false;
             }
         }
@@ -1734,7 +1734,7 @@ bool TOutHorizontalJoinOptimizer::HandleGroup(TPositionHandle pos, const TGroupK
 
         TExprNode::TPtr updatedLambda = map.Mapper().Ptr();
         if (MaxJobMemoryLimit) {
-            if (UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx).Level != IGraphTransformer::TStatus::Ok) {
+            if (UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx, false).Level != IGraphTransformer::TStatus::Ok) {
                 return false;
             }
         }
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.cpp
@@ -452,17 +452,21 @@ void RegisterYtMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler) {
         [](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) {
             TYtTableContent tableContent(&node);
             TMaybe<ui64> itemsCount;
+            TString name = ToString(TYtTableContent::CallableName());
             if (auto setting = NYql::GetSetting(tableContent.Settings().Ref(), EYtSettingType::ItemsCount)) {
                 itemsCount = FromString<ui64>(setting->Child(1)->Content());
             }
+            if (NYql::HasSetting(tableContent.Settings().Ref(), EYtSettingType::Small)) {
+                name.prepend("Small");
+            }
             if (auto maybeRead = tableContent.Input().Maybe<TYtReadTable>()) {
                 auto read = maybeRead.Cast();
-                return BuildTableContentCall(
+                return BuildTableContentCall(name,
                     NCommon::BuildType(node, *node.GetTypeAnn()->Cast<TListExprType>()->GetItemType(), ctx.ProgramBuilder),
                     read.DataSource().Cluster().Value(), read.Input().Ref(), itemsCount, ctx, true);
             } else {
                 auto output = tableContent.Input().Cast<TYtOutput>();
-                return BuildTableContentCall(
+                return BuildTableContentCall(name,
                     NCommon::BuildType(node, *node.GetTypeAnn()->Cast<TListExprType>()->GetItemType(), ctx.ProgramBuilder),
                     GetOutputOp(output).DataSink().Cluster().Value(), output.Ref(), itemsCount, ctx, true);
             }
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.cpp
@@ -441,6 +441,7 @@ bool ValidateSettings(const TExprNode& settingsNode, EYtSettingTypes accepted, T
         case EYtSettingType::MonotonicKeys:
         case EYtSettingType::BlockInputReady:
         case EYtSettingType::BlockInputApplied:
+        case EYtSettingType::Small:
             if (!EnsureTupleSize(*setting, 1, ctx)) {
                 return false;
             }
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h b/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h
@@ -74,6 +74,7 @@ enum class EYtSettingType: ui64 {
     MemUsage                 /* "memUsage" */,
     ItemsCount               /* "itemsCount" */,
     RowFactor                /* "rowFactor" */,
+    Small                    /* "small" */,
     // Operations
     Ordered                  /* "ordered" */,                  // hybrid supported
     KeyFilter                /* "keyFilter" */,
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_optimize.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_optimize.cpp
@@ -424,7 +424,9 @@ TExprNode::TPtr OptimizeReadWithSettings(const TExprNode::TPtr& node, bool allow
     return res;
 }
 
-IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr& input, TExprNode::TPtr& output, const TYtState::TPtr& state, TExprContext& ctx) {
+IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr& input, TExprNode::TPtr& output, const TYtState::TPtr& state,
+    TExprContext& ctx, bool estimateTableContentWeight)
+{
     auto current = input;
     output.Reset();
     for (;;) {
@@ -447,10 +449,10 @@ IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr&
 
         TExprNode::TPtr newCurrent;
         auto status = OptimizeExpr(current, newCurrent,
-            [&parentsMap, current, state](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
+            [&parentsMap, current, state, estimateTableContentWeight](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
                 if (auto maybeContent = TMaybeNode<TYtTableContent>(node)) {
                     auto content = maybeContent.Cast();
-                    if (NYql::HasSetting(content.Settings().Ref(), EYtSettingType::MemUsage)) {
+                    if (NYql::HasAnySetting(content.Settings().Ref(), EYtSettingType::MemUsage | EYtSettingType::Small)) {
                         return node;
                     }
 
@@ -498,6 +500,7 @@ IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr&
                     }
 
                     ui64 memUsage = 0;
+                    ui64 dataWeight = 0;
                     ui64 itemsCount = 0;
                     bool useItemsCount = !NYql::HasSetting(content.Settings().Ref(), EYtSettingType::ItemsCount);
 
@@ -506,7 +509,7 @@ IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr&
                         memUsage = 16_MB;
                         useItemsCount = false;
                     }
-                    else {
+                    if (estimateTableContentWeight || !factors.empty()) {
                         if (auto maybeRead = content.Input().Maybe<TYtReadTable>()) {
                             TVector<ui64> records;
                             TVector<TYtPathInfo::TPtr> tableInfos;
@@ -556,6 +559,7 @@ IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr&
                                         for (auto& factor: factors) {
                                             memUsage += factor.first * dataSizes->at(i) + factor.second * records.at(i);
                                         }
+                                        dataWeight += dataSizes->at(i);
                                     }
                                 } else {
                                     return {};
@@ -571,6 +575,7 @@ IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr&
                                     memUsage += factor.first * dataSize + factor.second * records;
                                 }
                                 itemsCount += records;
+                                dataWeight += dataSize;
                             }
                             else {
                                 YQL_CLOG(INFO, ProviderYt) << "Assume 1Gb memory usage for YtTableContent #"
@@ -586,6 +591,9 @@ IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr&
                     if (useItemsCount) {
                         settings = NYql::AddSetting(*settings, EYtSettingType::ItemsCount, ctx.NewAtom(node->Pos(), ToString(itemsCount), TNodeFlags::Default), ctx);
                     }
+                    if (estimateTableContentWeight && dataWeight < state->Configuration->TableContentLocalExecution.Get().GetOrElse(DEFAULT_TABLE_CONTENT_LOCAL_EXEC)) {
+                        settings = NYql::AddSetting(*settings, EYtSettingType::Small, {}, ctx);
+                    }
 
                     return ctx.WrapByCallableIf(wrapToCollect, "Collect", ctx.ChangeChild(*node, TYtTableContent::idx_Settings, std::move(settings)));
                 }
@@ -650,9 +658,9 @@ struct TPeepholePipelineConfigurator : public IPipelineConfigurator {
 };
 
 IGraphTransformer::TStatus PeepHoleOptimizeBeforeExec(TExprNode::TPtr input, TExprNode::TPtr& output,
-    const TYtState::TPtr& state, bool& hasNonDeterministicFunctions, TExprContext& ctx)
+    const TYtState::TPtr& state, bool& hasNonDeterministicFunctions, TExprContext& ctx, bool estimateTableContentWeight)
 {
-    if (const auto status = UpdateTableContentMemoryUsage(input, output, state, ctx);
+    if (const auto status = UpdateTableContentMemoryUsage(input, output, state, ctx, estimateTableContentWeight);
         status.Level != IGraphTransformer::TStatus::Ok) {
         return status;
     }
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_optimize.h b/ydb/library/yql/providers/yt/provider/yql_yt_optimize.h
@@ -29,9 +29,9 @@ TExprNode::TPtr OptimizeReadWithSettings(const TExprNode::TPtr& node, bool allow
     const TYtState::TPtr& state, TExprContext& ctx);
 
 IGraphTransformer::TStatus UpdateTableContentMemoryUsage(const TExprNode::TPtr& input, TExprNode::TPtr& output,
-    const TYtState::TPtr& state, TExprContext& ctx);
+    const TYtState::TPtr& state, TExprContext& ctx, bool estimateTableContentWeight);
 
 IGraphTransformer::TStatus PeepHoleOptimizeBeforeExec(TExprNode::TPtr input, TExprNode::TPtr& output,
-    const TYtState::TPtr& state, bool& hasNonDeterministicFunctions, TExprContext& ctx);
+    const TYtState::TPtr& state, bool& hasNonDeterministicFunctions, TExprContext& ctx, bool estimateTableContentWeight);
 
 } //NYql
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp
@@ -2507,7 +2507,7 @@ class TYtPhysicalFinalizingTransformer : public TSyncTransformerBase {
 
                     TExprNode::TPtr updatedBody = lambda.Body().Ptr();
                     if (maxJobMemoryLimit) {
-                        auto status = UpdateTableContentMemoryUsage(lambda.Body().Ptr(), updatedBody, State_, ctx);
+                        auto status = UpdateTableContentMemoryUsage(lambda.Body().Ptr(), updatedBody, State_, ctx, false);
                         if (status.Level != TStatus::Ok) {
                             return status;
                         }
@@ -2526,7 +2526,7 @@ class TYtPhysicalFinalizingTransformer : public TSyncTransformerBase {
 
                                 updatedBody = outerMap.Mapper().Body().Ptr();
                                 if (maxJobMemoryLimit) {
-                                    auto status = UpdateTableContentMemoryUsage(outerMap.Mapper().Body().Ptr(), updatedBody, State_, ctx);
+                                    auto status = UpdateTableContentMemoryUsage(outerMap.Mapper().Body().Ptr(), updatedBody, State_, ctx, false);
                                     if (status.Level != TStatus::Ok) {
                                         return status;
                                     }

Original file line number	Diff line number	Diff line change
`@@ -71,11 +71,13 @@ TGatewayTransformer::TGatewayTransformer(const TExecContextBase& execCtx, TYtSet`
`71`	`71`	`}`
`72`	`72`	`}`
`73`	`73`
`74`		`-TCallableVisitFunc TGatewayTransformer::operator()(TInternName name) {`
	`74`	`+TCallableVisitFunc TGatewayTransformer::operator()(TInternName internName) {`
	`75`	`+ auto name = internName.Str();`
	`76`	`+ const bool small = name.SkipPrefix("Small");`
`75`	`77`	`if (name == TYtTableContent::CallableName()) {`
`76`	`78`
`77`	`79`	`*TableContentFlag_ = true;`
`78`		`- RemoteExecutionFlag_ = RemoteExecutionFlag_ \|\| !Settings_->TableContentLocalExecution.Get().GetOrElse(DEFAULT_TABLE_CONTENT_LOCAL_EXEC);`
	`80`	`+ RemoteExecutionFlag_ = RemoteExecutionFlag_ \|\| !small;`
`79`	`81`
`80`	`82`	`if (EPhase::Content == Phase_ \|\| EPhase::All == Phase_) {`
`81`	`83`	`return [&](NMiniKQL::TCallable& callable, const TTypeEnvironment& env) {`
Original file line number	Diff line number	Diff line change
`@@ -227,7 +227,7 @@ class TYtDataSinkExecTransformer : public TExecTransformerBase {`
`227`	`227`	`}`
`228`	`228`
`229`	`229`	`bool hasNonDeterministicFunctions = false;`
`230`		`- if (const auto status = PeepHoleOptimizeBeforeExec(optimizedNode, optimizedNode, State_, hasNonDeterministicFunctions, ctx); status.Level != TStatus::Ok) {`
	`230`	`+ if (const auto status = PeepHoleOptimizeBeforeExec(optimizedNode, optimizedNode, State_, hasNonDeterministicFunctions, ctx, false); status.Level != TStatus::Ok) {`
`231`	`231`	`return SyncStatus(status);`
`232`	`232`	`}`
`233`	`233`
`@@ -652,7 +652,7 @@ class TYtDataSinkExecTransformer : public TExecTransformerBase {`
`652`	`652`	`}`
`653`	`653`
`654`	`654`	`bool hasNonDeterministicFunctions = false;`
`655`		`- if (const auto status = PeepHoleOptimizeBeforeExec(optimizedNode, optimizedNode, State_, hasNonDeterministicFunctions, ctx); status.Level != TStatus::Ok) {`
	`655`	`+ if (const auto status = PeepHoleOptimizeBeforeExec(optimizedNode, optimizedNode, State_, hasNonDeterministicFunctions, ctx, false); status.Level != TStatus::Ok) {`
`656`	`656`	`return SyncStatus(status);`
`657`	`657`	`}`
`658`	`658`
Original file line number	Diff line number	Diff line change
`@@ -135,7 +135,7 @@ class TYtDataSourceExecTransformer : public TExecTransformerBase {`
`135`	`135`	`}`
`136`	`136`
`137`	`137`	`bool hasNonDeterministicFunctions = false;`
`138`		`- if (const auto status = PeepHoleOptimizeBeforeExec(optimizedInput, optimizedInput, State_, hasNonDeterministicFunctions, ctx); status.Level != IGraphTransformer::TStatus::Ok) {`
	`138`	`+ if (const auto status = PeepHoleOptimizeBeforeExec(optimizedInput, optimizedInput, State_, hasNonDeterministicFunctions, ctx, true); status.Level != IGraphTransformer::TStatus::Ok) {`
`139`	`139`	`return SyncStatus(status);`
`140`	`140`	`}`
`141`	`141`
Original file line number	Diff line number	Diff line change
`@@ -846,7 +846,8 @@ class TYtDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {`
`846`	`846`	`return TStatus::Error;`
`847`	`847`	`}`
`848`	`848`
`849`		`- if (!ValidateSettings(tableContent.Settings().Ref(), EYtSettingType::MemUsage \| EYtSettingType::ItemsCount \| EYtSettingType::RowFactor \| EYtSettingType::Split, ctx)) {`
	`849`	`+ const EYtSettingTypes allowed = EYtSettingType::MemUsage \| EYtSettingType::ItemsCount \| EYtSettingType::RowFactor \| EYtSettingType::Split \| EYtSettingType::Small;`
	`850`	`+ if (!ValidateSettings(tableContent.Settings().Ref(), allowed, ctx)) {`
`850`	`851`	`return TStatus::Error;`
`851`	`852`	`}`
`852`	`853`
Original file line number	Diff line number	Diff line change
`@@ -665,7 +665,7 @@ TExprNode::TPtr THorizontalJoinOptimizer::HandleList(const TExprNode::TPtr& node`
`665`	`665`
`666`	`666`	`TExprNode::TPtr updatedLambda = map.Mapper().Ptr();`
`667`	`667`	`if (MaxJobMemoryLimit) {`
`668`		`- auto status = UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx);`
	`668`	`+ auto status = UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx, false);`
`669`	`669`	`if (status.Level != IGraphTransformer::TStatus::Ok) {`
`670`	`670`	`return {};`
`671`	`671`	`}`
`@@ -1346,7 +1346,7 @@ bool TMultiHorizontalJoinOptimizer::HandleGroup(const TVector<TYtMap>& maps, TEx`
`1346`	`1346`
`1347`	`1347`	`TExprNode::TPtr updatedLambda = map.Mapper().Ptr();`
`1348`	`1348`	`if (MaxJobMemoryLimit) {`
`1349`		`- if (UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx).Level != IGraphTransformer::TStatus::Ok) {`
	`1349`	`+ if (UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx, false).Level != IGraphTransformer::TStatus::Ok) {`
`1350`	`1350`	`return false;`
`1351`	`1351`	`}`
`1352`	`1352`	`}`
`@@ -1734,7 +1734,7 @@ bool TOutHorizontalJoinOptimizer::HandleGroup(TPositionHandle pos, const TGroupK`
`1734`	`1734`
`1735`	`1735`	`TExprNode::TPtr updatedLambda = map.Mapper().Ptr();`
`1736`	`1736`	`if (MaxJobMemoryLimit) {`
`1737`		`- if (UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx).Level != IGraphTransformer::TStatus::Ok) {`
	`1737`	`+ if (UpdateTableContentMemoryUsage(map.Mapper().Ptr(), updatedLambda, State_, ctx, false).Level != IGraphTransformer::TStatus::Ok) {`
`1738`	`1738`	`return false;`
`1739`	`1739`	`}`
`1740`	`1740`	`}`