Skip to content

Commit 514ea79

Browse files
alexv-smirnovalexv-smirnov
authored
Merge 8f7c3ef into 61e223d 
2 parents 61e223d + 8f7c3ef commit 514ea79

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1571
-99
lines changed

‎build/conf/compilers/gnu_compiler.conf‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,13 @@ when ($MSAN_TRACK_ORIGIN == "yes") {
8484

8585
when ($ARCH_XTENSA == "yes") {
8686
FSTACK=
87+
CFLAGS+=-Wno-c++14-extensions
88+
when ($ARCH_XTENSA_HIFI4 == "yes") {
89+
CFLAGS+=-Wno-c++1z-extensions
90+
}
91+
otherwise {
92+
CFLAGS+=-Wno-c++17-extensions
93+
}
8794
}
8895

8996
when ($OS_EMSCRIPTEN == "yes") {

‎ydb/ci/rightlib.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
23e9865bb938b83e7e32b670ba055c407f75494b
1+
796e6186c6652f49958e68c7eb0f06c52827e702

‎ydb/core/kqp/opt/kqp_opt_kql.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1017,7 +1017,7 @@ TMaybe<TKqlQueryList> BuildKqlQuery(TKiDataQueryBlocks dataQueryBlocks, const TK
10171017
auto dataSource = typesCtx.DataSourceMap.FindPtr(dataSourceName);
10181018
YQL_ENSURE(dataSource);
10191019
if (auto dqIntegration = (*dataSource)->GetDqIntegration()) {
1020-
auto newRead = dqIntegration->WrapRead(NYql::TDqSettings(), input.Cast().Ptr(), ctx);
1020+
auto newRead = dqIntegration->WrapRead(input.Cast().Ptr(), ctx, {});
10211021
if (newRead.Get() != input.Raw()) {
10221022
return newRead;
10231023
}

‎ydb/core/kqp/query_compiler/kqp_query_compiler.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1092,7 +1092,10 @@ class TKqpQueryCompiler : public IKqpQueryCompiler {
10921092
// We prepare a lot of partitions and distribute them between these tasks
10931093
// Constraint of 1 task per partition is NOT valid anymore
10941094
auto maxTasksPerStage = Config->MaxTasksPerStage.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerStage);
1095-
dqIntegration->Partition(NYql::TDqSettings(), maxTasksPerStage, source.Ref(), partitionParams, &clusterName, ctx, false);
1095+
IDqIntegration::TPartitionSettings pSettings;
1096+
pSettings.MaxPartitions = maxTasksPerStage;
1097+
pSettings.CanFallback = false;
1098+
dqIntegration->Partition(source.Ref(), partitionParams, &clusterName, ctx, pSettings);
10961099
externalSource.SetTaskParamKey(TString(dataSourceCategory));
10971100
for (const TString& partitionParam : partitionParams) {
10981101
externalSource.AddPartitionedTaskParams(partitionParam);

‎ydb/library/yql/dq/opt/dq_opt_log.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,7 @@ NNodes::TExprBase DqReplicateFieldSubset(NNodes::TExprBase node, TExprContext& c
336336
return node;
337337
}
338338

339-
IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& typesCtx, const TDqSettings& config) {
339+
IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& typesCtx, const IDqIntegration::TWrapReadSettings& wrSettings) {
340340
TOptimizeExprSettings settings{&typesCtx};
341341
auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) {
342342
if (auto maybeRead = TMaybeNode<TCoRight>(node).Input()) {
@@ -345,7 +345,7 @@ IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPt
345345
auto dataSource = typesCtx.DataSourceMap.FindPtr(dataSourceName);
346346
YQL_ENSURE(dataSource);
347347
if (auto dqIntegration = (*dataSource)->GetDqIntegration()) {
348-
auto newRead = dqIntegration->WrapRead(config, maybeRead.Cast().Ptr(), ctx);
348+
auto newRead = dqIntegration->WrapRead(maybeRead.Cast().Ptr(), ctx, wrSettings);
349349
if (newRead.Get() != maybeRead.Raw()) {
350350
return newRead;
351351
}

‎ydb/library/yql/dq/opt/dq_opt_log.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include <yql/essentials/core/dq_integration/yql_dq_integration.h>
34
#include <yql/essentials/ast/yql_expr.h>
45
#include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
56
#include <yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h>
@@ -11,7 +12,6 @@
1112
namespace NYql {
1213
class IOptimizationContext;
1314
struct TTypeAnnotationContext;
14-
struct TDqSettings;
1515
struct IProviderContext;
1616
struct TRelOptimizerNode;
1717
struct TOptimizerStatistics;
@@ -38,7 +38,7 @@ NNodes::TExprBase DqSqlInDropCompact(NNodes::TExprBase node, TExprContext& ctx);
3838

3939
NNodes::TExprBase DqReplicateFieldSubset(NNodes::TExprBase node, TExprContext& ctx, const TParentsMap& parents);
4040

41-
IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& typesCtx, const TDqSettings& config);
41+
IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& typesCtx, const IDqIntegration::TWrapReadSettings& wrSettings);
4242

4343
NNodes::TExprBase DqExpandMatchRecognize(NNodes::TExprBase node, TExprContext& ctx, TTypeAnnotationContext& typeAnnCtx);
4444

‎ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ class TClickHouseDqIntegration: public TDqIntegrationBase {
3333
return Nothing();
3434
}
3535

36-
TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr& read, TExprContext& ctx) override {
36+
TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& ) override {
3737
if (const auto maybeClReadTable = TMaybeNode<TClReadTable>(read)) {
3838
const auto clReadTable = maybeClReadTable.Cast();
3939
const auto token = TString("cluster:default_") += clReadTable.DataSource().Cluster().StringValue();
@@ -66,7 +66,7 @@ class TClickHouseDqIntegration: public TDqIntegrationBase {
6666
return read;
6767
}
6868

69-
ui64 Partition(const TDqSettings&, size_t, const TExprNode&, TVector<TString>& partitions, TString*, TExprContext&, bool) override {
69+
ui64 Partition(const TExprNode&, TVector<TString>& partitions, TString*, TExprContext&, const TPartitionSettings&) override {
7070
partitions.clear();
7171
NCH::TRange range;
7272
// range.SetRange("limit 42 offset 42 order by ...."); // Possible set range like this.

‎ydb/library/yql/providers/dq/planner/execution_planner.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,13 @@ namespace NYql::NDqs {
544544
TVector<TString> parts;
545545
if (auto dqIntegration = (*datasource)->GetDqIntegration()) {
546546
TString clusterName;
547-
_MaxDataSizePerJob = Max(_MaxDataSizePerJob, dqIntegration->Partition(*Settings, maxPartitions, *read, parts, &clusterName, ExprContext, canFallback));
547+
IDqIntegration::TPartitionSettings settings {
548+
.DataSizePerJob = Settings->DataSizePerJob.Get(),
549+
.MaxPartitions = maxPartitions,
550+
.EnableComputeActor = Settings->EnableComputeActor.Get(),
551+
.CanFallback = canFallback
552+
};
553+
_MaxDataSizePerJob = Max(_MaxDataSizePerJob, dqIntegration->Partition(*read, parts, &clusterName, ExprContext, settings));
548554
TMaybe<::google::protobuf::Any> sourceSettings;
549555
TString sourceType;
550556
if (dqSource) {
@@ -585,7 +591,7 @@ namespace NYql::NDqs {
585591
YQL_ENSURE(dataSource);
586592
auto dqIntegration = (*dataSource)->GetDqIntegration();
587593
YQL_ENSURE(dqIntegration);
588-
594+
589595
google::protobuf::Any providerSpecificLookupSourceSettings;
590596
TString sourceType;
591597
dqIntegration->FillLookupSourceSettings(*rightInput.Raw(), providerSpecificLookupSourceSettings, sourceType);

‎ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,13 @@ class TDqsRecaptureTransformer : public TSyncTransformerBase {
9797

9898
State_->TypeCtx->DqFallbackPolicy = State_->Settings->FallbackPolicy.Get().GetOrElse(EFallbackPolicy::Default);
9999

100-
IGraphTransformer::TStatus status = NDq::DqWrapIO(input, output, ctx, *State_->TypeCtx, *State_->Settings);
100+
IDqIntegration::TWrapReadSettings wrSettings {
101+
.WatermarksMode = State_->Settings->WatermarksMode.Get(),
102+
.WatermarksGranularityMs = State_->Settings->WatermarksGranularityMs.Get(),
103+
.WatermarksLateArrivalDelayMs = State_->Settings->WatermarksLateArrivalDelayMs.Get(),
104+
.WatermarksEnableIdlePartitions = State_->Settings->WatermarksEnableIdlePartitions.Get()
105+
};
106+
IGraphTransformer::TStatus status = NDq::DqWrapIO(input, output, ctx, *State_->TypeCtx, wrSettings);
101107
if (input != output) {
102108
YQL_CLOG(INFO, ProviderDq) << "DqsRecapture";
103109
// TODO: Add before/after recapture transformers

‎ydb/library/yql/providers/generic/provider/ut/pushdown/pushdown_ut.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ class TBuildDqSourceSettingsTransformer: public TOptimizeTransformerBase {
162162
UNIT_ASSERT(genericDataSource != Types->DataSourceMap.end());
163163
auto dqIntegration = genericDataSource->second->GetDqIntegration();
164164
UNIT_ASSERT(dqIntegration);
165-
auto newRead = dqIntegration->WrapRead(TDqSettings(), input.Ptr(), ctx);
165+
auto newRead = dqIntegration->WrapRead(input.Ptr(), ctx, IDqIntegration::TWrapReadSettings{});
166166
BuildSettings(newRead, dqIntegration, ctx);
167167
return newRead;
168168
}

0 commit comments

Comments
 (0)