Skip to content

Commit 10fc4f2

Browse files
maximyurchukmaximyurchuk
authored
Merge pull request #12104 from ydb-platform/mergelibs-241128-1502
Library import 241128-1502
2 parents f7b8420 + c069549 commit 10fc4f2

File tree

70 files changed

+1780
-261
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+1780
-261
lines changed

‎build/conf/compilers/gnu_compiler.conf‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,13 @@ when ($MSAN_TRACK_ORIGIN == "yes") {
8484

8585
when ($ARCH_XTENSA == "yes") {
8686
FSTACK=
87+
CFLAGS+=-Wno-c++14-extensions
88+
when ($ARCH_XTENSA_HIFI4 == "yes") {
89+
CFLAGS+=-Wno-c++1z-extensions
90+
}
91+
otherwise {
92+
CFLAGS+=-Wno-c++17-extensions
93+
}
8794
}
8895

8996
when ($OS_EMSCRIPTEN == "yes") {

‎ydb/ci/rightlib.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
23e9865bb938b83e7e32b670ba055c407f75494b
1+
796e6186c6652f49958e68c7eb0f06c52827e702

‎ydb/core/kqp/host/kqp_host.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <ydb/library/yql/providers/generic/provider/yql_generic_state.h>
2727
#include <ydb/library/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
2828
#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h>
29+
#include <ydb/library/yql/providers/dq/helper/yql_dq_helper_impl.h>
2930
#include <yql/essentials/minikql/invoke_builtins/mkql_builtins.h>
3031

3132
#include <library/cpp/cache/cache.h>
@@ -1818,7 +1819,7 @@ class TKqpHost : public IKqpHost {
18181819
}
18191820

18201821
TString sessionId = CreateGuidAsString();
1821-
auto [ytState, statWriter] = CreateYtNativeState(FederatedQuerySetup->YtGateway, userName, sessionId, &FederatedQuerySetup->YtGatewayConfig, TypesCtx, NDq::MakeCBOOptimizerFactory());
1822+
auto [ytState, statWriter] = CreateYtNativeState(FederatedQuerySetup->YtGateway, userName, sessionId, &FederatedQuerySetup->YtGatewayConfig, TypesCtx, NDq::MakeCBOOptimizerFactory(), MakeDqHelper());
18221823

18231824
ytState->PassiveExecution = true;
18241825
ytState->Gateway->OpenSession(

‎ydb/core/kqp/host/ya.make‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ PEERDIR(
2525
yql/essentials/core
2626
yql/essentials/providers/common/codec
2727
ydb/library/yql/dq/opt
28+
ydb/library/yql/providers/dq/helper
2829
ydb/library/yql/providers/common/http_gateway
2930
yql/essentials/providers/common/udf_resolve
3031
yql/essentials/providers/config

‎ydb/core/kqp/opt/kqp_opt_kql.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1017,7 +1017,7 @@ TMaybe<TKqlQueryList> BuildKqlQuery(TKiDataQueryBlocks dataQueryBlocks, const TK
10171017
auto dataSource = typesCtx.DataSourceMap.FindPtr(dataSourceName);
10181018
YQL_ENSURE(dataSource);
10191019
if (auto dqIntegration = (*dataSource)->GetDqIntegration()) {
1020-
auto newRead = dqIntegration->WrapRead(NYql::TDqSettings(), input.Cast().Ptr(), ctx);
1020+
auto newRead = dqIntegration->WrapRead(input.Cast().Ptr(), ctx, {});
10211021
if (newRead.Get() != input.Raw()) {
10221022
return newRead;
10231023
}

‎ydb/core/kqp/query_compiler/kqp_query_compiler.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1092,7 +1092,10 @@ class TKqpQueryCompiler : public IKqpQueryCompiler {
10921092
// We prepare a lot of partitions and distribute them between these tasks
10931093
// Constraint of 1 task per partition is NOT valid anymore
10941094
auto maxTasksPerStage = Config->MaxTasksPerStage.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerStage);
1095-
dqIntegration->Partition(NYql::TDqSettings(), maxTasksPerStage, source.Ref(), partitionParams, &clusterName, ctx, false);
1095+
IDqIntegration::TPartitionSettings pSettings;
1096+
pSettings.MaxPartitions = maxTasksPerStage;
1097+
pSettings.CanFallback = false;
1098+
dqIntegration->Partition(source.Ref(), partitionParams, &clusterName, ctx, pSettings);
10961099
externalSource.SetTaskParamKey(TString(dataSourceCategory));
10971100
for (const TString& partitionParam : partitionParams) {
10981101
externalSource.AddPartitionedTaskParams(partitionParam);

‎ydb/library/yql/dq/opt/dq_opt_log.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,7 @@ NNodes::TExprBase DqReplicateFieldSubset(NNodes::TExprBase node, TExprContext& c
336336
return node;
337337
}
338338

339-
IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& typesCtx, const TDqSettings& config) {
339+
IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& typesCtx, const IDqIntegration::TWrapReadSettings& wrSettings) {
340340
TOptimizeExprSettings settings{&typesCtx};
341341
auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) {
342342
if (auto maybeRead = TMaybeNode<TCoRight>(node).Input()) {
@@ -345,7 +345,7 @@ IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPt
345345
auto dataSource = typesCtx.DataSourceMap.FindPtr(dataSourceName);
346346
YQL_ENSURE(dataSource);
347347
if (auto dqIntegration = (*dataSource)->GetDqIntegration()) {
348-
auto newRead = dqIntegration->WrapRead(config, maybeRead.Cast().Ptr(), ctx);
348+
auto newRead = dqIntegration->WrapRead(maybeRead.Cast().Ptr(), ctx, wrSettings);
349349
if (newRead.Get() != maybeRead.Raw()) {
350350
return newRead;
351351
}

‎ydb/library/yql/dq/opt/dq_opt_log.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include <yql/essentials/core/dq_integration/yql_dq_integration.h>
34
#include <yql/essentials/ast/yql_expr.h>
45
#include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
56
#include <yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h>
@@ -11,7 +12,6 @@
1112
namespace NYql {
1213
class IOptimizationContext;
1314
struct TTypeAnnotationContext;
14-
struct TDqSettings;
1515
struct IProviderContext;
1616
struct TRelOptimizerNode;
1717
struct TOptimizerStatistics;
@@ -38,7 +38,7 @@ NNodes::TExprBase DqSqlInDropCompact(NNodes::TExprBase node, TExprContext& ctx);
3838

3939
NNodes::TExprBase DqReplicateFieldSubset(NNodes::TExprBase node, TExprContext& ctx, const TParentsMap& parents);
4040

41-
IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& typesCtx, const TDqSettings& config);
41+
IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& typesCtx, const IDqIntegration::TWrapReadSettings& wrSettings);
4242

4343
NNodes::TExprBase DqExpandMatchRecognize(NNodes::TExprBase node, TExprContext& ctx, TTypeAnnotationContext& typeAnnCtx);
4444

‎ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ class TClickHouseDqIntegration: public TDqIntegrationBase {
3333
return Nothing();
3434
}
3535

36-
TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr& read, TExprContext& ctx) override {
36+
TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& ) override {
3737
if (const auto maybeClReadTable = TMaybeNode<TClReadTable>(read)) {
3838
const auto clReadTable = maybeClReadTable.Cast();
3939
const auto token = TString("cluster:default_") += clReadTable.DataSource().Cluster().StringValue();
@@ -66,7 +66,7 @@ class TClickHouseDqIntegration: public TDqIntegrationBase {
6666
return read;
6767
}
6868

69-
ui64 Partition(const TDqSettings&, size_t, const TExprNode&, TVector<TString>& partitions, TString*, TExprContext&, bool) override {
69+
ui64 Partition(const TExprNode&, TVector<TString>& partitions, TString*, TExprContext&, const TPartitionSettings&) override {
7070
partitions.clear();
7171
NCH::TRange range;
7272
// range.SetRange("limit 42 offset 42 order by ...."); // Possible set range like this.

‎ydb/library/yql/providers/dq/helper/ya.make‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
LIBRARY()
2+
3+
PEERDIR(
4+
yql/essentials/core/dq_integration
5+
ydb/library/yql/dq/expr_nodes
6+
ydb/library/yql/dq/opt
7+
ydb/library/yql/dq/type_ann
8+
)
9+
10+
SRCS(
11+
yql_dq_helper_impl.cpp
12+
)
13+
14+
END()

0 commit comments

Comments
 (0)