Skip to content

Commit 168f726

Browse files
doroolegdorooleg
authored
Merge 34b54e7 into c07ddf8 
2 parents c07ddf8 + 34b54e7 commit 168f726

File tree

12 files changed

+14
-49
lines changed

12 files changed

+14
-49
lines changed

‎ydb/core/fq/libs/actors/run_actor.cpp‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -779,11 +779,7 @@ class TRunActor : public NActors::TActorBootstrapped<TRunActor> {
779779
mkqlDefaultLimit = 8_GB;
780780
}
781781

782-
// This part is for backward compatibility. TODO: remove this part after migration to TS3GatewayConfig
783-
auto s3ReadDefaultInflightLimit = Params.Config.GetReadActorsFactoryConfig().GetS3ReadActorFactoryConfig().GetDataInflight();
784-
if (s3ReadDefaultInflightLimit == 0) {
785-
s3ReadDefaultInflightLimit = Params.Config.GetGateways().GetS3().GetDataInflight();
786-
}
782+
auto s3ReadDefaultInflightLimit = Params.Config.GetGateways().GetS3().GetDataInflight();
787783
if (s3ReadDefaultInflightLimit == 0) {
788784
s3ReadDefaultInflightLimit = 200_MB;
789785
}
@@ -1970,8 +1966,7 @@ class TRunActor : public NActors::TActorBootstrapped<TRunActor> {
19701966
}
19711967

19721968
{
1973-
dataProvidersInit.push_back(GetS3DataProviderInitializer(Params.S3Gateway, Params.CredentialsFactory,
1974-
Params.Config.GetReadActorsFactoryConfig().HasS3ReadActorFactoryConfig() ? Params.Config.GetReadActorsFactoryConfig().GetS3ReadActorFactoryConfig().GetAllowLocalFiles() : Params.Config.GetGateways().GetS3().GetAllowLocalFiles(), NActors::TActivationContext::ActorSystem())); // This part is for backward compatibility. TODO: remove this part after migration to TS3GatewayConfig
1969+
dataProvidersInit.push_back(GetS3DataProviderInitializer(Params.S3Gateway, Params.CredentialsFactory, NActors::TActivationContext::ActorSystem()));
19751970
}
19761971

19771972
{

‎ydb/core/fq/libs/config/protos/read_actors_factory.proto‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,11 @@ import "ydb/library/yql/providers/s3/proto/retry_config.proto";
88

99
////////////////////////////////////////////////////////////
1010

11-
message TS3ReadActorFactoryConfig {
12-
NYql.NS3.TRetryConfig RetryConfig = 1;
13-
uint64 RowsInBatch = 2; // Default = 1000
14-
uint64 MaxInflight = 3; // Default = 20
15-
uint64 DataInflight = 4; // Default = 200 MB
16-
bool AllowLocalFiles = 5;
17-
}
18-
1911
message TPqReadActorFactoryConfig {
2012
bool CookieCommitMode = 1; // Turn off RangesMode setting in PQ read session.
2113
}
2214

2315
message TReadActorsFactoryConfig {
24-
TS3ReadActorFactoryConfig S3ReadActorFactoryConfig = 1;
16+
reserved 1;
2517
TPqReadActorFactoryConfig PqReadActorFactoryConfig = 2;
2618
}

‎ydb/core/fq/libs/init/init.cpp‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -196,21 +196,9 @@ void Init(
196196
auto s3ActorsFactory = NYql::NDq::CreateS3ActorsFactory();
197197

198198
if (protoConfig.GetPrivateApi().GetEnabled()) {
199-
const auto& s3readConfig = protoConfig.GetReadActorsFactoryConfig().GetS3ReadActorFactoryConfig();
200199
auto s3HttpRetryPolicy = NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.MaxTime = TDuration::Max(), .RetriedCurlCodes = NYql::FqRetriedCurlCodes()});
201200
NYql::NDq::TS3ReadActorFactoryConfig readActorFactoryCfg = NYql::NDq::CreateReadActorFactoryConfig(protoConfig.GetGateways().GetS3());
202201

203-
// These fillings were left for the backward compatibility. TODO: remove this part after migration to TS3GatewayConfig
204-
if (const ui64 rowsInBatch = s3readConfig.GetRowsInBatch()) {
205-
readActorFactoryCfg.RowsInBatch = rowsInBatch;
206-
}
207-
if (const ui64 maxInflight = s3readConfig.GetMaxInflight()) {
208-
readActorFactoryCfg.MaxInflight = maxInflight;
209-
}
210-
if (const ui64 dataInflight = s3readConfig.GetDataInflight()) {
211-
readActorFactoryCfg.DataInflight = dataInflight;
212-
}
213-
214202
RegisterDqInputTransformLookupActorFactory(*asyncIoFactory);
215203
RegisterDqPqReadActorFactory(*asyncIoFactory, yqSharedResources->UserSpaceYdbDriver, credentialsFactory, yqCounters->GetSubgroup("subsystem", "DqSourceTracker"));
216204
RegisterYdbReadActorFactory(*asyncIoFactory, yqSharedResources->UserSpaceYdbDriver, credentialsFactory);

‎ydb/library/yql/providers/s3/provider/yql_s3_provider.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44

55
namespace NYql {
66

7-
TDataProviderInitializer GetS3DataProviderInitializer(IHTTPGateway::TPtr gateway, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, bool allowLocalFiles, NActors::TActorSystem* actorSystem) {
8-
return [gateway, credentialsFactory, allowLocalFiles, actorSystem] (
7+
TDataProviderInitializer GetS3DataProviderInitializer(IHTTPGateway::TPtr gateway, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, NActors::TActorSystem* actorSystem) {
8+
return [gateway, credentialsFactory, actorSystem] (
99
const TString& userName,
1010
const TString& sessionId,
1111
const TGatewaysConfig* gatewaysConfig,
@@ -35,7 +35,6 @@ TDataProviderInitializer GetS3DataProviderInitializer(IHTTPGateway::TPtr gateway
3535
if (gatewaysConfig) {
3636
state->Configuration->Init(gatewaysConfig->GetS3(), typeCtx);
3737
}
38-
state->Configuration->AllowLocalFiles = allowLocalFiles;
3938
state->Gateway = gateway;
4039

4140
TDataProviderInfo info;

‎ydb/library/yql/providers/s3/provider/yql_s3_provider.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ struct TS3State : public TThrRefBase
3535
NActors::TActorSystem* ActorSystem = nullptr;
3636
};
3737

38-
TDataProviderInitializer GetS3DataProviderInitializer(IHTTPGateway::TPtr gateway, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory = nullptr, bool allowLocalFiles = false, NActors::TActorSystem* actorSystem = nullptr);
38+
TDataProviderInitializer GetS3DataProviderInitializer(IHTTPGateway::TPtr gateway, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory = nullptr, NActors::TActorSystem* actorSystem = nullptr);
3939

4040
TIntrusivePtr<IDataProvider> CreateS3DataSource(TS3State::TPtr state);
4141
TIntrusivePtr<IDataProvider> CreateS3DataSink(TS3State::TPtr state);

‎ydb/library/yql/tools/dqrun/dqrun.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -949,13 +949,14 @@ int RunMain(int argc, const char* argv[])
949949
}
950950

951951
if (gatewaysConfig.HasS3()) {
952+
gatewaysConfig.MutableS3()->SetAllowLocalFiles(true);
952953
for (auto& cluster: gatewaysConfig.GetS3().GetClusterMapping()) {
953954
clusters.emplace(to_lower(cluster.GetName()), TString{S3ProviderName});
954955
}
955956
if (!httpGateway) {
956957
httpGateway = IHTTPGateway::Make(gatewaysConfig.HasHttpGateway() ? &gatewaysConfig.GetHttpGateway() : nullptr);
957958
}
958-
dataProvidersInit.push_back(GetS3DataProviderInitializer(httpGateway, nullptr, true, nullptr));
959+
dataProvidersInit.push_back(GetS3DataProviderInitializer(httpGateway, nullptr));
959960
}
960961

961962
if (gatewaysConfig.HasPq()) {

‎ydb/tests/fq/common/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818
@pytest.fixture
1919
def kikimr(request: pytest.FixtureRequest, yq_version: str):
2020
kikimr_extensions = [
21+
AddFormatSizeLimitExtension(),
2122
AddInflightExtension(),
2223
AddDataInflightExtension(),
23-
AddFormatSizeLimitExtension(),
2424
DefaultConfigExtension(''),
2525
YQv2Extension(yq_version),
2626
ComputeExtension(),

‎ydb/tests/fq/plans/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,9 @@ def is_s3_ready():
6161
@pytest.fixture
6262
def kikimr(request: pytest.FixtureRequest, s3: S3, yq_version: str, stats_mode: str):
6363
kikimr_extensions = [
64+
AddFormatSizeLimitExtension(),
6465
AddInflightExtension(),
6566
AddDataInflightExtension(),
66-
AddFormatSizeLimitExtension(),
6767
DefaultConfigExtension(s3.s3_url),
6868
YQv2Extension(yq_version),
6969
ComputeExtension(),

‎ydb/tests/fq/restarts/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,9 @@ def stats_mode():
6666
@pytest.fixture
6767
def kikimr(request: pytest.FixtureRequest, s3: S3, yq_version: str, stats_mode: str):
6868
kikimr_extensions = [
69+
AddFormatSizeLimitExtension(),
6970
AddInflightExtension(),
7071
AddDataInflightExtension(),
71-
AddFormatSizeLimitExtension(),
7272
DefaultConfigExtension(s3.s3_url),
7373
YQv2Extension(yq_version),
7474
ComputeExtension(),

‎ydb/tests/fq/s3/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,9 @@ def kikimr_params(request: pytest.FixtureRequest):
8787

8888
def get_kikimr_extensions(s3: S3, yq_version: str, kikimr_settings, mvp_external_ydb_endpoint):
8989
return [
90+
AddFormatSizeLimitExtension(),
9091
AddInflightExtension(),
9192
AddDataInflightExtension(),
92-
AddFormatSizeLimitExtension(),
9393
DefaultConfigExtension(s3.s3_url),
9494
YQv2Extension(yq_version, kikimr_settings.get("is_replace_if_exists", False)),
9595
ComputeExtension(),

0 commit comments

Comments
 (0)