Skip to content

Commit 998e11e

Browse files
evanevanevanevannnnevanevanevanevannnn
authored
Merge 00816fc into 52d9c30 
2 parents 52d9c30 + 00816fc commit 998e11e

File tree

7 files changed

+14
-96
lines changed

7 files changed

+14
-96
lines changed

‎ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2072,7 +2072,7 @@ std::pair<NYql::NDq::IDqComputeActorAsyncInput*, IActor*> CreateS3ReadActor(
20722072
const IFunctionRegistry& functionRegistry = *holderFactory.GetFunctionRegistry();
20732073

20742074
TPathList paths;
2075-
ReadPathsList(params, taskParams, readRanges, paths);
2075+
ReadPathsList(taskParams, readRanges, paths);
20762076

20772077
const auto token = secureParams.Value(params.GetToken(), TString{});
20782078
const TS3Credentials credentials(credentialsFactory, token);

‎ydb/library/yql/providers/s3/proto/source.proto‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ message TPath {
1313
message TSource {
1414
string Url = 1;
1515
string Token = 2;
16-
repeated TPath DeprecatedPath = 3; // deprecated
16+
reserved 3;
1717
optional string RowType = 4;
1818
optional string Format = 5;
1919
map<string, string> Settings = 6;

‎ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -479,7 +479,7 @@ class TS3DqIntegration: public TDqIntegrationBase {
479479
range.Save(&out);
480480

481481
paths.clear();
482-
ReadPathsList(srcDesc, {}, serialized, paths);
482+
ReadPathsList({}, serialized, paths);
483483

484484
const NDq::TS3ReadActorFactoryConfig& readActorConfig = State_->Configuration->S3ReadActorFactoryConfig;
485485
ui64 fileSizeLimit = readActorConfig.FileSizeLimit;

‎ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
107107
b.Save(&range);
108108

109109
TPathList paths;
110-
ReadPathsList({}, MakeParams(range), {}, paths);
110+
ReadPathsList(MakeParams(range), {}, paths);
111111

112112
UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1);
113113
UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "name");
@@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
124124
b.Save(&range);
125125

126126
TPathList paths;
127-
ReadPathsList({}, MakeParams(range), {}, paths);
127+
ReadPathsList(MakeParams(range), {}, paths);
128128

129129
UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1);
130130
UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "a///b");
@@ -142,7 +142,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
142142
b.Save(&range);
143143

144144
TPathList paths;
145-
ReadPathsList({}, MakeParams(range), {}, paths);
145+
ReadPathsList(MakeParams(range), {}, paths);
146146

147147
UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2);
148148
UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "root/name/");
@@ -165,7 +165,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
165165
b.Save(&range);
166166

167167
TPathList paths;
168-
ReadPathsList({}, MakeParams(range), {}, paths);
168+
ReadPathsList(MakeParams(range), {}, paths);
169169

170170
UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2);
171171

@@ -190,7 +190,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
190190
b.Save(&range);
191191

192192
TPathList paths;
193-
ReadPathsList({}, MakeParams(range), {}, paths);
193+
ReadPathsList(MakeParams(range), {}, paths);
194194

195195
UNIT_ASSERT_VALUES_EQUAL(paths.size(), 3);
196196

‎ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ static void BuildPathsFromTree(const google::protobuf::RepeatedPtrField<NYql::NS
3838
BuildPathsFromTree(children, paths, currentPath, currentDepth, nextPathIndex);
3939
}
4040

41-
void DecodeS3Range(const NS3::TSource& sourceDesc, const TString& data, TPathList& paths) {
41+
void DecodeS3Range(const TString& data, TPathList& paths) {
4242
NS3::TRange range;
4343
TStringInput input(data);
4444
range.Load(&input);
@@ -49,35 +49,15 @@ void DecodeS3Range(const NS3::TSource& sourceDesc, const TString& data, TPathLis
4949
TString buf;
5050
return BuildPathsFromTree(range.GetPaths(), paths, buf, 0, startPathIndex);
5151
}
52-
53-
std::unordered_map<TString, size_t> map(sourceDesc.GetDeprecatedPath().size());
54-
for (auto i = 0; i < sourceDesc.GetDeprecatedPath().size(); ++i) {
55-
map.emplace(sourceDesc.GetDeprecatedPath().Get(i).GetPath(), sourceDesc.GetDeprecatedPath().Get(i).GetSize());
56-
}
57-
58-
for (auto i = 0; i < range.GetDeprecatedPath().size(); ++i) {
59-
const auto& path = range.GetDeprecatedPath().Get(i);
60-
auto it = map.find(path);
61-
YQL_ENSURE(it != map.end());
62-
paths.emplace_back(TPath{path, it->second, false, i + startPathIndex});
63-
}
6452
}
6553

66-
void ReadPathsList(const NS3::TSource& sourceDesc, const THashMap<TString, TString>& taskParams, const TVector<TString>& readRanges, TPathList& paths) {
54+
void ReadPathsList(const THashMap<TString, TString>& taskParams, const TVector<TString>& readRanges, TPathList& paths) {
6755
if (!readRanges.empty()) {
6856
for (auto readRange : readRanges) {
69-
DecodeS3Range(sourceDesc, readRange, paths);
57+
DecodeS3Range(readRange, paths);
7058
}
7159
} else if (const auto taskParamsIt = taskParams.find(S3ProviderName); taskParamsIt != taskParams.cend()) {
72-
DecodeS3Range(sourceDesc, taskParamsIt->second, paths);
73-
} else {
74-
for (auto i = 0; i < sourceDesc.GetDeprecatedPath().size(); ++i) {
75-
paths.emplace_back(TPath{
76-
sourceDesc.GetDeprecatedPath().Get(i).GetPath(),
77-
sourceDesc.GetDeprecatedPath().Get(i).GetSize(),
78-
false,
79-
static_cast<ui64>(i)});
80-
}
60+
DecodeS3Range(taskParamsIt->second, paths);
8161
}
8262
}
8363

‎ydb/library/yql/providers/s3/range_helpers/path_list_reader.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ struct TPath {
2323
};
2424
using TPathList = std::vector<TPath>;
2525

26-
void ReadPathsList(const NS3::TSource& sourceDesc, const THashMap<TString, TString>& taskParams, const TVector<TString>& readRanges, TPathList& paths);
26+
void ReadPathsList(const THashMap<TString, TString>& taskParams, const TVector<TString>& readRanges, TPathList& paths);
2727

2828
void PackPathsList(const TPathList& paths, TString& packed, bool& isTextEncoded);
2929
void UnpackPathsList(TStringBuf packed, bool isTextEncoded, TPathList& paths);

‎ydb/library/yql/providers/s3/range_helpers/path_list_reader_ut.cpp‎

Lines changed: 1 addition & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -18,63 +18,6 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) {
1818
return map;
1919
}
2020

21-
Y_UNIT_TEST(ReadsFilesListFromSourceSettings) {
22-
NS3::TSource src;
23-
{
24-
auto* p = src.AddDeprecatedPath();
25-
p->SetPath("my/path");
26-
p->SetSize(100500);
27-
}
28-
{
29-
auto* p = src.AddDeprecatedPath();
30-
p->SetPath("other/path");
31-
p->SetSize(1);
32-
}
33-
34-
TPathList paths;
35-
ReadPathsList(src, {}, {}, paths);
36-
37-
UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2);
38-
39-
UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "my/path");
40-
UNIT_ASSERT_VALUES_EQUAL(paths[0].Size, 100500);
41-
UNIT_ASSERT_VALUES_EQUAL(paths[0].IsDirectory, false);
42-
UNIT_ASSERT_VALUES_EQUAL(paths[0].PathIndex, 0);
43-
44-
UNIT_ASSERT_VALUES_EQUAL(paths[1].Path, "other/path");
45-
UNIT_ASSERT_VALUES_EQUAL(paths[1].Size, 1);
46-
UNIT_ASSERT_VALUES_EQUAL(paths[1].IsDirectory, false);
47-
UNIT_ASSERT_VALUES_EQUAL(paths[1].PathIndex, 1);
48-
}
49-
50-
Y_UNIT_TEST(ReadsFilesListFromParamsAndSourceSettings) {
51-
NS3::TSource src;
52-
{
53-
auto* p = src.AddDeprecatedPath();
54-
p->SetPath("my/path");
55-
p->SetSize(100500);
56-
}
57-
{
58-
auto* p = src.AddDeprecatedPath();
59-
p->SetPath("other/path");
60-
p->SetSize(1);
61-
}
62-
63-
NS3::TRange range;
64-
range.SetStartPathIndex(42);
65-
range.AddDeprecatedPath("my/path");
66-
67-
TPathList paths;
68-
ReadPathsList(src, MakeParams(range), {}, paths);
69-
70-
UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1);
71-
72-
UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "my/path");
73-
UNIT_ASSERT_VALUES_EQUAL(paths[0].Size, 100500);
74-
UNIT_ASSERT_VALUES_EQUAL(paths[0].IsDirectory, false);
75-
UNIT_ASSERT_VALUES_EQUAL(paths[0].PathIndex, 42);
76-
}
77-
7821
NYql::NS3::TRange::TPath* SetPath(NYql::NS3::TRange::TPath* path, const TString& name = {}, ui64 size = 0, bool read = false) {
7922
path->SetName(name);
8023
path->SetSize(size);
@@ -84,11 +27,6 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) {
8427

8528
Y_UNIT_TEST(ReadsFilesListFromTreeParams) {
8629
NS3::TSource src;
87-
{
88-
auto* p = src.AddDeprecatedPath();
89-
p->SetPath("my/path");
90-
p->SetSize(100500);
91-
}
9230

9331
NS3::TRange range;
9432
range.SetStartPathIndex(42);
@@ -110,7 +48,7 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) {
11048
}
11149

11250
TPathList paths;
113-
ReadPathsList(src, MakeParams(range), {}, paths);
51+
ReadPathsList(MakeParams(range), {}, paths);
11452

11553
UNIT_ASSERT_VALUES_EQUAL(paths.size(), 5);
11654

0 commit comments

Comments
 (0)