Skip to content

Commit 2329e39

Browse files
kungakunga
authored
Merge 7fbfc96 into f5dc352 
2 parents f5dc352 + 7fbfc96 commit 2329e39

10 files changed

+246
-130
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,4 @@ util/all_thread.cpp
7474
util/all_util.cpp
7575
util/charset/all_charset.cpp
7676

77+
list_result.log

‎ydb/core/tablet_flat/benchmark/b_part.cpp

Lines changed: 86 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,19 @@
99
#include <ydb/core/tablet_flat/test/libs/table/test_make.h>
1010
#include <ydb/core/tablet_flat/test/libs/table/test_mixer.h>
1111
#include "ydb/core/tablet_flat/flat_part_btree_index_iter.h"
12+
#include "ydb/core/tablet_flat/flat_stat_table.h"
1213
#include "ydb/core/tablet_flat/test/libs/table/wrap_iter.h"
14+
#include "ydb/core/tx/datashard/datashard.h"
1315
#include <ydb/core/tablet_flat/test/libs/table/test_writer.h>
1416
#include <ydb/core/tablet_flat/test/libs/table/test_envs.h>
1517
#include <ydb/core/tablet_flat/test/libs/table/wrap_part.h>
1618
#include <ydb/core/tablet_flat/test/libs/table/test_steps.h>
1719

18-
namespace NKikimr {
19-
namespace NTable {
20+
#ifndef BENCHMARK_MAKE_LARGE_PART
21+
#define BENCHMARK_MAKE_LARGE_PART 0
22+
#endif
23+
24+
namespace NKikimr::NTable {
2025

2126
namespace {
2227
using namespace NTest;
@@ -26,81 +31,39 @@ namespace {
2631

2732
NPage::TConf PageConf(size_t groups, bool writeBTreeIndex) noexcept
2833
{
29-
NPage::TConf conf{ true, 1024 };
34+
NPage::TConf conf;
3035

3136
conf.Groups.resize(groups);
32-
for (size_t group : xrange(groups)) {
33-
conf.Group(group).PageSize = 1024;
34-
conf.Group(group).BTreeIndexNodeTargetSize = 1024;
35-
}
36-
37+
3738
conf.WriteBTreeIndex = writeBTreeIndex;
3839

3940
conf.SliceSize = conf.Group(0).PageSize * 4;
4041

4142
return conf;
4243
}
4344

44-
struct TPartEggsFixture : public benchmark::Fixture {
45-
using TGroupId = NPage::TGroupId;
46-
47-
void SetUp(const ::benchmark::State& state)
48-
{
49-
const bool groups = state.range(1);
50-
51-
TLayoutCook lay;
52-
53-
lay
54-
.Col(0, 0, NScheme::NTypeIds::Uint32)
55-
.Col(0, 1, NScheme::NTypeIds::Uint32)
56-
.Col(0, 2, NScheme::NTypeIds::Uint32)
57-
.Col(0, 3, NScheme::NTypeIds::Uint32)
58-
.Col(groups ? 1 : 0, 4, NScheme::NTypeIds::Uint32)
59-
.Key({0, 1, 2});
60-
61-
TPartCook cook(lay, PageConf(groups ? 2 : 1, true));
62-
63-
for (ui32 i = 0; (groups ? cook.GetDataBytes(0) + cook.GetDataBytes(1) : cook.GetDataBytes(0)) < 100ull*1024*1024; i++) {
64-
cook.Add(*TSchemedCookRow(*lay).Col(i / 10000, i / 100 % 100, i % 100, i, i));
65-
}
66-
67-
Eggs = cook.Finish();
68-
69-
const auto part = Eggs.Lone();
70-
71-
Cerr << "DataBytes = " << part->Stat.Bytes << " DataPages = " << IndexTools::CountMainPages(*part) << Endl;
72-
Cerr << "FlatIndexBytes = " << part->GetPageSize(part->IndexPages.Groups[groups ? 1 : 0], {}) << " BTreeIndexBytes = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].IndexSize << Endl;
73-
Cerr << "Levels = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].LevelCount << Endl;
74-
75-
// 100 MB
76-
UNIT_ASSERT_GE(part->Stat.Bytes, 100ull*1024*1024);
77-
UNIT_ASSERT_LE(part->Stat.Bytes, 100ull*1024*1024 + 10ull*1024*1024);
78-
79-
GroupId = TGroupId(groups ? 1 : 0);
80-
}
81-
82-
TPartEggs Eggs;
83-
TTestEnv Env;
84-
TGroupId GroupId;
85-
};
86-
87-
struct TPartSubsetFixture : public benchmark::Fixture {
45+
struct TPartFixture : public benchmark::Fixture {
8846
using TGroupId = NPage::TGroupId;
8947

90-
void SetUp(const ::benchmark::State& state)
48+
void SetUp(::benchmark::State& state)
9149
{
9250
const bool useBTree = state.range(0);
9351
const bool groups = state.range(1);
9452
const bool history = state.range(2);
9553

96-
Mass = new NTest::TMass(new NTest::TModelStd(groups), history ? 1000000 : 300000);
54+
ui64 rows = history ? 300000 : 1000000;
55+
if (BENCHMARK_MAKE_LARGE_PART) {
56+
rows *= 10;
57+
}
58+
Mass = new NTest::TMass(new NTest::TModelStd(groups), rows);
9759
Subset = TMake(*Mass, PageConf(Mass->Model->Scheme->Families.size(), useBTree)).Mixed(0, 1, TMixerOne{ }, history ? 0.7 : 0);
9860

99-
for (const auto& part : Subset->Flatten) {
100-
Cerr << "DataBytes = " << part->Stat.Bytes << " DataPages = " << IndexTools::CountMainPages(*part) << Endl;
101-
Cerr << "FlatIndexBytes = " << part->GetPageSize(part->IndexPages.Groups[groups ? 1 : 0], {}) << " BTreeIndexBytes = " << (useBTree ? part->IndexPages.BTreeGroups[groups ? 1 : 0].IndexSize : 0) << Endl;
61+
for (const auto& part : Subset->Flatten) { // single part
62+
state.counters["DataBytes"] = part->Stat.Bytes;
63+
state.counters["DataPages"] = IndexTools::CountMainPages(*part);
64+
state.counters["IndexBytes"] = part->IndexesRawSize;
10265
if (useBTree) {
103-
Cerr << "Levels = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].LevelCount << Endl;
66+
state.counters["Levels{0}"] = part->IndexPages.BTreeGroups[0].LevelCount;
10467
}
10568
}
10669

@@ -111,6 +74,9 @@ namespace {
11174
Checker = new TCheckIt(*Subset, {new TTestEnv()});
11275
CheckerReverse = new TCheckReverseIt(*Subset, {new TTestEnv()});
11376
}
77+
78+
GroupId = TGroupId(groups, history);
79+
Part = Subset->Flatten[0].Part.Get();
11480
}
11581

11682
TMersenne<ui64> Rnd;
@@ -119,87 +85,99 @@ namespace {
11985
TAutoPtr<TCheckIt> Checker;
12086
TAutoPtr<TCheckReverseIt> CheckerReverse;
12187
TTestEnv Env;
88+
TGroupId GroupId;
89+
TPart const* Part;
12290
};
12391
}
12492

125-
BENCHMARK_DEFINE_F(TPartEggsFixture, SeekRowId)(benchmark::State& state) {
93+
BENCHMARK_DEFINE_F(TPartFixture, SeekRowId)(benchmark::State& state) {
12694
const bool useBTree = state.range(0);
12795

12896
for (auto _ : state) {
12997
THolder<IIndexIter> iter;
13098

13199
if (useBTree) {
132-
iter = MakeHolder<TPartBtreeIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
100+
iter = MakeHolder<TPartBtreeIndexIt>(Part, &Env, GroupId);
133101
} else {
134-
iter = MakeHolder<TPartIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
102+
iter = MakeHolder<TPartIndexIt>(Part, &Env, GroupId);
135103
}
136104

137-
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
105+
iter->Seek(RandomNumber<ui32>(Part->Stat.Rows));
138106
}
139107
}
140108

141-
BENCHMARK_DEFINE_F(TPartEggsFixture, Next)(benchmark::State& state) {
109+
BENCHMARK_DEFINE_F(TPartFixture, Next)(benchmark::State& state) {
142110
const bool useBTree = state.range(0);
143111

144112
THolder<IIndexIter> iter;
145113

146114
if (useBTree) {
147-
iter = MakeHolder<TPartBtreeIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
115+
iter = MakeHolder<TPartBtreeIndexIt>(Part, &Env, GroupId);
148116
} else {
149-
iter = MakeHolder<TPartIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
117+
iter = MakeHolder<TPartIndexIt>(Part, &Env, GroupId);
150118
}
151119

152-
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
120+
iter->Seek(RandomNumber<ui32>(Part->Stat.Rows));
153121

154122
for (auto _ : state) {
155123
if (!iter->IsValid()) {
156-
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
124+
iter->Seek(RandomNumber<ui32>(Part->Stat.Rows));
157125
}
158126
iter->Next();
159127
}
160128
}
161129

162-
BENCHMARK_DEFINE_F(TPartEggsFixture, Prev)(benchmark::State& state) {
130+
BENCHMARK_DEFINE_F(TPartFixture, Prev)(benchmark::State& state) {
163131
const bool useBTree = state.range(0);
164132

165133
THolder<IIndexIter> iter;
166134

167135
if (useBTree) {
168-
iter = MakeHolder<TPartBtreeIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
136+
iter = MakeHolder<TPartBtreeIndexIt>(Part, &Env, GroupId);
169137
} else {
170-
iter = MakeHolder<TPartIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
138+
iter = MakeHolder<TPartIndexIt>(Part, &Env, GroupId);
171139
}
172140

173-
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
141+
iter->Seek(RandomNumber<ui32>(Part->Stat.Rows));
174142

175143
for (auto _ : state) {
176144
if (!iter->IsValid()) {
177-
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
145+
iter->Seek(RandomNumber<ui32>(Part->Stat.Rows));
178146
}
179147
iter->Prev();
180148
}
181149
}
182150

183-
BENCHMARK_DEFINE_F(TPartEggsFixture, SeekKey)(benchmark::State& state) {
151+
BENCHMARK_DEFINE_F(TPartFixture, SeekKey)(benchmark::State& state) {
184152
const bool useBTree = state.range(0);
185-
const ESeek seek = ESeek(state.range(2));
153+
const ESeek seek = ESeek(state.range(3));
154+
155+
TRowTool rowTool(*Subset->Scheme);
156+
auto tags = TVector<TTag>();
157+
for (auto c : Subset->Scheme->Cols) {
158+
tags.push_back(c.Tag);
159+
}
186160

187161
for (auto _ : state) {
188162
THolder<IIndexIter> iter;
189163

190164
if (useBTree) {
191-
iter = MakeHolder<TPartBtreeIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
165+
iter = MakeHolder<TPartBtreeIndexIt>(Part, &Env, GroupId);
192166
} else {
193-
iter = MakeHolder<TPartIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
167+
iter = MakeHolder<TPartIndexIt>(Part, &Env, GroupId);
194168
}
195169

196-
ui32 rowId = RandomNumber<ui32>(Eggs.Lone()->Stat.Rows);
197-
TVector<TCell> key{TCell::Make(rowId / 10000), TCell::Make(rowId / 100 % 100), TCell::Make(rowId % 100)};
198-
iter->Seek(seek, key, Eggs.Scheme->Keys.Get());
170+
state.PauseTiming();
171+
auto& row = *Mass->Saved.Any(Rnd);
172+
auto key_ = rowTool.LookupKey(row);
173+
const TCelled key(key_, *Subset->Scheme->Keys, false);
174+
state.ResumeTiming();
175+
176+
iter->Seek(seek, key, Subset->Scheme->Keys.Get());
199177
}
200178
}
201179

202-
BENCHMARK_DEFINE_F(TPartSubsetFixture, DoReads)(benchmark::State& state) {
180+
BENCHMARK_DEFINE_F(TPartFixture, DoReads)(benchmark::State& state) {
203181
const bool reverse = state.range(3);
204182
const ESeek seek = static_cast<ESeek>(state.range(4));
205183
const ui32 items = state.range(5);
@@ -221,7 +199,7 @@ BENCHMARK_DEFINE_F(TPartSubsetFixture, DoReads)(benchmark::State& state) {
221199
}
222200
}
223201

224-
BENCHMARK_DEFINE_F(TPartSubsetFixture, DoCharge)(benchmark::State& state) {
202+
BENCHMARK_DEFINE_F(TPartFixture, DoCharge)(benchmark::State& state) {
225203
const bool reverse = state.range(3);
226204
const ui32 items = state.range(4);
227205

@@ -245,32 +223,43 @@ BENCHMARK_DEFINE_F(TPartSubsetFixture, DoCharge)(benchmark::State& state) {
245223
}
246224
}
247225

248-
BENCHMARK_REGISTER_F(TPartEggsFixture, SeekRowId)
226+
BENCHMARK_DEFINE_F(TPartFixture, BuildStats)(benchmark::State& state) {
227+
for (auto _ : state) {
228+
TStats stats;
229+
BuildStats(*Subset, stats, NDataShard::gDbStatsRowCountResolution, NDataShard::gDbStatsDataSizeResolution, &Env);
230+
}
231+
}
232+
233+
BENCHMARK_REGISTER_F(TPartFixture, SeekRowId)
249234
->ArgsProduct({
250235
/* b-tree */ {0, 1},
251-
/* groups: */ {0, 1}})
236+
/* groups: */ {0, 1},
237+
/* history: */ {0}})
252238
->Unit(benchmark::kMicrosecond);
253239

254-
BENCHMARK_REGISTER_F(TPartEggsFixture, Next)
240+
BENCHMARK_REGISTER_F(TPartFixture, Next)
255241
->ArgsProduct({
256242
/* b-tree */ {0, 1},
257-
/* groups: */ {0, 1}})
243+
/* groups: */ {0, 1},
244+
/* history: */ {0}})
258245
->Unit(benchmark::kMicrosecond);
259246

260-
BENCHMARK_REGISTER_F(TPartEggsFixture, Prev)
247+
BENCHMARK_REGISTER_F(TPartFixture, Prev)
261248
->ArgsProduct({
262249
/* b-tree */ {0, 1},
263-
/* groups: */ {0, 1}})
250+
/* groups: */ {0, 1},
251+
/* history: */ {0}})
264252
->Unit(benchmark::kMicrosecond);
265253

266-
BENCHMARK_REGISTER_F(TPartEggsFixture, SeekKey)
254+
BENCHMARK_REGISTER_F(TPartFixture, SeekKey)
267255
->ArgsProduct({
268256
/* b-tree */ {0, 1},
269257
/* groups: */ {0, 1},
258+
/* history: */ {0},
270259
/* ESeek: */ {1}})
271260
->Unit(benchmark::kMicrosecond);
272261

273-
BENCHMARK_REGISTER_F(TPartSubsetFixture, DoReads)
262+
BENCHMARK_REGISTER_F(TPartFixture, DoReads)
274263
->ArgsProduct({
275264
/* b-tree */ {0, 1},
276265
/* groups: */ {1},
@@ -280,7 +269,7 @@ BENCHMARK_REGISTER_F(TPartSubsetFixture, DoReads)
280269
/* items */ {1, 50, 1000}})
281270
->Unit(benchmark::kMicrosecond);
282271

283-
BENCHMARK_REGISTER_F(TPartSubsetFixture, DoCharge)
272+
BENCHMARK_REGISTER_F(TPartFixture, DoCharge)
284273
->ArgsProduct({
285274
/* b-tree */ {0, 1},
286275
/* groups: */ {1},
@@ -289,5 +278,11 @@ BENCHMARK_REGISTER_F(TPartSubsetFixture, DoCharge)
289278
/* items */ {1, 50, 1000}})
290279
->Unit(benchmark::kMicrosecond);
291280

292-
}
281+
BENCHMARK_REGISTER_F(TPartFixture, BuildStats)
282+
->ArgsProduct({
283+
/* b-tree */ {0, 1},
284+
/* groups: */ {0, 1},
285+
/* history: */ {0, 1}})
286+
->Unit(benchmark::kMicrosecond);
287+
293288
}

‎ydb/core/tablet_flat/benchmark/ya.make

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,14 @@ G_BENCHMARK()
22

33
TAG(ya:fat)
44
SIZE(LARGE)
5-
TIMEOUT(1200)
5+
TIMEOUT(600)
6+
7+
IF (BENCHMARK_MAKE_LARGE_PART)
8+
CFLAGS(
9+
-DBENCHMARK_MAKE_LARGE_PART=1
10+
)
11+
TIMEOUT(1200)
12+
ENDIF()
613

714
SRCS(
815
b_part.cpp

0 commit comments

Comments
 (0)