Skip to content

Commit 3449c4f

Browse files
StekPerepolnenStekPerepolnen
committed
added pdisk filter
1 parent e029ee3 commit 3449c4f

File tree

3 files changed

+115
-50
lines changed

3 files changed

+115
-50
lines changed

‎ydb/core/viewer/json_storage.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,6 @@ class TJsonStorage : public TJsonStorageBase {
125125
}
126126

127127
void Bootstrap() override {
128-
Cerr << "iiiii Bootstrap" << Endl;
129128
TIntrusivePtr<TDomainsInfo> domains = AppData()->DomainsInfo;
130129
ui64 hiveId = domains->GetHive();
131130
if (hiveId != TDomainsInfo::BadTabletId) {
@@ -251,7 +250,7 @@ class TJsonStorage : public TJsonStorageBase {
251250
}
252251

253252
bool CheckGroupFilters(const TString& groupId, const TString& poolName, const TGroupRow& groupRow) {
254-
if (!EffectiveFilterGroupIds.empty() && !EffectiveFilterGroupIds.contains(groupId)) {
253+
if (!EffectiveGroupFilter.contains(groupId)) {
255254
return false;
256255
}
257256
switch (With) {
@@ -498,6 +497,7 @@ struct TJsonRequestParameters<TJsonStorage> {
498497
{"name":"tenant","in":"query","description":"tenant name","required":false,"type":"string"},
499498
{"name":"pool","in":"query","description":"storage pool name","required":false,"type":"string"},
500499
{"name":"node_id","in":"query","description":"node id","required":false,"type":"integer"},
500+
{"name":"pdisk_id","in":"query","description":"pdisk id","required":false,"type":"integer"},
501501
{"name":"group_id","in":"query","description":"group id","required":false,"type":"integer"},
502502
{"name":"need_groups","in":"query","description":"return groups information","required":false,"type":"boolean","default":true},
503503
{"name":"need_disks","in":"query","description":"return disks information","required":false,"type":"boolean","default":true},

‎ydb/core/viewer/json_storage_base.h‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,11 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
7272
ui32 Timeout = 0;
7373
TString FilterTenant;
7474
THashSet<TString> FilterStoragePools;
75-
TVector<TString> FilterGroupIds;
7675
TString Filter;
76+
std::unordered_set<TString> FilterGroupIds;
7777
std::unordered_set<TNodeId> FilterNodeIds;
78-
THashSet<TString> EffectiveFilterGroupIds;
78+
std::unordered_set<TNodeId> FilterPDiskIds;
79+
THashSet<TString> EffectiveGroupFilter;
7980
std::unordered_set<TNodeId> NodeIds;
8081
bool NeedAdditionalNodesRequests;
8182

@@ -133,9 +134,9 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
133134
FilterStoragePools.emplace(filterStoragePool);
134135
}
135136
SplitIds(params.Get("node_id"), ',', FilterNodeIds);
137+
SplitIds(params.Get("pdisk_id"), ',', FilterPDiskIds);
136138
NeedAdditionalNodesRequests = !FilterNodeIds.empty();
137139
SplitIds(params.Get("group_id"), ',', FilterGroupIds);
138-
Sort(FilterGroupIds);
139140
Filter = params.Get("filter");
140141
if (params.Get("with") == "missing") {
141142
With = EWith::MissingDisks;
@@ -263,7 +264,6 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
263264
}
264265

265266
void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) {
266-
Cerr << "iiiiiiiiii TEvNodesInfo 1" << Endl;
267267
ui32 maxAllowedNodeId = std::numeric_limits<ui32>::max();
268268
TIntrusivePtr<TDynamicNameserviceConfig> dynamicNameserviceConfig = AppData()->DynamicNameserviceConfig;
269269
if (dynamicNameserviceConfig) {
@@ -275,7 +275,6 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
275275
SendNodeRequests(ni.NodeId);
276276
}
277277
}
278-
Cerr << "iiiiiiiiii TEvNodesInfo 2" << Endl;
279278
RequestDone();
280279
}
281280

@@ -314,7 +313,6 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
314313
}
315314

316315
void Handle(TEvHive::TEvResponseHiveStorageStats::TPtr& ev) {
317-
Cerr << "iiiii TEvResponseHiveStorageStats" << Endl;
318316
HiveStorageStats[ev->Cookie] = ev->Release();
319317
RequestDone();
320318
}
@@ -354,41 +352,39 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
354352
}
355353

356354
void Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev) {
357-
Cerr << "iiiii TEvVDiskStateResponse" << Endl;
358355
ui64 nodeId = ev.Get()->Cookie;
359-
Cerr << "iiiii nodeid" << nodeId << Endl;
360356
auto& vDiskInfo = VDiskInfo[nodeId] = std::move(ev->Get()->Record);
361357
for (auto& vDiskStateInfo : *(vDiskInfo.MutableVDiskStateInfo())) {
362358
vDiskStateInfo.SetNodeId(nodeId);
363359
VDiskId2vDiskStateInfo[VDiskIDFromVDiskID(vDiskStateInfo.GetVDiskId())] = &vDiskStateInfo;
360+
361+
if ((FilterNodeIds.empty() || FilterNodeIds.contains(nodeId))
362+
&& (FilterPDiskIds.empty() || FilterPDiskIds.contains(vDiskStateInfo.GetPDiskId()))
363+
&& (FilterGroupIds.empty() || FilterGroupIds.contains(ToString(vDiskStateInfo.GetVDiskId().GetGroupID())))) {
364+
EffectiveGroupFilter.insert(ToString(vDiskStateInfo.GetVDiskId().GetGroupID()));
365+
}
364366
}
365367
RequestDone();
366368
}
367369

368370
void Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev) {
369371
ui64 nodeId = ev.Get()->Cookie;
372+
370373
PDiskInfo[nodeId] = std::move(ev->Get()->Record);
371374
RequestDone();
372375
}
373376

374377
void Handle(TEvWhiteboard::TEvBSGroupStateResponse::TPtr& ev) {
375378
ui64 nodeId = ev.Get()->Cookie;
376-
Cerr << "iiiiiiiiiiiii Cookie " << nodeId << Endl;
377379
for (const auto& info : ev->Get()->Record.GetBSGroupStateInfo()) {
378380
TString storagePoolName = info.GetStoragePoolName();
379381
if (storagePoolName.empty()) {
380382
continue;
381383
}
382-
Cerr << "iiiiiiiiiiiii nodeId " << info.GetNodeId() << Endl;
383384
if (FilterNodeIds.empty() || FilterNodeIds.contains(info.GetNodeId())) {
384385
StoragePoolInfo[storagePoolName].Groups.emplace(ToString(info.GetGroupID()));
385-
TString groupId(ToString(info.GetGroupID()));
386-
if (FilterGroupIds.empty() || BinarySearch(FilterGroupIds.begin(), FilterGroupIds.end(), groupId)) {
387-
EffectiveFilterGroupIds.insert(groupId);
388-
}
389386
}
390387
for (const auto& vDiskNodeId : info.GetVDiskNodeIds()) {
391-
Cerr << "iiiiiiiiiiiii vDiskNodeId " << vDiskNodeId << Endl;
392388
Group2NodeId[info.GetGroupID()].push_back(vDiskNodeId);
393389
}
394390
}

‎ydb/core/viewer/viewer_ut.cpp‎

Lines changed: 102 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,6 @@ Y_UNIT_TEST_SUITE(Viewer) {
269269

270270
auto sample = nodes[0];
271271
nodes.clear();
272-
273272
for (int nodeId = 0; nodeId < nodesTotal; nodeId++) {
274273
sample.NodeId = nodeId;
275274
nodes.emplace_back(sample);
@@ -1107,50 +1106,83 @@ Y_UNIT_TEST_SUITE(Viewer) {
11071106
}
11081107

11091108
void ChangeBSGroupStateResponse(TEvWhiteboard::TEvBSGroupStateResponse::TPtr* ev) {
1109+
ui64 nodeId = (*ev)->Cookie;
11101110
auto& pbRecord = (*ev)->Get()->Record;
1111-
// pbRecord.clear_bsgroupstateinfo();
1112-
// auto state = pbRecord.add_bsgroupstateinfo();
1113-
Cerr << pbRecord.ShortDebugString() << Endl;
1111+
1112+
pbRecord.clear_bsgroupstateinfo();
1113+
1114+
for (ui64 groupId = 1; groupId <= 9; groupId++) {
1115+
if (groupId == nodeId) {
1116+
continue;
1117+
}
1118+
auto state = pbRecord.add_bsgroupstateinfo();
1119+
state->set_groupid(groupId);
1120+
state->set_storagepoolname("/Root:test");
1121+
state->set_nodeid(nodeId);
1122+
for (int k = 1; k <= 8; k++) {
1123+
auto vdisk = groupId * 8 + k;
1124+
auto vdiskId = state->add_vdiskids();
1125+
vdiskId->set_groupid(groupId);
1126+
vdiskId->set_groupgeneration(1);
1127+
vdiskId->set_vdisk(vdisk);
1128+
}
1129+
}
11141130
}
11151131

11161132
void ChangePDiskStateResponse(TEvWhiteboard::TEvPDiskStateResponse::TPtr* ev) {
11171133
auto& pbRecord = (*ev)->Get()->Record;
1118-
// pbRecord.clear_pdiskstateinfo();
1119-
// auto state = pbRecord.add_pdiskstateinfo();
1120-
Cerr << pbRecord.ShortDebugString() << Endl;
1134+
pbRecord.clear_pdiskstateinfo();
1135+
for (int k = 0; k < 2; k++) {
1136+
auto state = pbRecord.add_pdiskstateinfo();
1137+
state->set_pdiskid(k);
1138+
}
11211139
}
11221140

11231141

11241142
void ChangeVDiskStateOn9NodeResponse(NNodeWhiteboard::TEvWhiteboard::TEvVDiskStateResponse::TPtr* ev) {
11251143
ui64 nodeId = (*ev)->Cookie;
11261144
auto& pbRecord = (*ev)->Get()->Record;
11271145

1128-
auto sample = pbRecord.vdiskstateinfo(0);
11291146
pbRecord.clear_vdiskstateinfo();
11301147

1131-
for (int k = 1; k <= 8; k++) {
1132-
auto groupId = (nodeId + k) % 9;
1133-
ui32 pdisk = k <= 4 ? 0 : 1;
1148+
for (int k = 0; k < 8; k++) {
1149+
auto groupId = (nodeId + k) % 9 + 1;
1150+
auto vdisk = groupId * 8 + k + 1;
1151+
ui32 pdisk = k / 4;
11341152
ui32 slotid = k % 4;
1135-
auto vdisk = 0;
11361153
auto state = pbRecord.add_vdiskstateinfo();
1137-
state->CopyFrom(sample);
11381154
state->set_pdiskid(pdisk);
11391155
state->set_vdiskslotid(slotid);
1140-
state->mutable_vdiskid()->set_vdisk(vdisk++);
11411156
state->mutable_vdiskid()->set_groupid(groupId);
1157+
state->mutable_vdiskid()->set_groupgeneration(1);
1158+
state->mutable_vdiskid()->set_vdisk(vdisk++);
11421159
state->set_vdiskstate(NKikimrWhiteboard::EVDiskState::OK);
11431160
state->set_nodeid(nodeId);
11441161
}
11451162
}
11461163

1147-
void JsonStorage9NodesListingTest() {
1164+
void AddGroupsInControllerSelectGroupsResult(TEvBlobStorage::TEvControllerSelectGroupsResult::TPtr* ev, int groupCount) {
1165+
auto& pbRecord = (*ev)->Get()->Record;
1166+
auto pbMatchGroups = pbRecord.mutable_matchinggroups(0);
1167+
1168+
auto sample = pbMatchGroups->groups(0);
1169+
pbMatchGroups->ClearGroups();
1170+
1171+
for (int groupId = 1; groupId <= groupCount; groupId++) {
1172+
auto group = pbMatchGroups->add_groups();
1173+
group->CopyFrom(sample);
1174+
group->set_groupid(groupId++);
1175+
group->set_storagepoolname("/Root:test");
1176+
}
1177+
};
1178+
1179+
void JsonStorage9Nodes9GroupsListingTest(TString version, bool groupFilter, bool nodeFilter, bool pdiskFilter, ui32 expectedFoundGroups, ui32 expectedTotalGroups) {
11481180
TPortManager tp;
11491181
ui16 port = tp.GetPort(2134);
11501182
ui16 grpcPort = tp.GetPort(2135);
11511183
auto settings = TServerSettings(port);
11521184
settings.InitKikimrRunConfig()
1153-
.SetNodeCount(1)
1185+
.SetNodeCount(9)
11541186
.SetUseRealThreads(false)
11551187
.SetDomainName("Root");
11561188
TServer server(settings);
@@ -1163,41 +1195,49 @@ Y_UNIT_TEST_SUITE(Viewer) {
11631195

11641196
THttpRequest httpReq(HTTP_METHOD_GET);
11651197
httpReq.CgiParameters.emplace("with", "all");
1166-
httpReq.CgiParameters.emplace("version", "v2");
1167-
// httpReq.CgiParameters.emplace("node_id", "1");
1198+
httpReq.CgiParameters.emplace("version", version);
1199+
if (groupFilter) {
1200+
httpReq.CgiParameters.emplace("group_id", "1");
1201+
}
1202+
if (nodeFilter) {
1203+
httpReq.CgiParameters.emplace("node_id", "1");
1204+
}
1205+
if (pdiskFilter) {
1206+
httpReq.CgiParameters.emplace("pdisk_id", "0");
1207+
}
11681208
auto page = MakeHolder<TMonPage>("viewer", "title");
11691209
TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/storage", nullptr);
11701210
auto request = MakeHolder<NMon::TEvHttpInfo>(monReq);
11711211

11721212
auto observerFunc = [&](TAutoPtr<IEventHandle>& ev) {
11731213
Y_UNUSED(ev);
11741214
switch (ev->GetTypeRewrite()) {
1175-
case TEvInterconnect::EvNodesInfo: {
1176-
auto *x = reinterpret_cast<TEvInterconnect::TEvNodesInfo::TPtr*>(&ev);
1177-
Cerr << "aaaa EvNodesInfo 1" << Endl;
1178-
ChangeListNodes(x, 9);
1179-
Cerr << "aaaa EvNodesInfo 2" << Endl;
1215+
case NConsole::TEvConsole::EvListTenantsResponse: {
1216+
auto *x = reinterpret_cast<NConsole::TEvConsole::TEvListTenantsResponse::TPtr*>(&ev);
1217+
Ydb::Cms::ListDatabasesResult listTenantsResult;
1218+
(*x)->Get()->Record.GetResponse().operation().result().UnpackTo(&listTenantsResult);
1219+
listTenantsResult.Addpaths("/Root");
1220+
(*x)->Get()->Record.MutableResponse()->mutable_operation()->mutable_result()->PackFrom(listTenantsResult);
11801221
break;
11811222
}
11821223
case TEvWhiteboard::EvBSGroupStateResponse: {
11831224
auto *x = reinterpret_cast<TEvWhiteboard::TEvBSGroupStateResponse::TPtr*>(&ev);
1184-
Cerr << "aaaa EvBSGroupStateResponse 1" << Endl;
11851225
ChangeBSGroupStateResponse(x);
1186-
Cerr << "aaaa EvBSGroupStateResponse 2" << Endl;
11871226
break;
11881227
}
11891228
case TEvWhiteboard::EvVDiskStateResponse: {
11901229
auto *x = reinterpret_cast<TEvWhiteboard::TEvVDiskStateResponse::TPtr*>(&ev);
1191-
Cerr << "aaaa EvVDiskStateResponse 1" << Endl;
11921230
ChangeVDiskStateOn9NodeResponse(x);
1193-
Cerr << "aaaa EvVDiskStateResponse 2" << Endl;
11941231
break;
11951232
}
11961233
case TEvWhiteboard::EvPDiskStateResponse: {
11971234
auto *x = reinterpret_cast<TEvWhiteboard::TEvPDiskStateResponse::TPtr*>(&ev);
1198-
Cerr << "aaaa EvPDiskStateResponse 1" << Endl;
11991235
ChangePDiskStateResponse(x);
1200-
Cerr << "aaaa EvPDiskStateResponse 2" << Endl;
1236+
break;
1237+
}
1238+
case TEvBlobStorage::EvControllerSelectGroupsResult: {
1239+
auto *x = reinterpret_cast<TEvBlobStorage::TEvControllerSelectGroupsResult::TPtr*>(&ev);
1240+
AddGroupsInControllerSelectGroupsResult(x, 9);
12011241
break;
12021242
}
12031243
}
@@ -1211,18 +1251,47 @@ Y_UNIT_TEST_SUITE(Viewer) {
12111251

12121252
size_t pos = result->Answer.find('{');
12131253
TString jsonResult = result->Answer.substr(pos);
1214-
Cerr << "json result: " << jsonResult << Endl;
12151254
NJson::TJsonValue json;
12161255
try {
12171256
NJson::ReadJsonTree(jsonResult, &json, true);
12181257
}
12191258
catch (yexception ex) {
12201259
Ctest << ex.what() << Endl;
12211260
}
1222-
// UNIT_ASSERT_VALUES_EQUAL(json.GetMap().contains("StorageGroups"), isExpectingGroup);
1261+
1262+
UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("FoundGroups"), ToString(expectedFoundGroups));
1263+
UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("TotalGroups"), ToString(expectedTotalGroups));
1264+
}
1265+
1266+
Y_UNIT_TEST(JsonStorageListingV1) {
1267+
JsonStorage9Nodes9GroupsListingTest("v1", false, false, false, 9, 9);
1268+
}
1269+
1270+
Y_UNIT_TEST(JsonStorageListingV2) {
1271+
JsonStorage9Nodes9GroupsListingTest("v2", false, false, false, 9, 9);
1272+
}
1273+
1274+
Y_UNIT_TEST(JsonStorageListingV1GroupIdFilter) {
1275+
JsonStorage9Nodes9GroupsListingTest("v1", true, false, false, 1, 9);
1276+
}
1277+
1278+
Y_UNIT_TEST(JsonStorageListingV2GroupIdFilter) {
1279+
JsonStorage9Nodes9GroupsListingTest("v2", true, false, false, 1, 9);
1280+
}
1281+
1282+
Y_UNIT_TEST(JsonStorageListingV1NodeIdFilter) {
1283+
JsonStorage9Nodes9GroupsListingTest("v1", false, true, false, 8, 8);
1284+
}
1285+
1286+
Y_UNIT_TEST(JsonStorageListingV2NodeIdFilter) {
1287+
JsonStorage9Nodes9GroupsListingTest("v2", false, true, false, 8, 8);
1288+
}
1289+
1290+
Y_UNIT_TEST(JsonStorageListingV1PDiskIdFilter) {
1291+
JsonStorage9Nodes9GroupsListingTest("v1", false, true, true, 4, 8);
12231292
}
12241293

1225-
Y_UNIT_TEST(JsonStorage9NodesListingV1NodeIdFilter) {
1226-
JsonStorage9NodesListingTest();
1294+
Y_UNIT_TEST(JsonStorageListingV2PDiskIdFilter) {
1295+
JsonStorage9Nodes9GroupsListingTest("v2", false, true, true, 4, 8);
12271296
}
12281297
}

0 commit comments

Comments
 (0)