Skip to content

Commit 18ab692

Browse files
StekPerepolnenStekPerepolnen
committed
added pdisk filter
1 parent 72e47d6 commit 18ab692

File tree

3 files changed

+114
-51
lines changed

3 files changed

+114
-51
lines changed

‎ydb/core/viewer/json_storage.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,6 @@ class TJsonStorage : public TJsonStorageBase {
125125
}
126126

127127
void Bootstrap() override {
128-
Cerr << "iiiii Bootstrap" << Endl;
129128
TIntrusivePtr<TDomainsInfo> domains = AppData()->DomainsInfo;
130129
ui64 hiveId = domains->GetHive();
131130
if (hiveId != TDomainsInfo::BadTabletId) {
@@ -251,7 +250,7 @@ class TJsonStorage : public TJsonStorageBase {
251250
}
252251

253252
bool CheckGroupFilters(const TString& groupId, const TString& poolName, const TGroupRow& groupRow) {
254-
if (!EffectiveFilterGroupIds.empty() && !EffectiveFilterGroupIds.contains(groupId)) {
253+
if (!EffectiveGroupFilter.contains(groupId)) {
255254
return false;
256255
}
257256
switch (With) {
@@ -498,6 +497,7 @@ struct TJsonRequestParameters<TJsonStorage> {
498497
{"name":"tenant","in":"query","description":"tenant name","required":false,"type":"string"},
499498
{"name":"pool","in":"query","description":"storage pool name","required":false,"type":"string"},
500499
{"name":"node_id","in":"query","description":"node id","required":false,"type":"integer"},
500+
{"name":"pdisk_id","in":"query","description":"pdisk id","required":false,"type":"integer"},
501501
{"name":"group_id","in":"query","description":"group id","required":false,"type":"integer"},
502502
{"name":"need_groups","in":"query","description":"return groups information","required":false,"type":"boolean","default":true},
503503
{"name":"need_disks","in":"query","description":"return disks information","required":false,"type":"boolean","default":true},

‎ydb/core/viewer/json_storage_base.h‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,11 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
7272
ui32 Timeout = 0;
7373
TString FilterTenant;
7474
THashSet<TString> FilterStoragePools;
75-
TVector<TString> FilterGroupIds;
7675
TString Filter;
76+
std::unordered_set<TString> FilterGroupIds;
7777
std::unordered_set<TNodeId> FilterNodeIds;
78-
THashSet<TString> EffectiveFilterGroupIds;
78+
std::unordered_set<TNodeId> FilterPDiskIds;
79+
THashSet<TString> EffectiveGroupFilter;
7980
std::unordered_set<TNodeId> NodeIds;
8081
bool NeedAdditionalNodesRequests;
8182

@@ -133,9 +134,9 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
133134
FilterStoragePools.emplace(filterStoragePool);
134135
}
135136
SplitIds(params.Get("node_id"), ',', FilterNodeIds);
137+
SplitIds(params.Get("pdisk_id"), ',', FilterPDiskIds);
136138
NeedAdditionalNodesRequests = !FilterNodeIds.empty();
137139
SplitIds(params.Get("group_id"), ',', FilterGroupIds);
138-
Sort(FilterGroupIds);
139140
Filter = params.Get("filter");
140141
if (params.Get("with") == "missing") {
141142
With = EWith::MissingDisks;
@@ -263,7 +264,6 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
263264
}
264265

265266
void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) {
266-
Cerr << "iiiiiiiiii TEvNodesInfo 1" << Endl;
267267
ui32 maxAllowedNodeId = std::numeric_limits<ui32>::max();
268268
TIntrusivePtr<TDynamicNameserviceConfig> dynamicNameserviceConfig = AppData()->DynamicNameserviceConfig;
269269
if (dynamicNameserviceConfig) {
@@ -275,7 +275,6 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
275275
SendNodeRequests(ni.NodeId);
276276
}
277277
}
278-
Cerr << "iiiiiiiiii TEvNodesInfo 2" << Endl;
279278
RequestDone();
280279
}
281280

@@ -314,7 +313,6 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
314313
}
315314

316315
void Handle(TEvHive::TEvResponseHiveStorageStats::TPtr& ev) {
317-
Cerr << "iiiii TEvResponseHiveStorageStats" << Endl;
318316
HiveStorageStats[ev->Cookie] = ev->Release();
319317
RequestDone();
320318
}
@@ -354,13 +352,17 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
354352
}
355353

356354
void Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev) {
357-
Cerr << "iiiii TEvVDiskStateResponse" << Endl;
358355
ui64 nodeId = ev.Get()->Cookie;
359-
Cerr << "iiiii nodeid" << nodeId << Endl;
360356
auto& vDiskInfo = VDiskInfo[nodeId] = std::move(ev->Get()->Record);
361357
for (auto& vDiskStateInfo : *(vDiskInfo.MutableVDiskStateInfo())) {
362358
vDiskStateInfo.SetNodeId(nodeId);
363359
VDiskId2vDiskStateInfo[VDiskIDFromVDiskID(vDiskStateInfo.GetVDiskId())] = &vDiskStateInfo;
360+
361+
if ((FilterNodeIds.empty() || FilterNodeIds.contains(nodeId))
362+
&& (FilterPDiskIds.empty() || FilterPDiskIds.contains(vDiskStateInfo.GetPDiskId()))
363+
&& (FilterGroupIds.empty() || FilterGroupIds.contains(ToString(vDiskStateInfo.GetVDiskId().GetGroupID())))) {
364+
EffectiveGroupFilter.insert(ToString(vDiskStateInfo.GetVDiskId().GetGroupID()));
365+
}
364366
}
365367
RequestDone();
366368
}
@@ -373,22 +375,15 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
373375

374376
void Handle(TEvWhiteboard::TEvBSGroupStateResponse::TPtr& ev) {
375377
ui64 nodeId = ev.Get()->Cookie;
376-
Cerr << "iiiiiiiiiiiii Cookie " << nodeId << Endl;
377378
for (const auto& info : ev->Get()->Record.GetBSGroupStateInfo()) {
378379
TString storagePoolName = info.GetStoragePoolName();
379380
if (storagePoolName.empty()) {
380381
continue;
381382
}
382-
Cerr << "iiiiiiiiiiiii nodeId " << info.GetNodeId() << Endl;
383383
if (FilterNodeIds.empty() || FilterNodeIds.contains(info.GetNodeId())) {
384384
StoragePoolInfo[storagePoolName].Groups.emplace(ToString(info.GetGroupID()));
385-
TString groupId(ToString(info.GetGroupID()));
386-
if (FilterGroupIds.empty() || BinarySearch(FilterGroupIds.begin(), FilterGroupIds.end(), groupId)) {
387-
EffectiveFilterGroupIds.insert(groupId);
388-
}
389385
}
390386
for (const auto& vDiskNodeId : info.GetVDiskNodeIds()) {
391-
Cerr << "iiiiiiiiiiiii vDiskNodeId " << vDiskNodeId << Endl;
392387
Group2NodeId[info.GetGroupID()].push_back(vDiskNodeId);
393388
}
394389
}

‎ydb/core/viewer/viewer_ut.cpp‎

Lines changed: 102 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,6 @@ Y_UNIT_TEST_SUITE(Viewer) {
272272

273273
auto sample = nodes[0];
274274
nodes.clear();
275-
276275
for (int nodeId = 0; nodeId < nodesTotal; nodeId++) {
277276
sample.NodeId = nodeId;
278277
nodes.emplace_back(sample);
@@ -1320,50 +1319,82 @@ Y_UNIT_TEST_SUITE(Viewer) {
13201319
}
13211320

13221321
void ChangeBSGroupStateResponse(TEvWhiteboard::TEvBSGroupStateResponse::TPtr* ev) {
1322+
ui64 nodeId = (*ev)->Cookie;
13231323
auto& pbRecord = (*ev)->Get()->Record;
1324-
// pbRecord.clear_bsgroupstateinfo();
1325-
// auto state = pbRecord.add_bsgroupstateinfo();
1326-
Cerr << pbRecord.ShortDebugString() << Endl;
1324+
1325+
pbRecord.clear_bsgroupstateinfo();
1326+
1327+
for (ui64 groupId = 1; groupId <= 9; groupId++) {
1328+
if (groupId == nodeId) {
1329+
continue;
1330+
}
1331+
auto state = pbRecord.add_bsgroupstateinfo();
1332+
state->set_groupid(groupId);
1333+
state->set_storagepoolname("/Root:test");
1334+
state->set_nodeid(nodeId);
1335+
for (int k = 1; k <= 8; k++) {
1336+
auto vdisk = groupId * 8 + k;
1337+
auto vdiskId = state->add_vdiskids();
1338+
vdiskId->set_groupid(groupId);
1339+
vdiskId->set_groupgeneration(1);
1340+
vdiskId->set_vdisk(vdisk);
1341+
}
1342+
}
13271343
}
13281344

13291345
void ChangePDiskStateResponse(TEvWhiteboard::TEvPDiskStateResponse::TPtr* ev) {
13301346
auto& pbRecord = (*ev)->Get()->Record;
1331-
// pbRecord.clear_pdiskstateinfo();
1332-
// auto state = pbRecord.add_pdiskstateinfo();
1333-
Cerr << pbRecord.ShortDebugString() << Endl;
1347+
pbRecord.clear_pdiskstateinfo();
1348+
for (int k = 0; k < 2; k++) {
1349+
auto state = pbRecord.add_pdiskstateinfo();
1350+
state->set_pdiskid(k);
1351+
}
13341352
}
13351353

1336-
13371354
void ChangeVDiskStateOn9NodeResponse(NNodeWhiteboard::TEvWhiteboard::TEvVDiskStateResponse::TPtr* ev) {
13381355
ui64 nodeId = (*ev)->Cookie;
13391356
auto& pbRecord = (*ev)->Get()->Record;
13401357

1341-
auto sample = pbRecord.vdiskstateinfo(0);
13421358
pbRecord.clear_vdiskstateinfo();
13431359

1344-
for (int k = 1; k <= 8; k++) {
1345-
auto groupId = (nodeId + k) % 9;
1346-
ui32 pdisk = k <= 4 ? 0 : 1;
1360+
for (int k = 0; k < 8; k++) {
1361+
auto groupId = (nodeId + k) % 9 + 1;
1362+
auto vdisk = groupId * 8 + k + 1;
1363+
ui32 pdisk = k / 4;
13471364
ui32 slotid = k % 4;
1348-
auto vdisk = 0;
13491365
auto state = pbRecord.add_vdiskstateinfo();
1350-
state->CopyFrom(sample);
13511366
state->set_pdiskid(pdisk);
13521367
state->set_vdiskslotid(slotid);
1353-
state->mutable_vdiskid()->set_vdisk(vdisk++);
13541368
state->mutable_vdiskid()->set_groupid(groupId);
1369+
state->mutable_vdiskid()->set_groupgeneration(1);
1370+
state->mutable_vdiskid()->set_vdisk(vdisk++);
13551371
state->set_vdiskstate(NKikimrWhiteboard::EVDiskState::OK);
13561372
state->set_nodeid(nodeId);
13571373
}
13581374
}
13591375

1360-
void JsonStorage9NodesListingTest() {
1376+
void AddGroupsInControllerSelectGroupsResult(TEvBlobStorage::TEvControllerSelectGroupsResult::TPtr* ev, int groupCount) {
1377+
auto& pbRecord = (*ev)->Get()->Record;
1378+
auto pbMatchGroups = pbRecord.mutable_matchinggroups(0);
1379+
1380+
auto sample = pbMatchGroups->groups(0);
1381+
pbMatchGroups->ClearGroups();
1382+
1383+
for (int groupId = 1; groupId <= groupCount; groupId++) {
1384+
auto group = pbMatchGroups->add_groups();
1385+
group->CopyFrom(sample);
1386+
group->set_groupid(groupId++);
1387+
group->set_storagepoolname("/Root:test");
1388+
}
1389+
};
1390+
1391+
void JsonStorage9Nodes9GroupsListingTest(TString version, bool groupFilter, bool nodeFilter, bool pdiskFilter, ui32 expectedFoundGroups, ui32 expectedTotalGroups) {
13611392
TPortManager tp;
13621393
ui16 port = tp.GetPort(2134);
13631394
ui16 grpcPort = tp.GetPort(2135);
13641395
auto settings = TServerSettings(port);
13651396
settings.InitKikimrRunConfig()
1366-
.SetNodeCount(1)
1397+
.SetNodeCount(9)
13671398
.SetUseRealThreads(false)
13681399
.SetDomainName("Root");
13691400
TServer server(settings);
@@ -1376,41 +1407,49 @@ Y_UNIT_TEST_SUITE(Viewer) {
13761407

13771408
THttpRequest httpReq(HTTP_METHOD_GET);
13781409
httpReq.CgiParameters.emplace("with", "all");
1379-
httpReq.CgiParameters.emplace("version", "v2");
1380-
// httpReq.CgiParameters.emplace("node_id", "1");
1410+
httpReq.CgiParameters.emplace("version", version);
1411+
if (groupFilter) {
1412+
httpReq.CgiParameters.emplace("group_id", "1");
1413+
}
1414+
if (nodeFilter) {
1415+
httpReq.CgiParameters.emplace("node_id", "1");
1416+
}
1417+
if (pdiskFilter) {
1418+
httpReq.CgiParameters.emplace("pdisk_id", "0");
1419+
}
13811420
auto page = MakeHolder<TMonPage>("viewer", "title");
13821421
TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/storage", nullptr);
13831422
auto request = MakeHolder<NMon::TEvHttpInfo>(monReq);
13841423

13851424
auto observerFunc = [&](TAutoPtr<IEventHandle>& ev) {
13861425
Y_UNUSED(ev);
13871426
switch (ev->GetTypeRewrite()) {
1388-
case TEvInterconnect::EvNodesInfo: {
1389-
auto *x = reinterpret_cast<TEvInterconnect::TEvNodesInfo::TPtr*>(&ev);
1390-
Cerr << "aaaa EvNodesInfo 1" << Endl;
1391-
ChangeListNodes(x, 9);
1392-
Cerr << "aaaa EvNodesInfo 2" << Endl;
1427+
case NConsole::TEvConsole::EvListTenantsResponse: {
1428+
auto *x = reinterpret_cast<NConsole::TEvConsole::TEvListTenantsResponse::TPtr*>(&ev);
1429+
Ydb::Cms::ListDatabasesResult listTenantsResult;
1430+
(*x)->Get()->Record.GetResponse().operation().result().UnpackTo(&listTenantsResult);
1431+
listTenantsResult.Addpaths("/Root");
1432+
(*x)->Get()->Record.MutableResponse()->mutable_operation()->mutable_result()->PackFrom(listTenantsResult);
13931433
break;
13941434
}
13951435
case TEvWhiteboard::EvBSGroupStateResponse: {
13961436
auto *x = reinterpret_cast<TEvWhiteboard::TEvBSGroupStateResponse::TPtr*>(&ev);
1397-
Cerr << "aaaa EvBSGroupStateResponse 1" << Endl;
13981437
ChangeBSGroupStateResponse(x);
1399-
Cerr << "aaaa EvBSGroupStateResponse 2" << Endl;
14001438
break;
14011439
}
14021440
case TEvWhiteboard::EvVDiskStateResponse: {
14031441
auto *x = reinterpret_cast<TEvWhiteboard::TEvVDiskStateResponse::TPtr*>(&ev);
1404-
Cerr << "aaaa EvVDiskStateResponse 1" << Endl;
14051442
ChangeVDiskStateOn9NodeResponse(x);
1406-
Cerr << "aaaa EvVDiskStateResponse 2" << Endl;
14071443
break;
14081444
}
14091445
case TEvWhiteboard::EvPDiskStateResponse: {
14101446
auto *x = reinterpret_cast<TEvWhiteboard::TEvPDiskStateResponse::TPtr*>(&ev);
1411-
Cerr << "aaaa EvPDiskStateResponse 1" << Endl;
14121447
ChangePDiskStateResponse(x);
1413-
Cerr << "aaaa EvPDiskStateResponse 2" << Endl;
1448+
break;
1449+
}
1450+
case TEvBlobStorage::EvControllerSelectGroupsResult: {
1451+
auto *x = reinterpret_cast<TEvBlobStorage::TEvControllerSelectGroupsResult::TPtr*>(&ev);
1452+
AddGroupsInControllerSelectGroupsResult(x, 9);
14141453
break;
14151454
}
14161455
}
@@ -1424,18 +1463,47 @@ Y_UNIT_TEST_SUITE(Viewer) {
14241463

14251464
size_t pos = result->Answer.find('{');
14261465
TString jsonResult = result->Answer.substr(pos);
1427-
Cerr << "json result: " << jsonResult << Endl;
14281466
NJson::TJsonValue json;
14291467
try {
14301468
NJson::ReadJsonTree(jsonResult, &json, true);
14311469
}
14321470
catch (yexception ex) {
14331471
Ctest << ex.what() << Endl;
14341472
}
1435-
// UNIT_ASSERT_VALUES_EQUAL(json.GetMap().contains("StorageGroups"), isExpectingGroup);
1473+
1474+
UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("FoundGroups"), ToString(expectedFoundGroups));
1475+
UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("TotalGroups"), ToString(expectedTotalGroups));
1476+
}
1477+
1478+
Y_UNIT_TEST(JsonStorageListingV1) {
1479+
JsonStorage9Nodes9GroupsListingTest("v1", false, false, false, 9, 9);
1480+
}
1481+
1482+
Y_UNIT_TEST(JsonStorageListingV2) {
1483+
JsonStorage9Nodes9GroupsListingTest("v2", false, false, false, 9, 9);
1484+
}
1485+
1486+
Y_UNIT_TEST(JsonStorageListingV1GroupIdFilter) {
1487+
JsonStorage9Nodes9GroupsListingTest("v1", true, false, false, 1, 9);
1488+
}
1489+
1490+
Y_UNIT_TEST(JsonStorageListingV2GroupIdFilter) {
1491+
JsonStorage9Nodes9GroupsListingTest("v2", true, false, false, 1, 9);
1492+
}
1493+
1494+
Y_UNIT_TEST(JsonStorageListingV1NodeIdFilter) {
1495+
JsonStorage9Nodes9GroupsListingTest("v1", false, true, false, 8, 8);
1496+
}
1497+
1498+
Y_UNIT_TEST(JsonStorageListingV2NodeIdFilter) {
1499+
JsonStorage9Nodes9GroupsListingTest("v2", false, true, false, 8, 8);
1500+
}
1501+
1502+
Y_UNIT_TEST(JsonStorageListingV1PDiskIdFilter) {
1503+
JsonStorage9Nodes9GroupsListingTest("v1", false, true, true, 4, 8);
14361504
}
14371505

1438-
Y_UNIT_TEST(JsonStorage9NodesListingV1NodeIdFilter) {
1439-
JsonStorage9NodesListingTest();
1506+
Y_UNIT_TEST(JsonStorageListingV2PDiskIdFilter) {
1507+
JsonStorage9Nodes9GroupsListingTest("v2", false, true, true, 4, 8);
14401508
}
14411509
}

0 commit comments

Comments
 (0)