Skip to content

KIKIMR-18397 HC node restarts issue #1111

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jan 18, 2024
Merged

KIKIMR-18397 HC node restarts issue #1111

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1d225a1
added RestartsPerPeriod info
StekPerepolnen Jan 17, 2024
3cd00af
added issue
StekPerepolnen Jan 17, 2024
28f77b4
node-check
StekPerepolnen Jan 18, 2024
f41531f
changed message to NODE UPTIME
StekPerepolnen Jan 18, 2024
7033417
changed messages
StekPerepolnen Jan 18, 2024
28b3ccc
changed message
StekPerepolnen Jan 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions ydb/core/health_check/health_check.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
SystemTabletState,
OverloadState,
SyncState,
Uptime,
};

struct TTenantInfo {
Expand Down Expand Up @@ -230,6 +231,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
TVector<TString> StoragePoolNames;
THashMap<std::pair<TTabletId, NNodeWhiteboard::TFollowerId>, const NKikimrHive::TTabletInfo*> MergedTabletState;
THashMap<TNodeId, TNodeTabletState> MergedNodeTabletState;
THashMap<TNodeId, ui32> NodeRestartsPerPeriod;
ui64 StorageQuota;
ui64 StorageUsage;
};
Expand Down Expand Up @@ -1056,6 +1058,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
TString path(itFilterDomainKey->second);
TDatabaseState& state(DatabaseState[path]);
state.ComputeNodeIds.emplace_back(hiveStat.GetNodeId());
state.NodeRestartsPerPeriod[hiveStat.GetNodeId()] = hiveStat.GetRestartsPerPeriod();
}
}
}
Expand Down Expand Up @@ -1246,9 +1249,18 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
}
}

void FillComputeNodeStatus(TNodeId nodeId, Ydb::Monitoring::ComputeNodeStatus& computeNodeStatus, TSelfCheckContext context) {
void FillComputeNodeStatus(TDatabaseState& databaseState,TNodeId nodeId, Ydb::Monitoring::ComputeNodeStatus& computeNodeStatus, TSelfCheckContext context) {
FillNodeInfo(nodeId, context.Location.mutable_compute()->mutable_node());

TSelfCheckContext rrContext(&context, "NODE_UPTIME");
if (databaseState.NodeRestartsPerPeriod[nodeId] >= 30) {
rrContext.ReportStatus(Ydb::Monitoring::StatusFlag::RED, "Node is restarting too often", ETags::Uptime);
} else if (databaseState.NodeRestartsPerPeriod[nodeId] >= 10) {
rrContext.ReportStatus(Ydb::Monitoring::StatusFlag::YELLOW, "The number of node restarts has increased", ETags::Uptime);
} else {
rrContext.ReportStatus(Ydb::Monitoring::StatusFlag::GREEN);
}

auto itNodeSystemState = MergedNodeSystemState.find(nodeId);
if (itNodeSystemState != MergedNodeSystemState.end()) {
const NKikimrWhiteboard::TSystemStateInfo& nodeSystemState(*itNodeSystemState->second);
Expand Down Expand Up @@ -1306,8 +1318,9 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
}
for (TNodeId nodeId : *computeNodeIds) {
auto& computeNode = *computeStatus.add_nodes();
FillComputeNodeStatus(nodeId, computeNode, {&context, "COMPUTE_NODE"});
FillComputeNodeStatus(databaseState, nodeId, computeNode, {&context, "COMPUTE_NODE"});
}
context.ReportWithMaxChildStatus("Some nodes are restarting too often", ETags::ComputeState, {ETags::Uptime});
context.ReportWithMaxChildStatus("Compute is overloaded", ETags::ComputeState, {ETags::OverloadState});
Ydb::Monitoring::StatusFlag::Status tabletsStatus = Ydb::Monitoring::StatusFlag::GREEN;
computeNodeIds->push_back(0); // for tablets without node
Expand Down