Skip to content

Health check for exclusive dynamic nodes KIKIMR-20818 #1223

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jan 30, 2024
Merged

Health check for exclusive dynamic nodes KIKIMR-20818 #1223

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c73ff7d
Health check for exclusive dynamic nodes KIKIMR-20818
pixcc Jan 23, 2024
59a3943
Add tests KIKIMR-20818
pixcc Jan 29, 2024
f998463
Add newline to end of file KIKIMR-20818
pixcc Jan 29, 2024
add87cf
Fix tests KIKIMR-20818
pixcc Jan 29, 2024
0d67b39
Fix tests build KIKIMR-20818
pixcc Jan 29, 2024
d001c63
Revert new issue when no storage pools info KIKIMR-20818
pixcc Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 30 additions & 21 deletions ydb/core/health_check/health_check.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,12 +228,13 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
TTabletId HiveId = {};
TPathId ResourcePathId = {};
TVector<TNodeId> ComputeNodeIds;
TVector<TString> StoragePoolNames;
THashSet<TString> StoragePoolNames;
THashMap<std::pair<TTabletId, NNodeWhiteboard::TFollowerId>, const NKikimrHive::TTabletInfo*> MergedTabletState;
THashMap<TNodeId, TNodeTabletState> MergedNodeTabletState;
THashMap<TNodeId, ui32> NodeRestartsPerPeriod;
ui64 StorageQuota;
ui64 StorageUsage;
TMaybeServerlessComputeResourcesMode ServerlessComputeResourcesMode;
};

struct TSelfCheckResult {
Expand Down Expand Up @@ -514,7 +515,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
TDuration Timeout = TDuration::MilliSeconds(20000);
static constexpr TStringBuf STATIC_STORAGE_POOL_NAME = "static";

bool IsSpecificDatabaseFilter() {
bool IsSpecificDatabaseFilter() const {
return FilterDatabase && FilterDatabase != DomainPath;
}

Expand Down Expand Up @@ -593,7 +594,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
StoragePoolState[storagePoolName].Groups.emplace(group.groupid());

if (!IsSpecificDatabaseFilter()) {
DatabaseState[DomainPath].StoragePoolNames.emplace_back(storagePoolName);
DatabaseState[DomainPath].StoragePoolNames.emplace(storagePoolName);
}
}
}
Expand Down Expand Up @@ -869,12 +870,12 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
TDatabaseState& state(DatabaseState[path]);
for (const auto& storagePool : ev->Get()->GetRecord().pathdescription().domaindescription().storagepools()) {
TString storagePoolName = storagePool.name();
state.StoragePoolNames.emplace_back(storagePoolName);
state.StoragePoolNames.emplace(storagePoolName);
StoragePoolState[storagePoolName].Kind = storagePool.kind();
RequestSelectGroups(storagePoolName);
}
if (path == DomainPath) {
state.StoragePoolNames.emplace_back(STATIC_STORAGE_POOL_NAME);
state.StoragePoolNames.emplace(STATIC_STORAGE_POOL_NAME);
}
state.StorageUsage = ev->Get()->GetRecord().pathdescription().domaindescription().diskspaceusage().tables().totalsize();
state.StorageQuota = ev->Get()->GetRecord().pathdescription().domaindescription().databasequotas().data_size_hard_quota();
Expand All @@ -888,12 +889,19 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
if (ev->Get()->Request->ResultSet.size() == 1 && ev->Get()->Request->ResultSet.begin()->Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) {
auto domainInfo = ev->Get()->Request->ResultSet.begin()->DomainInfo;
TString path = CanonizePath(ev->Get()->Request->ResultSet.begin()->Path);

if (domainInfo->DomainKey != domainInfo->ResourcesDomainKey) {
if (SharedDatabases.emplace(domainInfo->ResourcesDomainKey, path).second) {
RequestSchemeCacheNavigate(domainInfo->ResourcesDomainKey);
if (domainInfo->IsServerless()) {
if (NeedHealthCheckForServerless(domainInfo)) {
if (SharedDatabases.emplace(domainInfo->ResourcesDomainKey, path).second) {
RequestSchemeCacheNavigate(domainInfo->ResourcesDomainKey);
}
DatabaseState[path].ResourcePathId = domainInfo->ResourcesDomainKey;
DatabaseState[path].ServerlessComputeResourcesMode = domainInfo->ServerlessComputeResourcesMode;
} else {
DatabaseState.erase(path);
DatabaseStatusByPath.erase(path);
RequestDone("TEvNavigateKeySetResult");
return;
}
DatabaseState[path].ResourcePathId = domainInfo->ResourcesDomainKey;
}
TTabletId hiveId = domainInfo->Params.GetHive();
if (hiveId) {
Expand All @@ -918,6 +926,11 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
RequestDone("TEvNavigateKeySetResult");
}

bool NeedHealthCheckForServerless(TIntrusivePtr<NSchemeCache::TDomainInfo> domainInfo) const {
return IsSpecificDatabaseFilter()
|| domainInfo->ServerlessComputeResourcesMode == NKikimrSubDomains::EServerlessComputeResourcesModeExclusive;
}

void Handle(TEvHive::TEvResponseHiveDomainStats::TPtr& ev) {
TTabletId hiveId = TabletRequests.CompleteRequest(ev->Cookie);
for (const NKikimrHive::THiveDomainStats& hiveStat : ev->Get()->Record.GetDomainStats()) {
Expand Down Expand Up @@ -951,15 +964,9 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
Ydb::Cms::GetDatabaseStatusResult getTenantStatusResult;
operation.result().UnpackTo(&getTenantStatusResult);
TString path = getTenantStatusResult.path();

bool ignoreServerlessDatabases = !IsSpecificDatabaseFilter(); // we don't ignore sl database if it was exactly specified
if (getTenantStatusResult.has_serverless_resources() && ignoreServerlessDatabases) {
DatabaseState.erase(path);
} else {
DatabaseStatusByPath[path] = std::move(getTenantStatusResult);
DatabaseState[path];
RequestSchemeCacheNavigate(path);
}
DatabaseStatusByPath[path] = std::move(getTenantStatusResult);
DatabaseState[path];
RequestSchemeCacheNavigate(path);
}
RequestDone("TEvGetTenantStatusResponse");
}
Expand Down Expand Up @@ -1298,7 +1305,9 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {

void FillCompute(TDatabaseState& databaseState, Ydb::Monitoring::ComputeStatus& computeStatus, TSelfCheckContext context) {
TVector<TNodeId>* computeNodeIds = &databaseState.ComputeNodeIds;
if (databaseState.ResourcePathId) {
if (databaseState.ResourcePathId
&& databaseState.ServerlessComputeResourcesMode != NKikimrSubDomains::EServerlessComputeResourcesModeExclusive)
{
auto itDatabase = FilterDomainKey.find(TSubDomainKey(databaseState.ResourcePathId.OwnerId, databaseState.ResourcePathId.LocalPathId));
if (itDatabase != FilterDomainKey.end()) {
const TString& sharedDatabaseName = itDatabase->second;
Expand Down Expand Up @@ -2124,7 +2133,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
TDatabaseState unknownDatabase;
for (auto& [name, pool] : StoragePoolState) {
if (StoragePoolSeen.count(name) == 0) {
unknownDatabase.StoragePoolNames.push_back(name);
unknownDatabase.StoragePoolNames.insert(name);
}
}
if (!unknownDatabase.StoragePoolNames.empty()) {
Expand Down
Loading