use health check status for cluster status (#27305)

Author: adameat

ydb/core/viewer/json_handlers_viewer.cpp

@@ -233,7 +233,7 @@ void InitViewerStorageUsageJsonHandler(TJsonHandlers &handlers) {
 }
 
 void InitViewerClusterJsonHandler(TJsonHandlers& handlers) {
-    handlers.AddHandler("/viewer/cluster", new TJsonHandler<TJsonCluster>(TJsonCluster::GetSwagger()), 7);
+    handlers.AddHandler("/viewer/cluster", new TJsonHandler<TJsonCluster>(TJsonCluster::GetSwagger()), 8);
 }
 
 void InitViewerLabeledCountersJsonHandler(TJsonHandlers &handlers) {

ydb/core/viewer/viewer_cluster.h

@@ -4,6 +4,7 @@
 #include "viewer.h"
 #include "viewer_helper.h"
 #include "viewer_tabletinfo.h"
+#include <ydb/library/actors/interconnect/interconnect.h>
 #include <ydb/public/api/protos/ydb_bridge_common.pb.h>
 
 namespace NKikimr::NViewer {
@@ -32,6 +33,10 @@ class TJsonCluster : public TViewerPipeClient {
     std::unordered_map<TNodeId, TRequestResponse<TEvViewer::TEvViewerResponse>> SystemViewerResponse;
     std::unordered_map<TNodeId, TRequestResponse<TEvViewer::TEvViewerResponse>> TabletViewerResponse;
 
+    std::optional<TRequestResponse<NHealthCheck::TEvSelfCheckResult>> SelfCheckResult; // from the local health check
+    std::optional<TRequestResponse<NHealthCheck::TEvSelfCheckResultProto>> SelfCheckResultProto; // from the metadata cache service
+    TViewerPipeClient::TRequestResponse<TEvStateStorage::TEvBoardInfo> MetadataCacheEndpointsLookup;
+
     struct TNode {
         TEvInterconnect::TNodeInfo NodeInfo;
         NKikimrWhiteboard::TSystemStateInfo SystemState;
@@ -91,26 +96,27 @@ class TJsonCluster : public TViewerPipeClient {
     std::unordered_set<TTabletId> FilterTablets;
     bool OffloadMerge = true;
     size_t OffloadMergeAttempts = 2;
+    bool UseHealthCheck = true;
+    bool UseHealthCheckCache = false; // doesn't work for domain ?
+    TString DomainName;
     TTabletId RootHiveId = 0;
-    TJsonSettings JsonSettings;
-    ui32 Timeout;
     bool Tablets = false;
 
 public:
     TJsonCluster(IViewer* viewer, NMon::TEvHttpInfo::TPtr& ev)
         : TViewerPipeClient(viewer, ev)
     {
-        const auto& params(Event->Get()->Request.GetParams());
-        JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), true);
-        JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false);
-        InitConfig(params);
-        Tablets = FromStringWithDefault<bool>(params.Get("tablets"), false);
-        Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000);
-        OffloadMerge = FromStringWithDefault<bool>(params.Get("offload_merge"), OffloadMerge);
-        OffloadMergeAttempts = FromStringWithDefault<bool>(params.Get("offload_merge_attempts"), OffloadMergeAttempts);
     }
 
     void Bootstrap() override {
+        Tablets = FromStringWithDefault<bool>(Params.Get("tablets"), false);
+        OffloadMerge = FromStringWithDefault<bool>(Params.Get("offload_merge"), OffloadMerge);
+        OffloadMergeAttempts = FromStringWithDefault<bool>(Params.Get("offload_merge_attempts"), OffloadMergeAttempts);
+        UseHealthCheck = FromStringWithDefault<bool>(Params.Get("use_health_check"), UseHealthCheck);
+        UseHealthCheckCache = FromStringWithDefault<bool>(Params.Get("use_health_check_cache"), UseHealthCheckCache);
+        if (!UseHealthCheck) {
+            UseHealthCheckCache = false;
+        }
         NodesInfoResponse = MakeRequest<TEvInterconnect::TEvNodesInfo>(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes());
         if (AppData()->BridgeModeEnabled) {
             NodeWardenStorageConfigResponse = MakeRequest<TEvNodeWardenStorageConfig>(MakeBlobStorageNodeWardenID(SelfId().NodeId()),
@@ -124,7 +130,8 @@ class TJsonCluster : public TViewerPipeClient {
         ListTenantsResponse = MakeRequestConsoleListTenants();
         if (AppData()->DomainsInfo && AppData()->DomainsInfo->Domain) {
             TIntrusivePtr<TDomainsInfo> domains = AppData()->DomainsInfo;
-            ClusterInfo.SetDomain(TStringBuilder() << "/" << AppData()->DomainsInfo->Domain->Name);
+            DomainName = TStringBuilder() << "/" << AppData()->DomainsInfo->Domain->Name;
+            ClusterInfo.SetDomain(DomainName);
             if (const auto& domain = domains->Domain) {
                 for (TTabletId id : domain->Coordinators) {
                     FilterTablets.insert(id);
@@ -146,8 +153,16 @@ class TJsonCluster : public TViewerPipeClient {
             FilterTablets.insert(MakeNodeBrokerID());
             FilterTablets.insert(MakeTenantSlotBrokerID());
             FilterTablets.insert(MakeConsoleID());
+
+            if (UseHealthCheck) {
+                if (UseHealthCheckCache && AppData()->FeatureFlags.GetEnableDbMetadataCache()) {
+                    MetadataCacheEndpointsLookup = MakeRequestStateStorageMetadataCacheEndpointsLookup(DomainName);
+                } else {
+                    SendHealthCheckRequest();
+                }
+            }
         }
-        Become(&TThis::StateWork, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup());
+        Become(&TThis::StateWork, Timeout, new TEvents::TEvWakeup());
     }
 
 private:
@@ -421,7 +436,7 @@ class TJsonCluster : public TViewerPipeClient {
             if (SystemViewerResponse.count(nodeId) == 0) {
                 auto viewerRequest = std::make_unique<TEvViewer::TEvViewerRequest>();
                 InitSystemWhiteboardRequest(viewerRequest->Record.MutableSystemRequest());
-                viewerRequest->Record.SetTimeout(Timeout / 2);
+                viewerRequest->Record.SetTimeout(Timeout.MilliSeconds() / 2);
                 for (const TNode* node : batch.NodesToAskAbout) {
                     viewerRequest->Record.MutableLocation()->AddNodeId(node->NodeId);
                 }
@@ -432,7 +447,7 @@ class TJsonCluster : public TViewerPipeClient {
             if (Tablets && batch.HasStaticNodes && TabletViewerResponse.count(nodeId) == 0) {
                 auto viewerRequest = std::make_unique<TEvViewer::TEvViewerRequest>();
                 InitTabletWhiteboardRequest(viewerRequest->Record.MutableTabletRequest());
-                viewerRequest->Record.SetTimeout(Timeout / 2);
+                viewerRequest->Record.SetTimeout(Timeout.MilliSeconds() / 2);
                 for (const TNode* node : batch.NodesToAskAbout) {
                     if (node->Static) {
                         viewerRequest->Record.MutableLocation()->AddNodeId(node->NodeId);
@@ -639,6 +654,45 @@ class TJsonCluster : public TViewerPipeClient {
         RequestDone();
     }
 
+    std::unique_ptr<NHealthCheck::TEvSelfCheckRequest> MakeSelfCheckRequest() {
+        auto request = std::make_unique<NHealthCheck::TEvSelfCheckRequest>();
+        request->Database = DomainName;
+        return request;
+    }
+
+    void SendHealthCheckRequest() {
+        SelfCheckResult = MakeRequest<NHealthCheck::TEvSelfCheckResult>(NHealthCheck::MakeHealthCheckID(), MakeSelfCheckRequest().release());
+    }
+
+    void Handle(NHealthCheck::TEvSelfCheckResult::TPtr& ev) {
+        if (SelfCheckResult->Set(std::move(ev))) {
+            RequestDone();
+        }
+    }
+
+    void Handle(NHealthCheck::TEvSelfCheckResultProto::TPtr& ev) {
+        if (SelfCheckResultProto->Set(std::move(ev))) {
+            RequestDone();
+        }
+    }
+
+    void Handle(TEvStateStorage::TEvBoardInfo::TPtr& ev) {
+        if (MetadataCacheEndpointsLookup.Set(std::move(ev))) {
+            if (MetadataCacheEndpointsLookup.IsOk()) {
+                auto activeNode = TDatabaseMetadataCache::PickActiveNode(MetadataCacheEndpointsLookup->InfoEntries);
+                if (activeNode != 0) {
+                    TActorId cache = MakeDatabaseMetadataCacheId(activeNode);
+                    auto request = std::make_unique<NHealthCheck::TEvSelfCheckRequestProto>();
+                    SelfCheckResultProto = MakeRequest<NHealthCheck::TEvSelfCheckResultProto>(cache, request.release(), IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, activeNode);
+                }
+            }
+            if (!SelfCheckResultProto) {
+                SendHealthCheckRequest();
+            }
+            RequestDone();
+        }
+    }
+
     void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) {
         if (NodesInfoResponse->Set(std::move(ev))) {
             ProcessResponses();
@@ -903,6 +957,8 @@ class TJsonCluster : public TViewerPipeClient {
             hFunc(NSysView::TEvSysView::TEvGetVSlotsResponse, Handle);
             hFunc(NSysView::TEvSysView::TEvGetGroupsResponse, Handle);
             hFunc(TEvHive::TEvResponseHiveNodeStats, Handle);
+            hFunc(NHealthCheck::TEvSelfCheckResult, Handle);
+            hFunc(NHealthCheck::TEvSelfCheckResultProto, Handle);
             hFunc(TEvents::TEvUndelivered, Undelivered);
             hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected);
             hFunc(TEvTabletPipe::TEvClientConnected, Handle);
@@ -922,6 +978,21 @@ class TJsonCluster : public TViewerPipeClient {
         }
     }
 
+    static NKikimrWhiteboard::EFlag GetClusterStateFromSelfCheck(const Ydb::Monitoring::SelfCheckResult& result) {
+        switch (result.self_check_result()) {
+            case Ydb::Monitoring::SelfCheck::GOOD:
+                return NKikimrWhiteboard::EFlag::Green;
+            case Ydb::Monitoring::SelfCheck::DEGRADED:
+                return NKikimrWhiteboard::EFlag::Yellow;
+            case Ydb::Monitoring::SelfCheck::MAINTENANCE_REQUIRED:
+                return NKikimrWhiteboard::EFlag::Red;
+            case Ydb::Monitoring::SelfCheck::EMERGENCY:
+                return NKikimrWhiteboard::EFlag::Red;
+            default:
+                return NKikimrWhiteboard::EFlag::Grey;
+        }
+    }
+
     void ReplyAndPassAway() override {
         if (StorageStatsResponse && StorageStatsResponse->IsOk()) {
             for (NKikimrSysView::TStorageStatsEntry& entry : *StorageStatsResponse->Get()->Record.MutableEntries()) {
@@ -1018,20 +1089,26 @@ class TJsonCluster : public TViewerPipeClient {
         if (CachedDataMaxAge) {
             ClusterInfo.SetCachedDataMaxAge(CachedDataMaxAge.MilliSeconds());
         }
-        NKikimrWhiteboard::EFlag worstState = NKikimrWhiteboard::EFlag::Grey;
-        ui64 worstNodes = 0;
-        for (NKikimrWhiteboard::EFlag flag = NKikimrWhiteboard::EFlag::Grey; flag <= NKikimrWhiteboard::EFlag::Red; flag = NKikimrWhiteboard::EFlag(flag + 1)) {
-            auto itNodes = ClusterInfo.GetMapNodeStates().find(NKikimrWhiteboard::EFlag_Name(flag));
-            if (itNodes == ClusterInfo.GetMapNodeStates().end()) {
-                continue;
-            }
-            auto& nodes = itNodes->second;
-            if (nodes > worstNodes / 100) { // only if it's more than 1% of all nodes
-                worstState = flag;
+        NKikimrWhiteboard::EFlag clusterState = NKikimrWhiteboard::EFlag::Grey;
+        if (SelfCheckResult && SelfCheckResult->IsOk()) {
+            clusterState = GetClusterStateFromSelfCheck(SelfCheckResult->Get()->Result);
+        } else if (SelfCheckResultProto && SelfCheckResultProto->IsOk()) {
+            clusterState = GetClusterStateFromSelfCheck(SelfCheckResultProto->Get()->Record);
+        } else {
+            ui64 worstNodes = 0;
+            for (NKikimrWhiteboard::EFlag flag = NKikimrWhiteboard::EFlag::Grey; flag <= NKikimrWhiteboard::EFlag::Red; flag = NKikimrWhiteboard::EFlag(flag + 1)) {
+                auto itNodes = ClusterInfo.GetMapNodeStates().find(NKikimrWhiteboard::EFlag_Name(flag));
+                if (itNodes == ClusterInfo.GetMapNodeStates().end()) {
+                    continue;
+                }
+                auto& nodes = itNodes->second;
+                if (nodes > worstNodes / 100) { // only if it's more than 1% of all nodes
+                    clusterState = flag;
+                }
+                worstNodes += nodes;
             }
-            worstNodes += nodes;
         }
-        ClusterInfo.SetOverall(GetViewerFlag(worstState));
+        ClusterInfo.SetOverall(GetViewerFlag(clusterState));
         TBase::ReplyAndPassAway(GetHTTPOKJSON(ClusterInfo));
     }