Added store-specific statistics to existing raft statistics (weaviate…

…#4689)

* Added store-specific statistics to existing raft statistics

* Update fakeMetaHandler to use any for map values as well

* Improve wording in Store.Stats docstring

* Move raft stats to a nested map to improve readability and update test/docs to reflect change

---------

Co-authored-by: Nate Wilkinson <nathan@weaviate.io>

cluster/store/service.go

@@ -274,7 +274,7 @@ func (s *Service) Remove(ctx context.Context, id string) error {
 	return err
 }
 
-func (s *Service) Stats() map[string]string {
+func (s *Service) Stats() map[string]any {
 	// log.Printf("membership.Stats")
 	return s.store.Stats()
 }

cluster/store/store.go

@@ -416,7 +416,61 @@ func (f *Store) FindSimilarClass(name string) string {
 	return f.db.Schema.ClassEqual(name)
 }
 
-func (st *Store) Stats() map[string]string { return st.raft.Stats() }
+// Stats returns internal statistics from this store, for informational/debugging purposes only.
+//
+// The statistics directly from raft are nested under the "raft" key. If the raft statistics are
+// not yet available, then the "raft" key will not exist.
+// See https://pkg.go.dev/github.com/hashicorp/raft#Raft.Stats for the default raft stats.
+//
+// The values of "leader_address" and "leader_id" are the respective address/ID for the current
+// leader of the cluster. They may be empty strings if there is no current leader or the leader is
+// unknown.
+//
+// The value of "ready" indicates whether this store is ready, see Store.Ready.
+//
+// The value of "is_voter" indicates whether this store is a voter, see Store.IsVoter.
+//
+// The value of "open" indicates whether this store is open, see Store.open.
+//
+// The value of "bootstrapped" indicates whether this store has completed bootstrapping,
+// see Store.bootstrapped.
+//
+// The value of "candidates" is a map[string]string of the current candidates IDs/addresses,
+// see Store.candidates.
+//
+// The value of "initial_last_applied_index" is the index of the last applied command found when
+// the store was opened, see Store.initialLastAppliedIndex.
+//
+// The value of "last_applied_index" is the index of the latest update to the store,
+// see Store.lastAppliedIndex.
+//
+// The value of "db_loaded" indicates whether the DB has finished loading, see Store.dbLoaded.
+//
+// Since this is for information/debugging we want to avoid enforcing unnecessary restrictions on
+// what can go in these stats, thus we're returning map[string]any. However, any values added to
+// this map should be able to be JSON encoded.
+func (st *Store) Stats() map[string]any {
+	stats := make(map[string]any)
+
+	// Add custom stats for this store
+	currentLeaderAddress, currentLeaderID := st.LeaderWithID()
+	stats["leader_address"] = currentLeaderAddress
+	stats["leader_id"] = currentLeaderID
+	stats["ready"] = st.Ready()
+	stats["is_voter"] = st.IsVoter()
+	stats["open"] = st.open.Load()
+	stats["bootstrapped"] = st.bootstrapped.Load()
+	stats["candidates"] = st.candidates
+	stats["initial_last_applied_index"] = st.initialLastAppliedIndex
+	stats["last_applied_index"] = st.lastAppliedIndex.Load()
+	stats["db_loaded"] = st.dbLoaded.Load()
+
+	// If the raft stats exist, add them as a nested map
+	if st.raft != nil {
+		stats["raft"] = st.raft.Stats()
+	}
+	return stats
+}
 
 // Leader is used to return the current leader address.
 // It may return empty strings if there is no current leader or the leader is unknown.

cluster/store/store_test.go

@@ -19,6 +19,8 @@ import (
 	"fmt"
 	"log/slog"
 	"reflect"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -242,7 +244,22 @@ func TestServiceEndpoints(t *testing.T) {
 
 	// Stats
 	stats := srv.Stats()
-	assert.Equal(t, "Leader", stats["state"])
+	// stats:raft_state
+	assert.Equal(t, "Leader", stats["raft"].(map[string]string)["state"])
+	// stats:leader_address
+	leaderAddress := string(stats["leader_address"].(raft.ServerAddress))
+	splitAddress := strings.Split(leaderAddress, ":")
+	assert.Len(t, splitAddress, 2)
+	ipAddress, portStr := splitAddress[0], splitAddress[1]
+	assert.Equal(t, "127.0.0.1", ipAddress)
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		t.Errorf("Port should have been parsable as an int but was: %v", portStr)
+	}
+	assert.GreaterOrEqual(t, port, 0)
+	// stats:leader_id
+	leaderID := string(stats["leader_id"].(raft.ServerID))
+	assert.Equal(t, m.store.nodeID, leaderID)
 
 	// create snapshot
 	assert.Nil(t, srv.store.raft.Barrier(2*time.Second).Error())

usecases/schema/fakes_test.go

@@ -83,8 +83,8 @@ func (f *fakeMetaHandler) Remove(ctx context.Context, nodeID string) error {
 	return args.Error(0)
 }
 
-func (f *fakeMetaHandler) Stats() map[string]string {
-	return map[string]string{}
+func (f *fakeMetaHandler) Stats() map[string]any {
+	return map[string]any{}
 }
 
 func (f *fakeMetaHandler) StoreSchemaV1() error {

usecases/schema/handler.go

@@ -51,7 +51,7 @@ type metaWriter interface {
 	// Cluster related operations
 	Join(_ context.Context, nodeID, raftAddr string, voter bool) error
 	Remove(_ context.Context, nodeID string) error
-	Stats() map[string]string
+	Stats() map[string]any
 	StoreSchemaV1() error
 }
 
@@ -239,7 +239,7 @@ func (h *Handler) RemoveNode(ctx context.Context, node string) error {
 }
 
 // Statistics is used to return a map of various internal stats. This should only be used for informative purposes or debugging.
-func (h *Handler) Statistics() map[string]string {
+func (h *Handler) Statistics() map[string]any {
 	return h.metaWriter.Stats()
 }