From aea57803f02d40b645b0fe876c14f7744ba17449 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Wed, 10 Jun 2020 13:39:57 +0200 Subject: [PATCH 01/13] Add metrics config Added some configs options allowing to configure the metrics package. --- config/config.go | 7 +++++++ configs/config.toml.SAMPLE | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/config/config.go b/config/config.go index 1f4f52b25c..9a6b61e056 100644 --- a/config/config.go +++ b/config/config.go @@ -19,6 +19,7 @@ type Config struct { Ethereum ethereum.Config LibP2P libp2p.Config Storage Storage + Metrics Metrics } // Storage stores meta-info about keeping data on disk @@ -26,6 +27,12 @@ type Storage struct { DataDir string } +// Metrics stores meta-info about metrics. +type Metrics struct { + Identifier string + Port int +} + var ( // KeepOpts contains global application settings KeepOpts Config diff --git a/configs/config.toml.SAMPLE b/configs/config.toml.SAMPLE index 2d73bb6e70..f48930d8d2 100644 --- a/configs/config.toml.SAMPLE +++ b/configs/config.toml.SAMPLE @@ -54,3 +54,7 @@ [Storage] DataDir = "/my/secure/location" + +# [Metrics] + # Identifier = "example-id" + # Port = 8080 From 57c9f68bd1646002ab335b989765060980fc30c4 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Wed, 10 Jun 2020 13:40:39 +0200 Subject: [PATCH 02/13] Add IsConnected method Added IsConnected method to the network provider interface to allow make some network metrics. --- pkg/net/libp2p/libp2p.go | 9 +++++++++ pkg/net/local/local.go | 4 ++++ pkg/net/net.go | 2 ++ 3 files changed, 15 insertions(+) diff --git a/pkg/net/libp2p/libp2p.go b/pkg/net/libp2p/libp2p.go index 2bd3063239..9e1ce2f7e9 100644 --- a/pkg/net/libp2p/libp2p.go +++ b/pkg/net/libp2p/libp2p.go @@ -216,6 +216,15 @@ func (cm *connectionManager) AddrStrings() []string { return multiaddrStrings } +func (cm *connectionManager) IsConnected(address string) bool { + peerInfos, err := extractMultiAddrFromPeers([]string{address}) + if err != nil { + return false + } + + return cm.Network().Connectedness(peerInfos[0].ID) == libp2pnet.Connected +} + func (cm *connectionManager) monitorConnectedPeers(ctx context.Context) { ticker := time.NewTicker(ConnectedPeersCheckTick) defer ticker.Stop() diff --git a/pkg/net/local/local.go b/pkg/net/local/local.go index 2c348a756f..03af06a30d 100644 --- a/pkg/net/local/local.go +++ b/pkg/net/local/local.go @@ -135,3 +135,7 @@ func (lcm *localConnectionManager) DisconnectPeer(connectedPeer string) { func (lcm *localConnectionManager) AddrStrings() []string { return make([]string, 0) } + +func (lcm *localConnectionManager) IsConnected(address string) bool { + panic("not implemented") +} diff --git a/pkg/net/net.go b/pkg/net/net.go index c7b0c997ab..2690746afa 100644 --- a/pkg/net/net.go +++ b/pkg/net/net.go @@ -77,6 +77,8 @@ type ConnectionManager interface { // AddrStrings returns all listen addresses of the provider. AddrStrings() []string + + IsConnected(address string) bool } // TaggedUnmarshaler is an interface that includes the proto.Unmarshaler From 3c8327126185577e836e00a74db6d50876816e65 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Wed, 10 Jun 2020 16:06:42 +0200 Subject: [PATCH 03/13] Bump keep-common version --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9791b8b170..b5bd9b6fd1 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/ipfs/go-datastore v0.1.1 github.com/ipfs/go-log v0.0.1 github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec - github.com/keep-network/keep-common v1.1.0 + github.com/keep-network/keep-common v1.1.1-0.20200610130035-55afd4237caf github.com/libp2p/go-addr-util v0.0.1 github.com/libp2p/go-libp2p v0.4.1 github.com/libp2p/go-libp2p-connmgr v0.1.0 diff --git a/go.sum b/go.sum index 9c5943ce63..d9c26da98e 100644 --- a/go.sum +++ b/go.sum @@ -248,6 +248,10 @@ github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec h github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec/go.mod h1:xR8jf3/VJAjh3nWu5tFe8Yxnt2HvWsqZHfGef1P5oDk= github.com/keep-network/keep-common v1.1.0 h1:m5ZDfUpH+DVqQz3qIi+E53utWHv7kVSooPD01kVG3n8= github.com/keep-network/keep-common v1.1.0/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= +github.com/keep-network/keep-common v1.1.1-0.20200610113255-c9d58a9011ff h1:MhKVpUYCChwjKFoDt4G7aBwX4B+d+lxQYt6keam2OKk= +github.com/keep-network/keep-common v1.1.1-0.20200610113255-c9d58a9011ff/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= +github.com/keep-network/keep-common v1.1.1-0.20200610130035-55afd4237caf h1:bNOzvCwlxnWANqoUzN+FWVKfzToA86EZrfLquERwcUY= +github.com/keep-network/keep-common v1.1.1-0.20200610130035-55afd4237caf/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= From 83c24afa2bfda9b7df04a2cbba6fcb618f2bd118 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Wed, 10 Jun 2020 16:08:54 +0200 Subject: [PATCH 04/13] Add predefined system metrics --- cmd/start.go | 58 ++++++++++++++++++ pkg/metrics/metrics.go | 131 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 pkg/metrics/metrics.go diff --git a/cmd/start.go b/cmd/start.go index 248acb0b59..1db769f0fe 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -5,6 +5,9 @@ import ( "fmt" "time" + "github.com/keep-network/keep-core/pkg/metrics" + "github.com/keep-network/keep-core/pkg/net" + "github.com/ipfs/go-log" "github.com/keep-network/keep-common/pkg/chain/ethereum/ethutil" "github.com/keep-network/keep-common/pkg/persistence" @@ -150,6 +153,11 @@ func Start(c *cli.Context) error { return fmt.Errorf("error initializing beacon: [%v]", err) } + err = initializeMetrics(ctx, config, netProvider, stakeMonitor) + if err != nil { + return fmt.Errorf("error initializing metrics: [%v]", err) + } + select { case <-ctx.Done(): if err != nil { @@ -195,3 +203,53 @@ func waitForStake(stakeMonitor chain.StakeMonitor, address string, timeout int) } return fmt.Errorf("timed out waiting for %s to have required minimum stake", address) } + +func initializeMetrics( + ctx context.Context, + config *config.Config, + netProvider net.Provider, + stakeMonitor chain.StakeMonitor, +) error { + registry, err := metrics.Initialize( + "keep-core", + config.Metrics.Identifier, + config.Metrics.Port, + ) + if err != nil { + logger.Infof("metrics are not configured") + return nil + } + + logger.Infof( + "enabled metrics with identifier [%v] on port [%v]", + config.Metrics.Identifier, + config.Metrics.Port, + ) + + observationTick := 1 * time.Minute + + metrics.ObserveConnectedPeersCount( + ctx, + registry, + netProvider, + observationTick, + ) + + metrics.ObserveConnectedBootstrapPercentage( + ctx, + registry, + netProvider, + config.LibP2P.Peers, + observationTick, + ) + + metrics.ObserveEthConnectivity( + ctx, + registry, + stakeMonitor, + config.Ethereum.Account.Address, + observationTick, + ) + + return nil +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000000..d5edcccef7 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,131 @@ +package metrics + +import ( + "context" + "fmt" + "time" + + "github.com/ipfs/go-log" + "github.com/keep-network/keep-common/pkg/metrics" + "github.com/keep-network/keep-core/pkg/chain" + "github.com/keep-network/keep-core/pkg/net" +) + +var logger = log.Logger("keep-metrics") + +// Initialize set up the metrics registry and enables metrics server. +func Initialize( + application string, + identifier string, + port int, +) (*metrics.Registry, error) { + if application == "" || identifier == "" || port == 0 { + return nil, fmt.Errorf("invalid parameters") + } + + registry := metrics.NewRegistry(application, identifier) + + registry.EnableServer(port) + + return registry, nil +} + +// ObserveConnectedPeersCount triggers an observation process of the +// connected_peers_count metric. +func ObserveConnectedPeersCount( + ctx context.Context, + registry *metrics.Registry, + netProvider net.Provider, + tick time.Duration, +) { + input := func() float64 { + connectedPeers := netProvider.ConnectionManager().ConnectedPeers() + return float64(len(connectedPeers)) + } + + observe( + ctx, + "connected_peers_count", + input, + registry, + tick, + ) +} + +// ObserveConnectedBootstrapPercentage triggers an observation process of the +// connected_bootstrap_percentage metric. +func ObserveConnectedBootstrapPercentage( + ctx context.Context, + registry *metrics.Registry, + netProvider net.Provider, + bootstraps []string, + tick time.Duration, +) { + input := func() float64 { + maxCount := len(bootstraps) + if maxCount == 0 { + return 0 + } + + currentCount := 0 + for _, address := range bootstraps { + if netProvider.ConnectionManager().IsConnected(address) { + currentCount++ + } + } + + return float64((currentCount * 100) / maxCount) + } + + observe( + ctx, + "connected_bootstrap_percentage", + input, + registry, + tick, + ) +} + +// ObserveEthConnectivity triggers an observation process of the +// eth_connectivity metric. +func ObserveEthConnectivity( + ctx context.Context, + registry *metrics.Registry, + stakeMonitor chain.StakeMonitor, + address string, + tick time.Duration, +) { + input := func() float64 { + _, err := stakeMonitor.HasMinimumStake(address) + + if err != nil { + return 0 + } + + return 1 + } + + observe( + ctx, + "eth_connectivity", + input, + registry, + tick, + ) +} + +func observe( + ctx context.Context, + name string, + input metrics.ObserverInput, + registry *metrics.Registry, + tick time.Duration, +) { + observer, err := registry.NewGaugeObserver(name, input) + if err != nil { + logger.Warningf("could not create gauge observer [%v]", name) + return + } + + observer.Observe(ctx, tick) +} From f0f42b38cee53719458c5d6183332bad36d29b4c Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Wed, 10 Jun 2020 16:14:41 +0200 Subject: [PATCH 05/13] Configurable observation tick --- cmd/start.go | 7 ++++++- config/config.go | 1 + configs/config.toml.SAMPLE | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/start.go b/cmd/start.go index 1db769f0fe..acdce7e63c 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -226,7 +226,12 @@ func initializeMetrics( config.Metrics.Port, ) - observationTick := 1 * time.Minute + var observationTick time.Duration + if tick := config.Metrics.Tick; tick != 0 { + observationTick = time.Duration(tick) * time.Second + } else { + observationTick = 60 * time.Second + } metrics.ObserveConnectedPeersCount( ctx, diff --git a/config/config.go b/config/config.go index 9a6b61e056..6ea8c8106a 100644 --- a/config/config.go +++ b/config/config.go @@ -31,6 +31,7 @@ type Storage struct { type Metrics struct { Identifier string Port int + Tick int } var ( diff --git a/configs/config.toml.SAMPLE b/configs/config.toml.SAMPLE index f48930d8d2..e8fc3445c9 100644 --- a/configs/config.toml.SAMPLE +++ b/configs/config.toml.SAMPLE @@ -58,3 +58,4 @@ # [Metrics] # Identifier = "example-id" # Port = 8080 + # Tick = 60 From 982cd268a7febe9e8c42d4939597abf57d014905 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Fri, 12 Jun 2020 13:42:08 +0200 Subject: [PATCH 06/13] Expose libp2p node id metric --- cmd/start.go | 5 +++++ go.mod | 2 +- go.sum | 4 ++++ pkg/metrics/metrics.go | 16 ++++++++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/cmd/start.go b/cmd/start.go index acdce7e63c..2a822df715 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -256,5 +256,10 @@ func initializeMetrics( observationTick, ) + metrics.ExposeLibP2PInfo( + registry, + netProvider, + ) + return nil } diff --git a/go.mod b/go.mod index b5bd9b6fd1..c61c9efd7a 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/ipfs/go-datastore v0.1.1 github.com/ipfs/go-log v0.0.1 github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec - github.com/keep-network/keep-common v1.1.1-0.20200610130035-55afd4237caf + github.com/keep-network/keep-common v1.1.1-0.20200612111801-12829c0d1e0f github.com/libp2p/go-addr-util v0.0.1 github.com/libp2p/go-libp2p v0.4.1 github.com/libp2p/go-libp2p-connmgr v0.1.0 diff --git a/go.sum b/go.sum index d9c26da98e..e1eb49431c 100644 --- a/go.sum +++ b/go.sum @@ -252,6 +252,10 @@ github.com/keep-network/keep-common v1.1.1-0.20200610113255-c9d58a9011ff h1:MhKV github.com/keep-network/keep-common v1.1.1-0.20200610113255-c9d58a9011ff/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/keep-network/keep-common v1.1.1-0.20200610130035-55afd4237caf h1:bNOzvCwlxnWANqoUzN+FWVKfzToA86EZrfLquERwcUY= github.com/keep-network/keep-common v1.1.1-0.20200610130035-55afd4237caf/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= +github.com/keep-network/keep-common v1.1.1-0.20200612104554-481df8250e84 h1:EgCeKoy6zRQEzZWi/z7Z5IQ1/5imb9Un/ucuJNqjCL0= +github.com/keep-network/keep-common v1.1.1-0.20200612104554-481df8250e84/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= +github.com/keep-network/keep-common v1.1.1-0.20200612111801-12829c0d1e0f h1:m0uA5CD9h/aNKTn4Z5r760D/gggBIqFjPSdcE4KGWKA= +github.com/keep-network/keep-common v1.1.1-0.20200612111801-12829c0d1e0f/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index d5edcccef7..737932b4fe 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -129,3 +129,19 @@ func observe( observer.Observe(ctx, tick) } + +// ExposeLibP2PInfo provides some basic information about libp2p config. +func ExposeLibP2PInfo( + registry *metrics.Registry, + netProvider net.Provider, +) { + name := "libp2p_info" + + id := metrics.NewLabel("id", netProvider.ID().String()) + + _, err := registry.NewInfo(name, id) + if err != nil { + logger.Warningf("could not create info metric [%v]", name) + return + } +} From 1e0d6439ee2a5f3970444f1b98bdf0bc52c39b90 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Fri, 12 Jun 2020 15:32:12 +0200 Subject: [PATCH 07/13] Remove metrics identifier --- cmd/start.go | 5 +---- config/config.go | 5 ++--- configs/config.toml.SAMPLE | 1 - go.mod | 2 +- go.sum | 2 ++ pkg/metrics/metrics.go | 10 ++++------ 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 2a822df715..2534389f83 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -211,8 +211,6 @@ func initializeMetrics( stakeMonitor chain.StakeMonitor, ) error { registry, err := metrics.Initialize( - "keep-core", - config.Metrics.Identifier, config.Metrics.Port, ) if err != nil { @@ -221,8 +219,7 @@ func initializeMetrics( } logger.Infof( - "enabled metrics with identifier [%v] on port [%v]", - config.Metrics.Identifier, + "enabled metrics on port [%v]", config.Metrics.Port, ) diff --git a/config/config.go b/config/config.go index 6ea8c8106a..860ffb25e3 100644 --- a/config/config.go +++ b/config/config.go @@ -29,9 +29,8 @@ type Storage struct { // Metrics stores meta-info about metrics. type Metrics struct { - Identifier string - Port int - Tick int + Port int + Tick int } var ( diff --git a/configs/config.toml.SAMPLE b/configs/config.toml.SAMPLE index e8fc3445c9..ae6d54e58b 100644 --- a/configs/config.toml.SAMPLE +++ b/configs/config.toml.SAMPLE @@ -56,6 +56,5 @@ DataDir = "/my/secure/location" # [Metrics] - # Identifier = "example-id" # Port = 8080 # Tick = 60 diff --git a/go.mod b/go.mod index c61c9efd7a..e1e9c2cbe9 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/ipfs/go-datastore v0.1.1 github.com/ipfs/go-log v0.0.1 github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec - github.com/keep-network/keep-common v1.1.1-0.20200612111801-12829c0d1e0f + github.com/keep-network/keep-common v1.1.1-0.20200612121439-e1944b162625 github.com/libp2p/go-addr-util v0.0.1 github.com/libp2p/go-libp2p v0.4.1 github.com/libp2p/go-libp2p-connmgr v0.1.0 diff --git a/go.sum b/go.sum index e1eb49431c..2cd5e4ce05 100644 --- a/go.sum +++ b/go.sum @@ -256,6 +256,8 @@ github.com/keep-network/keep-common v1.1.1-0.20200612104554-481df8250e84 h1:EgCe github.com/keep-network/keep-common v1.1.1-0.20200612104554-481df8250e84/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/keep-network/keep-common v1.1.1-0.20200612111801-12829c0d1e0f h1:m0uA5CD9h/aNKTn4Z5r760D/gggBIqFjPSdcE4KGWKA= github.com/keep-network/keep-common v1.1.1-0.20200612111801-12829c0d1e0f/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= +github.com/keep-network/keep-common v1.1.1-0.20200612121439-e1944b162625 h1:ZDb+ewPufSqbbO49hTPotyO8RMzVODcOPxGWm8xzHeg= +github.com/keep-network/keep-common v1.1.1-0.20200612121439-e1944b162625/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 737932b4fe..13c965a904 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -15,15 +15,13 @@ var logger = log.Logger("keep-metrics") // Initialize set up the metrics registry and enables metrics server. func Initialize( - application string, - identifier string, port int, ) (*metrics.Registry, error) { - if application == "" || identifier == "" || port == 0 { - return nil, fmt.Errorf("invalid parameters") + if port == 0 { + return nil, fmt.Errorf("invalid port") } - registry := metrics.NewRegistry(application, identifier) + registry := metrics.NewRegistry() registry.EnableServer(port) @@ -139,7 +137,7 @@ func ExposeLibP2PInfo( id := metrics.NewLabel("id", netProvider.ID().String()) - _, err := registry.NewInfo(name, id) + _, err := registry.NewInfo(name, []metrics.Label{id}) if err != nil { logger.Warningf("could not create info metric [%v]", name) return From 16350ad60fbe5138485975a50b2bb95c89f0fe3c Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Fri, 3 Jul 2020 10:11:25 +0200 Subject: [PATCH 08/13] Bump keep-core dependency --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e1e9c2cbe9..c3f2993138 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/ipfs/go-datastore v0.1.1 github.com/ipfs/go-log v0.0.1 github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec - github.com/keep-network/keep-common v1.1.1-0.20200612121439-e1944b162625 + github.com/keep-network/keep-common v1.1.1-0.20200701100953-c1c03e75403b github.com/libp2p/go-addr-util v0.0.1 github.com/libp2p/go-libp2p v0.4.1 github.com/libp2p/go-libp2p-connmgr v0.1.0 diff --git a/go.sum b/go.sum index 2cd5e4ce05..44bc6002fd 100644 --- a/go.sum +++ b/go.sum @@ -258,6 +258,8 @@ github.com/keep-network/keep-common v1.1.1-0.20200612111801-12829c0d1e0f h1:m0uA github.com/keep-network/keep-common v1.1.1-0.20200612111801-12829c0d1e0f/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/keep-network/keep-common v1.1.1-0.20200612121439-e1944b162625 h1:ZDb+ewPufSqbbO49hTPotyO8RMzVODcOPxGWm8xzHeg= github.com/keep-network/keep-common v1.1.1-0.20200612121439-e1944b162625/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= +github.com/keep-network/keep-common v1.1.1-0.20200701100953-c1c03e75403b h1:sQfkPbDdCp8zTp486gIG/LrpMJscbJ4CQvsYSuGg81I= +github.com/keep-network/keep-common v1.1.1-0.20200701100953-c1c03e75403b/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= From b7064ba77578b272edadf54c556c2b19e23d22ed Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Fri, 3 Jul 2020 10:12:15 +0200 Subject: [PATCH 09/13] Return bool instead of err from metrics.Initialize --- cmd/start.go | 4 ++-- pkg/metrics/metrics.go | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 2534389f83..0ae4147c73 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -210,10 +210,10 @@ func initializeMetrics( netProvider net.Provider, stakeMonitor chain.StakeMonitor, ) error { - registry, err := metrics.Initialize( + registry, isConfigured := metrics.Initialize( config.Metrics.Port, ) - if err != nil { + if !isConfigured { logger.Infof("metrics are not configured") return nil } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 13c965a904..2f24668307 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -2,7 +2,6 @@ package metrics import ( "context" - "fmt" "time" "github.com/ipfs/go-log" @@ -16,16 +15,16 @@ var logger = log.Logger("keep-metrics") // Initialize set up the metrics registry and enables metrics server. func Initialize( port int, -) (*metrics.Registry, error) { +) (*metrics.Registry, bool) { if port == 0 { - return nil, fmt.Errorf("invalid port") + return nil, false } registry := metrics.NewRegistry() registry.EnableServer(port) - return registry, nil + return registry, true } // ObserveConnectedPeersCount triggers an observation process of the From 0551999cec93215114abf2b9e7f66d6eba6611d5 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Fri, 3 Jul 2020 11:06:24 +0200 Subject: [PATCH 10/13] Rework metric ticks --- cmd/start.go | 13 +++---------- config/config.go | 5 +++-- configs/config.toml.SAMPLE | 3 ++- pkg/metrics/metrics.go | 25 ++++++++++++++++++++++--- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 0ae4147c73..408fe71f45 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -223,18 +223,11 @@ func initializeMetrics( config.Metrics.Port, ) - var observationTick time.Duration - if tick := config.Metrics.Tick; tick != 0 { - observationTick = time.Duration(tick) * time.Second - } else { - observationTick = 60 * time.Second - } - metrics.ObserveConnectedPeersCount( ctx, registry, netProvider, - observationTick, + time.Duration(config.Metrics.NetworkMetricsTick)*time.Second, ) metrics.ObserveConnectedBootstrapPercentage( @@ -242,7 +235,7 @@ func initializeMetrics( registry, netProvider, config.LibP2P.Peers, - observationTick, + time.Duration(config.Metrics.NetworkMetricsTick)*time.Second, ) metrics.ObserveEthConnectivity( @@ -250,7 +243,7 @@ func initializeMetrics( registry, stakeMonitor, config.Ethereum.Account.Address, - observationTick, + time.Duration(config.Metrics.EthereumMetricsTick)*time.Second, ) metrics.ExposeLibP2PInfo( diff --git a/config/config.go b/config/config.go index 860ffb25e3..7291ec8195 100644 --- a/config/config.go +++ b/config/config.go @@ -29,8 +29,9 @@ type Storage struct { // Metrics stores meta-info about metrics. type Metrics struct { - Port int - Tick int + Port int + NetworkMetricsTick int + EthereumMetricsTick int } var ( diff --git a/configs/config.toml.SAMPLE b/configs/config.toml.SAMPLE index ae6d54e58b..99a811914a 100644 --- a/configs/config.toml.SAMPLE +++ b/configs/config.toml.SAMPLE @@ -57,4 +57,5 @@ # [Metrics] # Port = 8080 - # Tick = 60 + # NetworkMetricsTick = 60 + # EthereumMetricsTick = 600 diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 2f24668307..1b913ee6a9 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -2,6 +2,7 @@ package metrics import ( "context" + "fmt" "time" "github.com/ipfs/go-log" @@ -12,6 +13,15 @@ import ( var logger = log.Logger("keep-metrics") +const ( + // DefaultNetworkMetricsTick is the default duration of the + // observation tick for network metrics. + DefaultNetworkMetricsTick = 1 * time.Minute + // DefaultEthereumMetricsTick is the default duration of the + // observation tick for Ethereum metrics. + DefaultEthereumMetricsTick = 10 * time.Minute +) + // Initialize set up the metrics registry and enables metrics server. func Initialize( port int, @@ -45,7 +55,7 @@ func ObserveConnectedPeersCount( "connected_peers_count", input, registry, - tick, + validateTick(tick, DefaultNetworkMetricsTick), ) } @@ -79,7 +89,7 @@ func ObserveConnectedBootstrapPercentage( "connected_bootstrap_percentage", input, registry, - tick, + validateTick(tick, DefaultNetworkMetricsTick), ) } @@ -107,7 +117,7 @@ func ObserveEthConnectivity( "eth_connectivity", input, registry, - tick, + validateTick(tick, DefaultEthereumMetricsTick), ) } @@ -118,6 +128,7 @@ func observe( registry *metrics.Registry, tick time.Duration, ) { + fmt.Printf("observe metric [%v] with tick: [%v]\n", name, tick) observer, err := registry.NewGaugeObserver(name, input) if err != nil { logger.Warningf("could not create gauge observer [%v]", name) @@ -127,6 +138,14 @@ func observe( observer.Observe(ctx, tick) } +func validateTick(tick time.Duration, defaultTick time.Duration) time.Duration { + if tick > 0 { + return tick + } + + return defaultTick +} + // ExposeLibP2PInfo provides some basic information about libp2p config. func ExposeLibP2PInfo( registry *metrics.Registry, From 85c70350746dfe6d5538ee339663124789b31452 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Fri, 3 Jul 2020 11:48:36 +0200 Subject: [PATCH 11/13] Refactor connected_bootstrap_percentage metric --- cmd/start.go | 2 +- pkg/metrics/metrics.go | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 408fe71f45..e72aee810e 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -230,7 +230,7 @@ func initializeMetrics( time.Duration(config.Metrics.NetworkMetricsTick)*time.Second, ) - metrics.ObserveConnectedBootstrapPercentage( + metrics.ObserveConnectedBootstrapCount( ctx, registry, netProvider, diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 1b913ee6a9..5c95ab2f98 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -59,9 +59,9 @@ func ObserveConnectedPeersCount( ) } -// ObserveConnectedBootstrapPercentage triggers an observation process of the -// connected_bootstrap_percentage metric. -func ObserveConnectedBootstrapPercentage( +// ObserveConnectedBootstrapCount triggers an observation process of the +// connected_bootstrap_count metric. +func ObserveConnectedBootstrapCount( ctx context.Context, registry *metrics.Registry, netProvider net.Provider, @@ -69,24 +69,20 @@ func ObserveConnectedBootstrapPercentage( tick time.Duration, ) { input := func() float64 { - maxCount := len(bootstraps) - if maxCount == 0 { - return 0 - } - currentCount := 0 + for _, address := range bootstraps { if netProvider.ConnectionManager().IsConnected(address) { currentCount++ } } - return float64((currentCount * 100) / maxCount) + return float64(currentCount) } observe( ctx, - "connected_bootstrap_percentage", + "connected_bootstrap_count", input, registry, validateTick(tick, DefaultNetworkMetricsTick), From 1cba9299b963d130bfae1c656337f5a20af1cd97 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Fri, 3 Jul 2020 14:35:13 +0200 Subject: [PATCH 12/13] Remove unnecessary printf --- pkg/metrics/metrics.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 5c95ab2f98..71f9c7edf2 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -2,7 +2,6 @@ package metrics import ( "context" - "fmt" "time" "github.com/ipfs/go-log" @@ -124,7 +123,6 @@ func observe( registry *metrics.Registry, tick time.Duration, ) { - fmt.Printf("observe metric [%v] with tick: [%v]\n", name, tick) observer, err := registry.NewGaugeObserver(name, input) if err != nil { logger.Warningf("could not create gauge observer [%v]", name) From 6c5dd769bc46c361e5607c3cb0b2a24fcdbeaa81 Mon Sep 17 00:00:00 2001 From: Lukasz Zimnoch Date: Fri, 3 Jul 2020 14:54:43 +0200 Subject: [PATCH 13/13] Bump keep-common dependency --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c3f2993138..472d127151 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/ipfs/go-datastore v0.1.1 github.com/ipfs/go-log v0.0.1 github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec - github.com/keep-network/keep-common v1.1.1-0.20200701100953-c1c03e75403b + github.com/keep-network/keep-common v1.1.1-0.20200703125023-d9872a19ebd1 github.com/libp2p/go-addr-util v0.0.1 github.com/libp2p/go-libp2p v0.4.1 github.com/libp2p/go-libp2p-connmgr v0.1.0 diff --git a/go.sum b/go.sum index 44bc6002fd..c0ae4d9a99 100644 --- a/go.sum +++ b/go.sum @@ -260,6 +260,8 @@ github.com/keep-network/keep-common v1.1.1-0.20200612121439-e1944b162625 h1:ZDb+ github.com/keep-network/keep-common v1.1.1-0.20200612121439-e1944b162625/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/keep-network/keep-common v1.1.1-0.20200701100953-c1c03e75403b h1:sQfkPbDdCp8zTp486gIG/LrpMJscbJ4CQvsYSuGg81I= github.com/keep-network/keep-common v1.1.1-0.20200701100953-c1c03e75403b/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= +github.com/keep-network/keep-common v1.1.1-0.20200703125023-d9872a19ebd1 h1:SCjStilprtxkLaY6+pw/xqvCJFPEm71n3GJyC+cbLU0= +github.com/keep-network/keep-common v1.1.1-0.20200703125023-d9872a19ebd1/go.mod h1:0uY+hufkP66nFnL+3GXMwWXRiKKsFyXFQTby1xR7AwY= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=