tmpnet: Reuse dynamically-allocated API port across restarts (#2857)

Author: maru-ava

tests/fixture/tmpnet/network.go

@@ -441,6 +441,23 @@ func (n *Network) Restart(ctx context.Context, w io.Writer) error {
 		return err
 	}
 	for _, node := range n.Nodes {
+		// Ensure the node reuses the same API port across restarts to ensure
+		// consistent labeling of metrics. Otherwise prometheus's automatic
+		// addition of the `instance` label (host:port) results in
+		// segmentation of results for a given node every time the port
+		// changes on restart. This segmentation causes graphs on the grafana
+		// dashboards to display multiple series per graph for a given node,
+		// one for each port that the node used.
+		//
+		// There is a non-zero chance of the port being allocatted to a
+		// different process and the node subsequently being unable to start,
+		// but the alternative is having to update the grafana dashboards
+		// query-by-query to ensure that node metrics ignore the instance
+		// label.
+		if err := node.SaveAPIPort(); err != nil {
+			return err
+		}
+
 		if err := node.Stop(ctx); err != nil {
 			return fmt.Errorf("failed to stop node %s: %w", node.NodeID, err)
 		}

tests/fixture/tmpnet/node.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -214,13 +215,14 @@ func (n *Node) Stop(ctx context.Context) error {
 // Sets networking configuration for the node.
 // Convenience method for setting networking flags.
 func (n *Node) SetNetworkingConfig(bootstrapIDs []string, bootstrapIPs []string) {
-	var (
-		// Use dynamic port allocation.
-		httpPort    uint16 = 0
-		stakingPort uint16 = 0
-	)
-	n.Flags[config.HTTPPortKey] = httpPort
-	n.Flags[config.StakingPortKey] = stakingPort
+	if _, ok := n.Flags[config.HTTPPortKey]; !ok {
+		// Default to dynamic port allocation
+		n.Flags[config.HTTPPortKey] = 0
+	}
+	if _, ok := n.Flags[config.StakingPortKey]; !ok {
+		// Default to dynamic port allocation
+		n.Flags[config.StakingPortKey] = 0
+	}
 	n.Flags[config.BootstrapIDsKey] = strings.Join(bootstrapIDs, ",")
 	n.Flags[config.BootstrapIPsKey] = strings.Join(bootstrapIPs, ",")
 }
@@ -348,3 +350,16 @@ func (n *Node) EnsureNodeID() error {
 
 	return nil
 }
+
+// Saves the currently allocated API port to the node's configuration
+// for use across restarts. Reusing the port ensures consistent
+// labeling of metrics.
+func (n *Node) SaveAPIPort() error {
+	hostPort := strings.TrimPrefix(n.URI, "http://")
+	_, port, err := net.SplitHostPort(hostPort)
+	if err != nil {
+		return err
+	}
+	n.Flags[config.HTTPPortKey] = port
+	return nil
+}