Add WritePrometheusServiceDiscoveryConfigFile helper

tests/fixture/tmpnet/monitor_processes.go

@@ -273,6 +273,55 @@ func getServiceDiscoveryDir(cmdName string) (string, error) {
 	return filepath.Join(tmpnetDir, cmdName, "file_sd_configs"), nil
 }
 
+// SDConfig represents a Prometheus service discovery config entry.
+//
+// file_sd_config docs: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config
+type SDConfig struct {
+	Targets []string          `json:"targets"`
+	Labels  map[string]string `json:"labels"`
+}
+
+// WritePrometheusSDConfig writes the SDConfig with the provided name
+// to the location expected by the prometheus instance start by tmpnet.
+//
+// If withGitHubLabels is true, checks env vars for GitHub-specific labels
+// and adds them as labels if present before writing the SDConfig.
+//
+// Returns the path to the written configuration file.
+func WritePrometheusSDConfig(name string, sdConfig SDConfig, withGitHubLabels bool) (string, error) {
+	serviceDiscoveryDir, err := GetPrometheusServiceDiscoveryDir()
+	if err != nil {
+		return "", fmt.Errorf("failed to get service discovery dir: %w", err)
+	}
+
+	if err := os.MkdirAll(serviceDiscoveryDir, perms.ReadWriteExecute); err != nil {
+		return "", fmt.Errorf("failed to create service discovery dir: %w", err)
+	}
+
+	if withGitHubLabels {
+		sdConfig = applyGitHubLabels(sdConfig)
+	}
+
+	configPath := filepath.Join(serviceDiscoveryDir, name+".json")
+	configData, err := DefaultJSONMarshal([]SDConfig{sdConfig})
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal config: %w", err)
+	}
+
+	if err := os.WriteFile(configPath, configData, perms.ReadWrite); err != nil {
+		return "", fmt.Errorf("failed to write config file: %w", err)
+	}
+
+	return configPath, nil
+}
+
+func applyGitHubLabels(sdConfig SDConfig) SDConfig {
+	for label, value := range GetGitHubLabels() {
+		sdConfig.Labels[label] = value
+	}
+	return sdConfig
+}
+
 func getLogFilename(cmdName string) string {
 	return cmdName + ".log"
 }

tests/fixture/tmpnet/network.go

@@ -925,7 +925,15 @@ func (n *Network) GetMonitoringLabels() map[string]string {
 		"is_ephemeral_node": "false",
 		"network_owner":     n.Owner,
 	}
-	// Include the values of github labels if available
+	for label, value := range GetGitHubLabels() {
+		labels[label] = value
+	}
+	return labels
+}
+
+// GetGitHubLabels returns a map of GitHub labels and their values if available.
+func GetGitHubLabels() map[string]string {
+	labels := map[string]string{}
 	for _, label := range githubLabels {
 		value := os.Getenv(strings.ToUpper(label))
 		if len(value) > 0 {

tests/load/c/main/main.go

@@ -69,22 +69,18 @@ func main() {
 	metrics, err := load.NewMetrics(registry)
 	require.NoError(err, "failed to register load metrics")
 
-	metricsServer := load.NewPrometheusServer("127.0.0.1:0", registry)
-	merticsErrCh, err := metricsServer.Start()
+	metricsServer, err := tests.NewPrometheusServer(registry)
 	require.NoError(err, "failed to start load metrics server")
-
-	monitoringConfigFilePath, err := metricsServer.GenerateMonitoringConfig(log, network.GetMonitoringLabels())
-	require.NoError(err, "failed to generate monitoring config file")
-
 	tc.DeferCleanup(func() {
-		select {
-		case err := <-merticsErrCh:
-			require.NoError(err, "metrics server exited with error")
-		default:
-			require.NoError(metricsServer.Stop(), "failed to stop metrics server")
-		}
+		require.NoError(metricsServer.Stop())
 	})
 
+	monitoringConfigFilePath, err := tmpnet.WritePrometheusSDConfig("load-test", tmpnet.SDConfig{
+		Targets: []string{metricsServer.Address()},
+		Labels:  network.GetMonitoringLabels(),
+	}, false)
+	require.NoError(err, "failed to generate monitoring config file")
+
 	tc.DeferCleanup(func() {
 		require.NoError(
 			os.Remove(monitoringConfigFilePath),

tests/load2/main/main.go

@@ -15,7 +15,6 @@ import (
 	"github.com/ava-labs/avalanchego/tests"
 	"github.com/ava-labs/avalanchego/tests/fixture/e2e"
 	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
-	"github.com/ava-labs/avalanchego/tests/load"
 	"github.com/ava-labs/avalanchego/tests/load2"
 )
 
@@ -74,28 +73,23 @@ func main() {
 	require.NoError(err)
 
 	registry := prometheus.NewRegistry()
-	metricsServer := load.NewPrometheusServer("127.0.0.1:0", registry)
-	metricsErrChan, err := metricsServer.Start()
+	metricsServer, err := tests.NewPrometheusServer(registry)
 	require.NoError(err)
-
 	tc.DeferCleanup(func() {
-		select {
-		case err := <-metricsErrChan:
-			require.NoError(err)
-		default:
-		}
-
-		require.NoError(metricsServer.Stop(), "failed to stop metrics server")
+		require.NoError(metricsServer.Stop())
 	})
 
-	monitoringConfigFilePath, err := metricsServer.GenerateMonitoringConfig(
-		log,
-		network.GetMonitoringLabels(),
-	)
-	require.NoError(err)
+	monitoringConfigFilePath, err := tmpnet.WritePrometheusSDConfig("load-test", tmpnet.SDConfig{
+		Targets: []string{metricsServer.Address()},
+		Labels:  network.GetMonitoringLabels(),
+	}, false)
+	require.NoError(err, "failed to generate monitoring config file")
 
 	tc.DeferCleanup(func() {
-		require.NoError(os.Remove(monitoringConfigFilePath))
+		require.NoError(
+			os.Remove(monitoringConfigFilePath),
+			"failed †o remove monitoring config file",
+		)
 	})
 
 	workers := make([]load2.Worker, len(keys))

tests/prometheus_server.go

@@ -0,0 +1,87 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package tests
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+const defaultPrometheusListenAddr = "127.0.0.1:0"
+
+// PrometheusServer is a HTTP server that serves Prometheus metrics from the provided
+// gahterer.
+// Listens on localhost with a dynamic port and serves metrics at /ext/metrics.
+type PrometheusServer struct {
+	gatherer prometheus.Gatherer
+	server   http.Server
+	errChan  chan error
+}
+
+// NewPrometheusServer creates and starts a Prometheus server with the provided gatherer
+// listening on 127.0.0.1:0 and serving /ext/metrics.
+func NewPrometheusServer(gatherer prometheus.Gatherer) (*PrometheusServer, error) {
+	server := &PrometheusServer{
+		gatherer: gatherer,
+	}
+
+	if err := server.start(); err != nil {
+		return nil, err
+	}
+
+	return server, nil
+}
+
+// start the Prometheus server on a dynamic port.
+func (s *PrometheusServer) start() error {
+	mux := http.NewServeMux()
+	mux.Handle("/ext/metrics", promhttp.HandlerFor(s.gatherer, promhttp.HandlerOpts{}))
+
+	listener, err := net.Listen("tcp", defaultPrometheusListenAddr)
+	if err != nil {
+		return err
+	}
+
+	s.server = http.Server{
+		Addr:              listener.Addr().String(),
+		Handler:           mux,
+		ReadHeaderTimeout: time.Second,
+		ReadTimeout:       time.Second,
+	}
+
+	s.errChan = make(chan error, 1)
+	go func() {
+		err := s.server.Serve(listener)
+		if !errors.Is(err, http.ErrServerClosed) {
+			s.errChan <- err
+		}
+		close(s.errChan)
+	}()
+
+	return nil
+}
+
+// Stop gracefully shuts down the Prometheus server.
+// Waits for the server to shut down and returns any error that occurred during shutdown.
+func (s *PrometheusServer) Stop() error {
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	return errors.Join(
+		s.server.Shutdown(shutdownCtx),
+		<-s.errChan,
+	)
+}
+
+// Address returns the address the server is listening on.
+// If the server has not started, the address will be empty.
+func (s *PrometheusServer) Address() string {
+	return s.server.Addr
+}