roachtest: add post-process step for perf metrics

There are many tests in which had some custom post processing done in
roachperf to emit the metrics.

To add this facility  in the roachtests itself, this change aims to :

1.Adds 2 new user facing structs `HistogramMetric`, that gives the user
access to the histogram metrics generated by the workload and
`PerfMetrics` which user can return with their aggregated metric.
2. Introduce a new `PostProcessPerfMetrics(HistogramMetrics)
PerfMetrics` step in test spec where the test writer can do custom
aggregation and provide the aggregated metric. If the test has not
implemented this step, the runner will use the `DefaultPostProcess`
function.
3. The step creates a new openmetrics file with the name
`aggregated_stats.om` that will have the aggregated metrics.
4. Implement `PostProcessPerfMetrics` in some roachtests that do
post processing in roachperf.

Epic: https://cockroachlabs.atlassian.net/browse/CRDB-41852

Release note: None

pkg/cmd/roachtest/clusterstats/BUILD.bazel

@@ -51,7 +51,6 @@ go_test(
     embedsrcs = ["openmetrics_expected.txt"],
     deps = [
         "//pkg/cmd/roachtest/cluster",
-        "//pkg/cmd/roachtest/registry",
         # Required for generated mocks
         "//pkg/cmd/roachtest/roachtestutil/task",  #keep
         "//pkg/cmd/roachtest/spec",
@@ -63,6 +62,7 @@ go_test(
         "@com_github_prometheus_client_golang//api/prometheus/v1:prometheus",
         "@com_github_prometheus_common//model",
         "@com_github_stretchr_testify//require",
+        "//pkg/cmd/roachtest/registry",
     ],
 )
 

pkg/cmd/roachtest/registry/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
     deps = [
         "//pkg/cmd/roachtest/cluster",
         "//pkg/cmd/roachtest/operation",
+        "//pkg/cmd/roachtest/roachtestutil",
         "//pkg/cmd/roachtest/spec",
         "//pkg/cmd/roachtest/test",
         "//pkg/internal/team",

pkg/cmd/roachtest/registry/test_spec.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
 )
@@ -25,6 +26,17 @@ var LibGEOS = []string{"libgeos", "libgeos_c"}
 // endpoint should use.
 var PrometheusNameSpace = "roachtest"
 
+var DefaultProcessFunction = func(test string, histograms *roachtestutil.HistogramMetric) (roachtestutil.AggregatedPerfMetrics, error) {
+	return roachtestutil.AggregatedPerfMetrics{
+		{
+			Name:           fmt.Sprintf("%s_%s", test, "total_ops_per_s"),
+			Value:          (histograms.TotalCount / histograms.TotalElapsed) * 1000,
+			Unit:           "ops/s",
+			IsHigherBetter: true,
+		},
+	}, nil
+}
+
 // testStats is internally populated based on its previous runs and used for
 // deciding on the current execution approach. This includes decisions like
 // executing the test on spot VM vs ondemand VM.
@@ -169,6 +181,10 @@ type TestSpec struct {
 
 	// stats are populated by test selector based on previous execution data
 	stats *testStats
+
+	// PostProcessPerfMetrics can be used to custom aggregated metrics
+	// from the histogram metrics that are emitted by the roachtest
+	PostProcessPerfMetrics func(string, *roachtestutil.HistogramMetric) (roachtestutil.AggregatedPerfMetrics, error)
 }
 
 // SetStats sets the stats for the test
@@ -184,6 +200,14 @@ func (ts *TestSpec) IsLastFailurePreempt() bool {
 	return ts.stats != nil && ts.stats.LastFailureIsPreempt
 }
 
+func (ts *TestSpec) GetPostProcessWorkloadMetricsFunction() func(string, *roachtestutil.HistogramMetric) (roachtestutil.AggregatedPerfMetrics, error) {
+	if ts.PostProcessPerfMetrics != nil {
+		return ts.PostProcessPerfMetrics
+	}
+
+	return DefaultProcessFunction
+}
+
 // PostValidation is a type of post-validation that runs after a test completes.
 type PostValidation int
 

pkg/cmd/roachtest/roachtestutil/BUILD.bazel

@@ -41,6 +41,8 @@ go_library(
         "//pkg/util/timeutil",
         "//pkg/workload/histogram/exporter",
         "@com_github_cockroachdb_errors//:errors",
+        "@com_github_prometheus_client_model//go",
+        "@com_github_prometheus_common//expfmt",
         "@com_github_stretchr_testify//require",
         "@org_golang_x_exp//maps",
     ],