PGO benchmarks: Only run a representative subset of benchmark tests.

Speeds up the PGO profile pipeline.

PiperOrigin-RevId: 784392458

Author: EtiennePerot

Makefile

@@ -591,7 +591,7 @@ benchmark-refresh-pgo: load-benchmarks $(RUNTIME_BIN) ## Refresh profiles of all
 				mkdir -p "$$(dirname "$${PGO_PROFILE_OLD}")"; \
 				mkdir -p "$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}"; \
 				$(call install_runtime,$${PLATFORM}_$${PGO_RUNTIME_KEY}_pgo_$${PGO_BENCHMARK_BASENAME},--platform $${PLATFORM} --profile --profile-cpu="$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}/$${PGO_BENCHMARK_BASENAME}.%YYYY%-%MM%-%DD%_%HH%-%II%-%SS%-%NN%.pgo.pprof.pb.gz"); \
-				$(call sudo,$${PGO_BENCHMARK_TARGET},-runtime=$${PLATFORM}_$${PGO_RUNTIME_KEY}_pgo_$${PGO_BENCHMARK_BASENAME} $(BENCHMARKS_ARGS_PGO)); \
+				$(call sudo,$${PGO_BENCHMARK_TARGET},-runtime=$${PLATFORM}_$${PGO_RUNTIME_KEY}_pgo_$${PGO_BENCHMARK_BASENAME} -pgo-benchmarks=true $(BENCHMARKS_ARGS_PGO)); \
 				$(call run,tools/profiletool,merge --out="$${PGO_PROFILE_NEW}" "$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}"); \
 				rm -rf --one-file-system "$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}"; \
 				if [[ ! -f "$${PGO_PROFILE_OLD}" ]]; then \

pkg/test/dockerutil/dockerutil.go

@@ -46,6 +46,10 @@ var (
 	// config is the default Docker daemon configuration path.
 	config = flag.String("config_path", "/etc/docker/daemon.json", "configuration file for reading paths")
 
+	// pgoBenchmarks is a flag that is set during PGO benchmarks.
+	// This flag restricts the set of benchmark targets to run.
+	pgoBenchmarks = flag.Bool("pgo-benchmarks", false, "set to true for PGO benchmarks; restricts the set of benchmark targets to run")
+
 	// The following flags are for the "pprof" profiler tool.
 
 	// pprofBaseDir allows the user to change the directory to which profiles are
@@ -173,6 +177,13 @@ func IsGVisorRuntime(ctx context.Context, t *testing.T) (bool, error) {
 	return strings.Contains(output, "gVisor"), nil
 }
 
+// SkipIfPGO skips the test if it is a PGO benchmark run.
+func SkipIfPGO[TB testing.TB](tb TB) {
+	if *pgoBenchmarks {
+		tb.Skip("Skipped as this is a PGO benchmark run.")
+	}
+}
+
 // UsingSystemdCgroup returns true if the docker configuration has the
 // native.cgroupdriver=systemd option set in "exec-opts", or if the
 // system is using cgroupv2, in which case systemd is the default driver.

test/benchmarks/database/redis_test.go

@@ -52,6 +52,7 @@ var operations []string = []string{
 // BenchmarkAllRedisOperations runs redis-benchmark against a redis instance and reports
 // data in queries per second. Each is reported by named operation (e.g. LPUSH).
 func BenchmarkAllRedisOperations(b *testing.B) {
+	dockerutil.SkipIfPGO(b)
 	doBenchmarkRedis(b, operations)
 }
 

test/benchmarks/fs/fsbench/fsbench.go

@@ -43,6 +43,8 @@ type FSBenchmark struct {
 	WantOutput string
 	// CleanCmd, if set, is run to clean up between benchmarks.
 	CleanCmd []string
+	// EnableForPGO is whether this benchmark should be run for PGO benchmarks.
+	EnableForPGO bool
 	// Variants is a list of benchmark variants to run.
 	// If unset, the typical set is used.
 	Variants []Variant
@@ -56,6 +58,8 @@ type FSBenchmark struct {
 // Dimensions here are clean/dirty cache (do or don't drop caches)
 // and if the mount on which we are compiling is a tmpfs/bind mount.
 type Variant struct {
+	// EnableForPGO is whether this variant should be run for PGO benchmarks.
+	EnableForPGO bool
 	// clearCache drops caches before running.
 	clearCache bool
 	// fsType is the type of filesystem to use.
@@ -67,8 +71,9 @@ func TypicalVariants() []Variant {
 	variants := make([]Variant, 0, 8)
 	for _, filesys := range []harness.FileSystemType{harness.BindFS, harness.TmpFS, harness.RootFS, harness.FuseFS} {
 		variants = append(variants, Variant{
-			clearCache: true,
-			fsType:     filesys,
+			EnableForPGO: true,
+			clearCache:   true,
+			fsType:       filesys,
 		})
 		variants = append(variants, Variant{
 			clearCache: false,
@@ -81,6 +86,9 @@ func TypicalVariants() []Variant {
 // RunWithDifferentFilesystems runs a
 func RunWithDifferentFilesystems(ctx context.Context, b *testing.B, machine harness.Machine, bm FSBenchmark) {
 	b.Helper()
+	if !bm.EnableForPGO {
+		dockerutil.SkipIfPGO(b)
+	}
 
 	benchmarkVariants := bm.Variants
 	if len(benchmarkVariants) == 0 {
@@ -105,6 +113,9 @@ func RunWithDifferentFilesystems(ctx context.Context, b *testing.B, machine harn
 		}
 
 		b.Run(name, func(b *testing.B) {
+			if !variant.EnableForPGO {
+				dockerutil.SkipIfPGO(b)
+			}
 			// Grab a container.
 			container := machine.GetContainer(ctx, b)
 			cu := cleanup.Make(func() {

test/benchmarks/fs/rubydev_test.go

@@ -57,10 +57,11 @@ func BenchmarkRubyNoOpTest(b *testing.B) {
 // https://github.com/fastlane/fastlane
 func BenchmarkRubySpecTest(b *testing.B) {
 	runRubyBenchmark(b, fsbench.FSBenchmark{
-		Image:      "benchmarks/rubydev",
-		WorkDir:    "/fastlane",
-		RunCmd:     []string{"bash", "/files/run_fastlane_tests.sh"},
-		WantOutput: "3613 examples, 0 failures",
+		Image:        "benchmarks/rubydev",
+		WorkDir:      "/fastlane",
+		RunCmd:       []string{"bash", "/files/run_fastlane_tests.sh"},
+		EnableForPGO: true,
+		WantOutput:   "3613 examples, 0 failures",
 		Callback: func(b *testing.B, output string) {
 			loadTime, err := tools.ExtractRubyLoadTime(output)
 			if err != nil {

test/benchmarks/network/iperf_test.go

@@ -28,6 +28,7 @@ import (
 )
 
 func BenchmarkIperfOneConnection(b *testing.B) {
+	dockerutil.SkipIfPGO(b)
 	clientMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine: %v", err)
@@ -120,11 +121,12 @@ func BenchmarkIperfManyConnections(b *testing.B) {
 	defer serverMachine.CleanUp()
 	ctx := context.Background()
 	for _, bm := range []struct {
-		name       string
-		length     int
-		parallel   int
-		clientFunc func(context.Context, testutil.Logger) *dockerutil.Container
-		serverFunc func(context.Context, testutil.Logger) *dockerutil.Container
+		name         string
+		enableForPGO bool
+		length       int
+		parallel     int
+		clientFunc   func(context.Context, testutil.Logger) *dockerutil.Container
+		serverFunc   func(context.Context, testutil.Logger) *dockerutil.Container
 	}{
 		// We are either measuring the server or the client. The other should be
 		// runc. e.g. Upload sees how fast the runtime under test uploads to a native
@@ -142,16 +144,18 @@ func BenchmarkIperfManyConnections(b *testing.B) {
 			serverFunc: serverMachine.GetContainer,
 		},
 		{
-			name:       "Upload",
-			parallel:   16,
-			clientFunc: clientMachine.GetContainer,
-			serverFunc: serverMachine.GetNativeContainer,
+			name:         "Upload",
+			parallel:     16,
+			enableForPGO: true,
+			clientFunc:   clientMachine.GetContainer,
+			serverFunc:   serverMachine.GetNativeContainer,
 		},
 		{
-			name:       "Download",
-			parallel:   16,
-			clientFunc: clientMachine.GetNativeContainer,
-			serverFunc: serverMachine.GetContainer,
+			name:         "Download",
+			parallel:     16,
+			enableForPGO: true,
+			clientFunc:   clientMachine.GetNativeContainer,
+			serverFunc:   serverMachine.GetContainer,
 		},
 		{
 			name:       "Upload",
@@ -177,6 +181,9 @@ func BenchmarkIperfManyConnections(b *testing.B) {
 			b.Fatalf("Failed to parse parameters: %v", err)
 		}
 		b.Run(name, func(b *testing.B) {
+			if !bm.enableForPGO {
+				dockerutil.SkipIfPGO(b)
+			}
 			// Set up the containers.
 			server := bm.serverFunc(ctx, b)
 			defer server.CleanUp(ctx)