measure startup, each bench run and shutdown separately

Author: mvorisek

benchmark/benchmark.php

@@ -63,7 +63,8 @@ function runSymfonyDemo(bool $jit): array {
     cloneRepo($dir, 'https://github.com/php/benchmarking-symfony-demo-2.2.3.git');
     runPhpCommand([$dir . '/bin/console', 'cache:clear']);
     runPhpCommand([$dir . '/bin/console', 'cache:warmup']);
-    return runValgrindPhpCgiCommand('symfony-demo', [$dir . '/public/index.php'], cwd: $dir, jit: $jit, warmup: 50, repeat: 50);
+
+    return runValgrindPhpCgiCommand('symfony-demo', [$dir . '/public/index.php'], cwd: $dir, jit: $jit, repeat: 100);
 }
 
 function runWordpress(bool $jit): array {
@@ -86,7 +87,8 @@ function runWordpress(bool $jit): array {
 
     // Warmup
     runPhpCommand([$dir . '/index.php'], $dir);
-    return runValgrindPhpCgiCommand('wordpress', [$dir . '/index.php'], cwd: $dir, jit: $jit, warmup: 50, repeat: 50);
+
+    return runValgrindPhpCgiCommand('wordpress', [$dir . '/index.php'], cwd: $dir, jit: $jit, repeat: 100);
 }
 
 function runPhpCommand(array $args, ?string $cwd = null): ProcessResult {
@@ -98,45 +100,86 @@ function runValgrindPhpCgiCommand(
     array $args,
     ?string $cwd = null,
     bool $jit = false,
-    int $warmup = 0,
     int $repeat = 1,
 ): array {
     global $phpCgi;
 
-    $profileOut = __DIR__ . "/profiles/callgrind.out.$name";
+    $profileOut = __DIR__ . "/profiles/callgrind.$name";
     if ($jit) {
-        $profileOut .= '.jit';
+        $profileOut .= '-jit';
     }
 
     $process = runCommand([
         'valgrind',
         '--tool=callgrind',
         '--dump-instr=yes',
+        '--collect-jumps=yes',
         "--callgrind-out-file=$profileOut",
+        '--verbose=yes',
         '--',
         $phpCgi,
-        '-T' . ($warmup ? $warmup . ',' : '') . $repeat,
+        '-T' . $repeat,
         '-d max_execution_time=0',
         '-d opcache.enable=1',
         '-d opcache.jit=' . ($jit ? 'tracing' : 'disable'),
         '-d opcache.jit_buffer_size=128M',
         '-d opcache.validate_timestamps=0',
         ...$args,
     ]);
-    $valgrindMetrics = extractMetricsFromValgrindOutput($process->stderr);
-    $instructions = $valgrindMetrics['Ir'];
-    if ($repeat > 1) {
-        $instructions = gmp_strval(gmp_div_q($instructions, $repeat));
+
+    // collect metrics for startup, each benchmark run and shutdown
+    $metricsArr = [];
+    foreach (['startup' => 1, ...range(2, $repeat + 1), 'shutdown' => ''] as $k => $kCallgrindOut) {
+        $profileOutSpecific = $profileOut . '.' . $k;
+        rename($profileOut . ($kCallgrindOut === '' ? '' : '.' . $kCallgrindOut), $profileOutSpecific);
+
+        $metricsArr[$k] = extractMetricsFromCallgrindFile($profileOutSpecific);
+    }
+
+    // print all collected metrics
+    print_r($metricsArr);
+
+    // find the fastest benchmark run
+    $bestRunIndex = 0;
+    foreach (range(0, $repeat - 1) as $k) {
+        if ($metricsArr[$k]['Ir'] < $metricsArr[$bestRunIndex]['Ir']) {
+            $bestRunIndex = $k;
+        }
     }
-    return ['instructions' => $instructions];
+
+    // remove non-fastest profiles from artifacts
+    foreach (range(0, $repeat - 1) as $k) {
+        $profileOutSpecific = $profileOut . '.' . $k;
+
+        if ($k !== $bestRunIndex) {
+            unlink($profileOutSpecific);
+        }
+    }
+
+    // annotate profiles for artifacts
+    foreach (['startup', $bestRunIndex, 'shutdown'] as $k) {
+        $profileOutSpecific = $profileOut . '.' . $k;
+
+        runCommand([
+            'callgrind_annotate',
+            '--threshold=100',
+            '--auto=yes',
+            '--show-percs=no',
+            $profileOutSpecific,
+            new UnescapedArg('>'),
+            "$profileOutSpecific.txt",
+        ]);
+    }
+
+    return ['instructions' => $metricsArr[$bestRunIndex]['Ir']];
 }
 
 /**
  * @return array<non-empty-string, numeric-string>
  */
-function extractMetricsFromValgrindOutput(string $output): array {
-    if (!preg_match('/==\d+== Events *:((?: +\w+)+)\n==\d+== Collected :((?: +\d+)+)\n/', $output, $matches)) {
-        throw new \Exception('Unexpected valgrind output: ' . $output);
+function extractMetricsFromCallgrindFile(string $path): array {
+    if (!preg_match('/\nevents:((?: +\w+)+)\nsummary:((?: +\d+)+)\n/', file_get_contents($path, length: 10_000), $matches)) {
+        throw new \Exception('Unexpected callgrind data');
     }
 
     return array_combine(explode(' ', ltrim($matches[1], ' ')), explode(' ', ltrim($matches[2], ' ')));

sapi/cgi/cgi_main.c

@@ -2216,13 +2216,6 @@ consult the installation file that came with this distribution, or visit \n\
 						if (comma) {
 							warmup_repeats = atoi(php_optarg);
 							repeats = atoi(comma + 1);
-#ifdef HAVE_VALGRIND
-							if (warmup_repeats > 0) {
-								CALLGRIND_STOP_INSTRUMENTATION;
-								/* We're not interested in measuring startup */
-								CALLGRIND_ZERO_STATS;
-							}
-#endif
 						} else {
 							repeats = atoi(php_optarg);
 						}
@@ -2264,6 +2257,13 @@ consult the installation file that came with this distribution, or visit \n\
 		}
 #endif
 		while (!fastcgi || fcgi_accept_request(request) >= 0) {
+#ifdef HAVE_VALGRIND
+			if (benchmark) {
+				/* measure startup and each benchmark run separately */
+				CALLGRIND_DUMP_STATS;
+			}
+#endif
+
 			SG(server_context) = fastcgi ? (void *)request : (void *) 1;
 			init_request_info(request);
 
@@ -2430,12 +2430,6 @@ consult the installation file that came with this distribution, or visit \n\
 				}
 			} /* end !cgi && !fastcgi */
 
-#ifdef HAVE_VALGRIND
-			if (warmup_repeats == 0) {
-				CALLGRIND_START_INSTRUMENTATION;
-			}
-#endif
-
 			/* request startup only after we've done all we can to
 			 * get path_translated */
 			if (php_request_startup() == FAILURE) {
@@ -2554,11 +2548,6 @@ consult the installation file that came with this distribution, or visit \n\
 				SG(request_info).query_string = NULL;
 			}
 
-#ifdef HAVE_VALGRIND
-			/* We're not interested in measuring shutdown */
-			CALLGRIND_STOP_INSTRUMENTATION;
-#endif
-
 			if (!fastcgi) {
 				if (benchmark) {
 					if (warmup_repeats) {
@@ -2586,6 +2575,12 @@ consult the installation file that came with this distribution, or visit \n\
 					script_file = NULL;
 					goto do_repeat;
 				}
+
+#ifdef HAVE_VALGRIND
+				/* measure shutdown separately */
+				CALLGRIND_DUMP_STATS;
+#endif
+
 				break;
 			}