Benchmarking mean (#11085)

* Use 50 runs and calculate mean

* Don't validate timestamps

* Don't profile PHP startup and shutdown in cgi with valgrind

.github/workflows/push.yml

@@ -230,6 +230,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install \
             bison \
+            libgmp-dev \
             libonig-dev \
             libsqlite3-dev \
             openssl \
@@ -253,6 +254,7 @@ jobs:
             --enable-werror \
             --prefix=/usr \
             --with-config-file-scan-dir=/etc/php.d \
+            --with-gmp \
             --with-mysqli=mysqlnd \
             --with-openssl \
             --with-pdo-sqlite \

benchmark/benchmark.php

@@ -55,7 +55,7 @@ function runSymfonyDemo(bool $jit): array {
     cloneRepo($dir, 'https://github.com/php/benchmarking-symfony-demo-2.2.3.git');
     runPhpCommand([$dir . '/bin/console', 'cache:clear']);
     runPhpCommand([$dir . '/bin/console', 'cache:warmup']);
-    return runValgrindPhpCgiCommand([$dir . '/public/index.php'], cwd: $dir, jit: $jit, warmup: 50);
+    return runValgrindPhpCgiCommand([$dir . '/public/index.php'], cwd: $dir, jit: $jit, warmup: 50, repeat: 50);
 }
 
 function runWordpress(bool $jit): array {
@@ -78,7 +78,7 @@ function runWordpress(bool $jit): array {
 
     // Warmup
     runPhpCommand([$dir . '/index.php'], $dir);
-    return runValgrindPhpCgiCommand([$dir . '/index.php'], cwd: $dir, jit: $jit, warmup: 50);
+    return runValgrindPhpCgiCommand([$dir . '/index.php'], cwd: $dir, jit: $jit, warmup: 50, repeat: 50);
 }
 
 function runPhpCommand(array $args, ?string $cwd = null): ProcessResult {
@@ -90,6 +90,7 @@ function runValgrindPhpCgiCommand(
     ?string $cwd = null,
     bool $jit = false,
     int $warmup = 0,
+    int $repeat = 1,
 ): array {
     global $phpCgi;
     $process = runCommand([
@@ -99,13 +100,17 @@ function runValgrindPhpCgiCommand(
         '--callgrind-out-file=/dev/null',
         '--',
         $phpCgi,
-        '-T' . ($warmup ? $warmup . ',' : '') . '1',
+        '-T' . ($warmup ? $warmup . ',' : '') . $repeat,
         '-d max_execution_time=0',
         '-d opcache.enable=1',
         '-d opcache.jit_buffer_size=' . ($jit ? '128M' : '0'),
+        '-d opcache.validate_timestamps=0',
         ...$args,
     ]);
     $instructions = extractInstructionsFromValgrindOutput($process->stderr);
+    if ($repeat > 1) {
+        $instructions = gmp_strval(gmp_div_q($instructions, $repeat));
+    }
     return ['instructions' => $instructions];
 }
 

sapi/cgi/cgi_main.c

@@ -2219,6 +2219,8 @@ parent_loop_end:
 #ifdef HAVE_VALGRIND
 							if (warmup_repeats > 0) {
 								CALLGRIND_STOP_INSTRUMENTATION;
+								/* We're not interested in measuring startup */
+								CALLGRIND_ZERO_STATS;
 							}
 #endif
 						} else {
@@ -2427,6 +2429,12 @@ parent_loop_end:
 				}
 			} /* end !cgi && !fastcgi */
 
+#ifdef HAVE_VALGRIND
+			if (warmup_repeats == 0) {
+				CALLGRIND_START_INSTRUMENTATION;
+			}
+#endif
+
 			/* request startup only after we've done all we can to
 			 * get path_translated */
 			if (php_request_startup() == FAILURE) {
@@ -2546,6 +2554,11 @@ fastcgi_request_done:
 				SG(request_info).query_string = NULL;
 			}
 
+#ifdef HAVE_VALGRIND
+			/* We're not interested in measuring shutdown */
+			CALLGRIND_STOP_INSTRUMENTATION;
+#endif
+
 			if (!fastcgi) {
 				if (benchmark) {
 					if (warmup_repeats) {
@@ -2555,9 +2568,6 @@ fastcgi_request_done:
 							gettimeofday(&start, NULL);
 #else
 							time(&start);
-#endif
-#ifdef HAVE_VALGRIND
-							CALLGRIND_START_INSTRUMENTATION;
 #endif
 						}
 						continue;