output: Fail starting to output buffer when the output layer is deactivated (#20846)

Fixes php/php-src#20837.

NEWS

@@ -2,6 +2,10 @@ PHP                                                                        NEWS
 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
 ?? ??? ????, PHP 8.4.18
 
+- Core:
+  . Fixed bug GH-20837 (NULL dereference when calling ob_start() in shutdown
+    function triggered by bailout in php_output_lock_error()). (timwolla)
+
 - MbString:
   . Fixed bug GH-20833 (mb_str_pad() divide by zero if padding string is
     invalid in the encoding). (ndossche)

main/output.c

@@ -538,6 +538,10 @@ PHPAPI zend_result php_output_handler_start(php_output_handler *handler)
 	HashTable *rconflicts;
 	php_output_handler_conflict_check_t conflict;
 
+	if (!(OG(flags) & PHP_OUTPUT_ACTIVATED)) {
+		return FAILURE;
+	}
+
 	if (php_output_lock_error(PHP_OUTPUT_HANDLER_START) || !handler) {
 		return FAILURE;
 	}

tests/output/gh20352.phpt

@@ -21,4 +21,7 @@ ob_start(new Test, 1);
 echo "trigger bug";
 ?>
 --EXPECTF--
+%r(Notice: ob_start\(\): Failed to create buffer in [^\r\n]+ on line \d+\r?\n(\r?\n)?)+%r
+Notice: ob_start(): Failed to create buffer in %s on line %d
+
 Fatal error: ob_start(): Cannot use output buffering in output buffering display handlers in %s on line %d

tests/output/gh20837.phpt

@@ -0,0 +1,23 @@
+--TEST--
+ob_start(): NULL dereference when calling ob_start() in shutdown function triggered by bailout in php_output_lock_error()
+--FILE--
+<?php
+
+register_shutdown_function(function () {
+    ob_start(function ($input) {
+        echo "bar";
+        return strtoupper($input);
+    });
+});
+
+ob_start(function () {
+    ob_start();
+}, 1);
+
+echo "foo";
+
+?>
+--EXPECTF--
+Fatal error: ob_start(): Cannot use output buffering in output buffering display handlers in %s on line %d
+
+Notice: ob_start(): Failed to create buffer in %s on line %d