Fix GH-17223: Memory leak in libxml encoding handling

This was a bug in both libxml and PHP.
We follow up with the same change as done in GNOME/libxml@b3871dd138.

Changing away from `xmlOutputBufferCreateFilenameDefault` is not
possible yet because this is a stable branch and would break BC.

Closes GH-17254.

NEWS

@@ -52,6 +52,9 @@ PHP                                                                        NEWS
 - Iconv:
   . Fixed bug GH-17047 (UAF on iconv filter failure). (nielsdos)
 
+- LibXML:
+  . Fixed bug GH-17223 (Memory leak in libxml encoding handling). (nielsdos)
+
 - MBString:
   . Fixed bug GH-17112 (Macro redefinitions). (nielsdos, cmb)
 

ext/dom/tests/gh17223.phpt

@@ -0,0 +1,12 @@
+--TEST--
+GH-17223 (Memory leak in libxml encoding handling)
+--EXTENSIONS--
+dom
+--FILE--
+<?php
+$doc = new DOMDocument("1.0", "Shift-JIS");
+@$doc->save("%00");
+echo "Done\n";
+?>
+--EXPECT--
+Done

ext/libxml/libxml.c

@@ -590,11 +590,11 @@ php_libxml_output_buffer_create_filename(const char *URI,
 	char *unescaped = NULL;
 
 	if (URI == NULL)
-		return(NULL);
+		goto err;
 
 	if (strstr(URI, "%00")) {
 		php_error_docref(NULL, E_WARNING, "URI must not contain percent-encoded NUL bytes");
-		return NULL;
+		goto err;
 	}
 
 	puri = xmlParseURI(URI);
@@ -615,7 +615,7 @@ php_libxml_output_buffer_create_filename(const char *URI,
 	}
 
 	if (context == NULL) {
-		return(NULL);
+		goto err;
 	}
 
 	/* Allocate the Output buffer front-end. */
@@ -627,6 +627,11 @@ php_libxml_output_buffer_create_filename(const char *URI,
 	}
 
 	return(ret);
+
+err:
+	/* Similarly to __xmlOutputBufferCreateFilename we should also close the encoder on failure. */
+	xmlCharEncCloseFunc(encoder);
+	return NULL;
 }
 
 static void _php_libxml_free_error(void *ptr)