mirror of
https://github.com/php/php-src.git
synced 2026-04-04 22:52:40 +02:00
91969e908f
This new implementation uses the new encoding conversion filters. Aside from fewer LOC and (hopefully) improved readability, the differences are as follows: BEHAVIOR CHANGES: - The old implementation used signed arithmetic when operating on the 'convmap'. This meant that results could be surprising when using convmap entries with 1 in the MSB. Further, types like 'int' were used rather than those with a specific bit width, such as 'int32_t'. This meant that results could also depend on the platform width of an 'int'. Now unsigned arithmetic is used, with explicit bit widths. - Similarly, while converting decimal numeric entities, the legacy implementation would ensure that the value never overflowed INT_MAX, and if it did, the entity would be treated as invalid and passed through unconverted. However, that again means that results depend on the platform size of an 'int'. So now, we use a value with explicit bit width (32 bits) to hold the value of a deconverted decimal entity, and ensure that the entity value does not overflow that. Further, because we are using an UNSIGNED 32-bit value rather than a signed one, the ceiling for how large a decimal entity can be is higher now. All of this will probably not affect anyone, since Unicode codepoints above U+10FFFF are invalid anyways. To see the difference, you need to be using a text encoding like UCS-4, which allows huge 'codepoints'. - If it saw something which looked like a hex entity, but turned out not to be a valid numeric entity, the old implementation would sometimes convert the hexadecimal digits a-f to A-F (uppercase). The new implementation passes invalid numeric entities through without performing case conversion. - The old implementation of mb_encode_numericentity was limited in how many decimal/hex digits it could emit. If a text encoding like UCS-4 was in use, where 'codepoints' can have huge values (larger than the valid range stipulated by the Unicode standard), it would not error out on a 'codepoint' whose value was too large for it, but would rather mangle the value and emit a numeric entity which decoded to some other random codepoint. The new implementation is able to emit enough digits to express any value which fits in 32 bits. PERFORMANCE: Based on micro-benchmarks run on my development machine: Decoding numeric HTML entities is about 4 times faster, for both decimal and hexadecimal entities, across a variety of input string lengths. Encoding is about 3 times faster.
183 lines
12 KiB
PHP
183 lines
12 KiB
PHP
--TEST--
|
|
Test mb_decode_numericentity() function : Convert HTML entities to text
|
|
--EXTENSIONS--
|
|
mbstring
|
|
--FILE--
|
|
<?php
|
|
|
|
function varDumpToString($var)
|
|
{
|
|
ob_start();
|
|
var_dump($var);
|
|
return trim(ob_get_clean());
|
|
}
|
|
|
|
function test($desc, $str, $expected, $convmap, $encoding) {
|
|
$result = mb_decode_numericentity($str, $convmap, $encoding);
|
|
echo $desc, ": ", varDumpToString($str), " => ", varDumpToString($result);
|
|
if ($result === $expected)
|
|
echo " (Good)\n";
|
|
else
|
|
echo " (BAD; expected ", varDumpToString($expected), ")\n";
|
|
}
|
|
|
|
function testNonAscii($desc, $str, $expected, $convmap, $encoding) {
|
|
$result = mb_decode_numericentity($str, $convmap, $encoding);
|
|
echo $desc, ": ", bin2hex($str), " => ", bin2hex($result);
|
|
if ($result === $expected)
|
|
echo " (Good)\n";
|
|
else
|
|
echo " (BAD; expected ", bin2hex($expected), ")\n";
|
|
}
|
|
|
|
$str1 = '¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ';
|
|
$str2 = 'ƒΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩαβγδεζηθικλμνξοπρςστυφχψωϑϒϖ•…′″‾⁄℘ℑℜ™ℵ←↑→↓↔↵⇐⇑⇒⇓⇔∀∂∃∅∇∈∉∋∏∑−∗√∝∞∠∧∨∩∪∫∴∼≅≈≠≡≤≥⊂⊃⊄⊆⊇⊕⊗⊥⋅⌈⌉⌊⌋〈〉◊♠♣♥♦';
|
|
$str3 = 'aŒbœcŠdše€fg';
|
|
|
|
// Typical convmap, typical numeric entity-encoded string
|
|
$convmap = array(0x0, 0x2FFFF, 0, 0xFFFF);
|
|
echo "1: " . mb_decode_numericentity($str1, $convmap, "UTF-8") . "\n";
|
|
echo "2: " . mb_decode_numericentity($str2, $convmap, "UTF-8") . "\n";
|
|
echo "3: " . mb_decode_numericentity($str3, $convmap, "UTF-8") . "\n";
|
|
|
|
// Numeric entities which are truncated at end of string
|
|
// We do NOT decode such entities; they can be terminated by any non-digit character, but not by the end of the string
|
|
echo "4: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "5: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "6: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "7: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "8: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "9: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "10: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "11: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
// Try with hex, not just decimal entities
|
|
echo "11b: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "11c: " . mb_decode_numericentity('�', $convmap), "\n";
|
|
echo "11d: " . mb_decode_numericentity('𐀀', $convmap), "\n";
|
|
|
|
// Large decimal entity, converting from non-ASCII input encoding
|
|
echo "12: " . bin2hex(mb_decode_numericentity(mb_convert_encoding('�', 'UCS-4', 'ASCII'), [0, 0x7FFFFFFF, 0, 0x7FFFFFFF], 'UCS-4')), "\n";
|
|
|
|
$convmap = [];
|
|
echo "13: " . mb_decode_numericentity('föo', $convmap, "UTF-8") . "\n";
|
|
|
|
$convmap = array(0x0, 0x2FFFF, 0); // 3 elements
|
|
try {
|
|
echo "14: " . mb_decode_numericentity($str3, $convmap, "UTF-8") . "\n";
|
|
} catch (ValueError $ex) {
|
|
echo "14: " . $ex->getMessage()."\n";
|
|
}
|
|
|
|
echo "15: " . bin2hex(mb_decode_numericentity('�', [0, 1, 0, 0xFFFF], 'UTF-8')) . "\n";
|
|
echo "16: " . bin2hex(mb_decode_numericentity('�', [0, 1, 0, 0xFFFF], 'UTF-8')) . "\n";
|
|
|
|
// Weird convmap
|
|
$convmap = [
|
|
0, 0, 0, 0, // Only one codepoint, empty mask
|
|
100, 50, 0, 0xFFFF // 'End' codepoint is before 'start' codepoint
|
|
];
|
|
echo "17: " . mb_decode_numericentity('föo', $convmap, "UTF-8") . "\n";
|
|
|
|
// Convmap with positive offset
|
|
$convmap = [0, 10, 1000, 0xFFFF];
|
|
echo "18: " . bin2hex(mb_decode_numericentity("ϩϪϫ", $convmap, "UTF-8")) . "\n";
|
|
echo "19: " . bin2hex(mb_decode_numericentity("ϩϪϫ", $convmap, "UTF-8")) . "\n";
|
|
|
|
echo "20: " . mb_decode_numericentity("{a;", [0, 0xFFFF, 0, 0xFFFF]) . "\n";
|
|
|
|
test("10 digits for decimal entity", "A", "A", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
test("More than 10 digits for decimal entity", "¥", "¥", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
|
|
test("8 digits for hex entity", "A", "A", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
test("More than 8 digits for hex entity", "Ł", "Ł", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
|
|
test("Single &", "&", "&", [0, 0xFFFF, 0, 0xFFFF], "ASCII");
|
|
|
|
// We don't allow an entity to come right after a preceding ampersand
|
|
// (This is for compatibility with the legacy behavior of mb_decode_numericentity)
|
|
test("Successive &", "&,", "&,", [0xffe9ade7, 0x6237b6ff, 0xaa597469, 0x612800], 'ASCII');
|
|
|
|
// We don't allow an entity to come right after a preceding &#
|
|
// (Also for compatibility with the legacy behavior of mb_decode_numericentity)
|
|
test("Successive &#", "&#2", "&#2", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
test("Successive &#x", "&#x2", "&#x2", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
|
|
// Don't allow the starting & of an entity to terminate a preceding entity
|
|
// (Also for compatibility with the legacy behavior of mb_decode_numericentity)
|
|
test("Successive A", "AA", "AA", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
|
|
// An entity CAN come right after an entity which is invalid because of being too long
|
|
test("Starting entity immediately after decimal entity which is too long", "�A", "�A", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
test("Starting entity immediately after hex entity which is too long", "�A", "�", [0, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
|
|
// The second entity is not accepted here, because it terminates a preceding entity
|
|
// (To test entities which are as large as possible without being too large, we need to use UCS-4;
|
|
// any other encoding would not allow codepoints that large)
|
|
$ucs4_test1 = mb_convert_encoding("�A", 'UCS-4BE', 'ASCII');
|
|
testNonAscii("Starting entity immediately after valid decimal entity which is just within maximum length", $ucs4_test1, "\x3B\x9A\xCA\x00\x00\x00\x00&\x00\x00\x00#\x00\x00\x006\x00\x00\x005\x00\x00\x00;", [0, 0xFFFFFFFF, 0, 0xFFFFFFFF], 'UCS-4BE');
|
|
$ucs4_test2 = mb_convert_encoding("�A", 'UCS-4BE', 'ASCII');
|
|
testNonAscii("Starting entity immediately after valid hex entity which is just within maximum length", $ucs4_test2, "\x11\x11\x11\x11\x00\x00\x00&\x00\x00\x00#\x00\x00\x006\x00\x00\x005\x00\x00\x00;", [0, 0xFFFFFFFF, 0, 0xFFFFFFFF], 'UCS-4BE');
|
|
|
|
test("Starting entity immediately after invalid decimal entity", "�A", "�A", [0x1, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
test("Starting entity immediately after invalid hex entity", "�A", "�A", [0x1, 0xFFFF, 0, 0xFFFF], 'ASCII');
|
|
|
|
test("Starting entity immediately after too-big decimal entity", "�A", "�A", [0, 0xFFFFFFFF, 0, 0xFFFFFFFF], 'ASCII');
|
|
|
|
// If the numeric entity decodes to 0xFFFFFFFF, that should be passed through
|
|
// Originally, the new implementation of mb_decode_numericentity used -1 as a marker indicating
|
|
// that the entity could not be successfully decoded, so if the entity decoded successfully to
|
|
// 0xFFFFFFFF (-1), it would be treated as an invalid entity
|
|
test("Regression test (entity which decodes to 0xFFFFFFFF)", "", "?", [0xFFFFFF86, 0xFFFFFFFF, 0xF, 0xFC015448], 'HZ');
|
|
|
|
// With the legacy conversion filters, a trailing & could be truncated by mb_decode_numericentity,
|
|
// because some text encodings did not properly invoke the next flush function in the chain
|
|
test("Regression test (truncation of successive & with JIS encoding)", "&&&", "&&&", [0x20FF37FF, 0x7202F569, 0xC4090023, 0xF160], "JIS");
|
|
|
|
// Previously, signed arithmetic was used on convmap entries
|
|
test("Regression test (convmap entries are now treated as unsigned)", ",", "?,", [0x22FFFF11, 0xBF111189, 0x67726511, 0x1161E719], "ASCII");
|
|
|
|
?>
|
|
--EXPECT--
|
|
1: ¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ
|
|
2: ƒΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩαβγδεζηθικλμνξοπρςστυφχψωϑϒϖ•…′″‾⁄℘ℑℜ™ℵ←↑→↓↔↵⇐⇑⇒⇓⇔∀∂∃∅∇∈∉∋∏∑−∗√∝∞∠∧∨∩∪∫∴∼≅≈≠≡≤≥⊂⊃⊄⊆⊇⊕⊗⊥⋅⌈⌉⌊⌋〈〉◊♠♣♥♦
|
|
3: aŒbœcŠdše€fg
|
|
4: �
|
|
5: �
|
|
6: �
|
|
7: �
|
|
8: �
|
|
9: �
|
|
10: �
|
|
11: �
|
|
11b: �
|
|
11c: �
|
|
11d: 𐀀
|
|
12: 00bc614e
|
|
13: föo
|
|
14: mb_decode_numericentity(): Argument #2 ($map) must have a multiple of 4 elements
|
|
15: 00
|
|
16: 00
|
|
17: föo
|
|
18: 010203
|
|
19: 010203
|
|
20: {a;
|
|
10 digits for decimal entity: string(13) "A" => string(1) "A" (Good)
|
|
More than 10 digits for decimal entity: string(14) "¥" => string(14) "¥" (Good)
|
|
8 digits for hex entity: string(12) "A" => string(1) "A" (Good)
|
|
More than 8 digits for hex entity: string(13) "Ł" => string(13) "Ł" (Good)
|
|
Single &: string(1) "&" => string(1) "&" (Good)
|
|
Successive &: string(5) "&," => string(5) "&," (Good)
|
|
Successive &#: string(8) "&#2" => string(8) "&#2" (Good)
|
|
Successive &#x: string(9) "&#x2" => string(9) "&#x2" (Good)
|
|
Successive A: string(9) "AA" => string(6) "AA" (Good)
|
|
Starting entity immediately after decimal entity which is too long: string(18) "�A" => string(14) "�A" (Good)
|
|
Starting entity immediately after hex entity which is too long: string(17) "�A" => string(13) "�" (Good)
|
|
Starting entity immediately after valid decimal entity which is just within maximum length: 000000260000002300000031000000300000003000000030000000300000003000000030000000300000003000000030000000260000002300000036000000350000003b => 3b9aca00000000260000002300000036000000350000003b (Good)
|
|
Starting entity immediately after valid hex entity which is just within maximum length: 0000002600000023000000780000003100000031000000310000003100000031000000310000003100000031000000260000002300000036000000350000003b => 11111111000000260000002300000036000000350000003b (Good)
|
|
Starting entity immediately after invalid decimal entity: string(8) "�A" => string(8) "�A" (Good)
|
|
Starting entity immediately after invalid hex entity: string(9) "�A" => string(9) "�A" (Good)
|
|
Starting entity immediately after too-big decimal entity: string(17) "�A" => string(13) "�A" (Good)
|
|
Regression test (entity which decodes to 0xFFFFFFFF): string(5) "" => string(1) "?" (Good)
|
|
Regression test (truncation of successive & with JIS encoding): string(3) "&&&" => string(3) "&&&" (Good)
|
|
Regression test (convmap entries are now treated as unsigned): string(4) "," => string(2) "?," (Good)
|