mirror of
https://github.com/php/php-src.git
synced 2026-04-04 22:52:40 +02:00
a789088527
When testing the preceding commits, I used a script to generate a large number of random strings and try to find strings which would yield different outputs from the new and old encoding conversion code. Some were found. In most cases, analysis revealed that the new code was correct and the old code was not. In all cases where the new code was incorrect, regression tests were added. However, there may be some value in adding regression tests for cases where the old code was incorrect as well. That is done here. This does not cover every case where the new and old code yielded different results. Some of them were very obscure, and it is proving difficult even to reproduce them (since I did not keep a record of all the input strings which triggered the differing output).
236 lines
8.3 KiB
PHP
236 lines
8.3 KiB
PHP
--TEST--
|
|
Test of ASCII and JIS X 0201/0208/0212 support in ISO-2022-JP and JIS7/8 encodings
|
|
--EXTENSIONS--
|
|
mbstring
|
|
--SKIPIF--
|
|
<?php
|
|
if (getenv("SKIP_SLOW_TESTS")) die("skip slow test");
|
|
?>
|
|
--FILE--
|
|
<?php
|
|
include('encoding_tests.inc');
|
|
mb_substitute_character(0x25); // '%'
|
|
|
|
/* Read in table of all characters in JISX-0212 charset */
|
|
readConversionTable(__DIR__ . '/data/JISX0212.txt', $jisx0212Chars, $unused);
|
|
|
|
/* Read in table of all characters in JISX-0208 charset */
|
|
$jisx0208Chars = array(); /* JISX0208 -> UTF-16BE */
|
|
$fp = fopen(__DIR__ . '/data/JISX0208.txt', 'r+');
|
|
while ($line = fgets($fp, 256)) {
|
|
if ($line[0] == '#')
|
|
continue;
|
|
|
|
if (sscanf($line, "0x%x\t0x%x\t0x%x", $shiftJIS, $jis0208Code, $unicodeCP) == 3) {
|
|
$jisx0208Chars[pack('n', $jis0208Code)] = pack('n', $unicodeCP);
|
|
}
|
|
}
|
|
|
|
/* Read in table of all characters in JISX-0201 charset */
|
|
readConversionTable(__DIR__ . '/data/JISX0201.txt', $jisx0201Chars, $unused);
|
|
|
|
/* The JIS X 0208 character set does not have a single, straightforward
|
|
* mapping to the Unicode character set
|
|
* mbstring converts one character differently from the mappings in
|
|
* ../docs/JISX0208.txt, which comes from the Unicode Consortium */
|
|
|
|
/* 0x2140 is a backslash; this can be mapped to 0x005C for an ordinary
|
|
* backslash, or 0xFF3C for a _fullwidth_ one */
|
|
$jisx0208Chars["\x21\x40"] = "\xFF\x3C";
|
|
|
|
function testValid($from, $to, $encoding, $bothWays = true) {
|
|
identifyValidString($from, $encoding);
|
|
convertValidString($from, $to, $encoding, 'UTF-16BE', false);
|
|
|
|
if ($bothWays) {
|
|
/* An 0xF at the beginning of a JIS7 string is redundant; it switches
|
|
* to ASCII mode, but ASCII mode is default */
|
|
if ($from[0] == "\x0F")
|
|
$from = substr($from, 1, strlen($from) - 1);
|
|
/* ESC ( B at the beginning is redundant, since ASCII mode is the default */
|
|
if (substr($from, 0, 3) == "\x1B(B")
|
|
$from = substr($from, 3, strlen($from) - 3);
|
|
/* If the string switches to a different charset, it should switch back to
|
|
* ASCII at the end */
|
|
if (strpos($from, "\x1B\$B") !== false || strpos($from, "\x1B(J") !== false)
|
|
$from .= "\x1B(B";
|
|
|
|
convertValidString($to, $from, 'UTF-16BE', $encoding, false);
|
|
}
|
|
}
|
|
|
|
function testInvalid($from, $to, $encoding) {
|
|
testInvalidString($from, $to, $encoding, 'UTF-16BE');
|
|
}
|
|
|
|
for ($i = 0; $i < 0x80; $i++) {
|
|
if ($i == 0xE || $i == 0xF || $i == 0x1B)
|
|
continue;
|
|
testValid(chr($i), "\x00" . chr($i), 'JIS');
|
|
testValid("\x0F" . chr($i), "\x00" . chr($i), 'JIS'); /* 0xF is 'Shift Out' code */
|
|
testValid("\x1B(B" . chr($i), "\x00" . chr($i), 'JIS');
|
|
testValid(chr($i), "\x00" . chr($i), 'ISO-2022-JP');
|
|
testValid("\x1B(B" . chr($i), "\x00" . chr($i), 'ISO-2022-JP');
|
|
}
|
|
|
|
for ($i = 0x80; $i < 256; $i++) {
|
|
if ($i >= 0xA1 && $i <= 0xDF) // We convert single bytes from 0xA1-0xDF as JIS X 0201 kana
|
|
continue;
|
|
testInvalid(chr($i), "\x00%", 'JIS');
|
|
testInvalid("\x0F" . chr($i), "\x00%", 'JIS');
|
|
testInvalid("\x1B(B" . chr($i), "\x00%", 'JIS');
|
|
testInvalid(chr($i), "\x00%", 'ISO-2022-JP');
|
|
testInvalid("\x1B(B" . chr($i), "\x00%", 'ISO-2022-JP');
|
|
}
|
|
|
|
echo "ASCII support OK\n";
|
|
|
|
/* All valid JIS X 0201 characters
|
|
* Those with a 1 in the high bit are JIS X 0201 kana; JIS7 encodes those
|
|
* with a 0 in the high bit and treats them as a separate charset
|
|
* (We don't test ISO-2022-JP here, as it does not support the JIS X 0201 charset) */
|
|
foreach ($jisx0201Chars as $jisx0201 => $utf16BE) {
|
|
if (ord($jisx0201) >= 128) {
|
|
$kana = chr(ord($jisx0201) - 128);
|
|
testValid("\x1B(I" . $kana, $utf16BE, 'JIS', false);
|
|
testValid("\x0E" . $kana, $utf16BE, 'JIS', false); /* 0xE is 'Shift In' code */
|
|
testValid($jisx0201, $utf16BE, 'JIS', false);
|
|
} else {
|
|
testValid("\x1B(J" . $jisx0201, $utf16BE, 'JIS', $utf16BE > "\x00\x80");
|
|
}
|
|
}
|
|
|
|
for ($i = 0x80; $i < 256; $i++) {
|
|
if ($i >= 0xA1 && $i <= 0xDF)
|
|
continue;
|
|
testInvalid("\x1B(I" . chr($i), "\x00%", 'JIS');
|
|
testInvalid("\x1B(J" . chr($i), "\x00%", 'JIS');
|
|
}
|
|
|
|
echo "JIS X 0201 support OK\n";
|
|
|
|
/* All valid JISX0208 characters */
|
|
foreach ($jisx0208Chars as $jisx0208 => $utf16BE) {
|
|
testValid("\x1B\$B" . $jisx0208, $utf16BE, 'JIS');
|
|
testValid("\x1B\$B" . $jisx0208, $utf16BE, 'ISO-2022-JP');
|
|
}
|
|
|
|
/* All invalid 2-byte JISX0208 characters */
|
|
for ($i = 0x21; $i <= 0x7E; $i++) {
|
|
for ($j = 0; $j < 256; $j++) {
|
|
$testString = chr($i) . chr($j);
|
|
if (!isset($jisx0208Chars[$testString])) {
|
|
testInvalid("\x1B\$B" . $testString, "\x00%", 'JIS');
|
|
testInvalid("\x1B\$B" . $testString, "\x00%", 'ISO-2022-JP');
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Try truncated JISX0208 characters */
|
|
for ($i = 0x21; $i <= 0x7E; $i++) {
|
|
testInvalid("\x1B\$B" . chr($i), "\x00%", 'JIS');
|
|
testInvalid("\x1B\$B" . chr($i), "\x00%", 'ISO-2022-JP');
|
|
}
|
|
|
|
/* Switch from Kanji to ASCII */
|
|
testValidString("\x30\x00\x00A", "\x1B\$B\x21\x21\x1B(BA", "UTF-16BE", "JIS", false);
|
|
testValidString("\x30\x00\x00A", "\x1B\$B\x21\x21\x1B(BA", "UTF-16BE", "ISO-2022-JP", false);
|
|
|
|
echo "JIS X 0208 support OK\n";
|
|
|
|
/* JIS7 supports escape to switch to JIS X 0212 charset, but ISO-2022-JP does not */
|
|
|
|
/* All valid JISX0212 characters */
|
|
foreach ($jisx0212Chars as $jisx0212 => $utf16BE) {
|
|
testValid("\x1B\$(D" . $jisx0212, $utf16BE, 'JIS', false);
|
|
}
|
|
|
|
/* All invalid 2-byte JISX0212 characters */
|
|
for ($i = 0x21; $i <= 0x7E; $i++) {
|
|
for ($j = 0; $j < 256; $j++) {
|
|
$testString = chr($i) . chr($j);
|
|
if (!isset($jisx0212Chars[$testString])) {
|
|
testInvalid("\x1B\$(D" . $testString, "\x00%", 'JIS');
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Try truncated JISX0212 characters */
|
|
for ($i = 0x21; $i <= 0x7E; $i++) {
|
|
testInvalid("\x1B\$(D" . chr($i), "\x00%", 'JIS');
|
|
}
|
|
|
|
testValidString("\x00\xA1", "\x1B\$(D\x22\x42\x1B(B", "UTF-16BE", "JIS", false);
|
|
// Check that ISO-2022-JP treats JISX 0212 chars as error
|
|
convertInvalidString("\x00\xA1", "%", "UTF-16BE", "ISO-2022-JP", false);
|
|
|
|
echo "JIS X 0212 support OK\n";
|
|
|
|
/* All possible escape sequences */
|
|
$validEscapes = ["\x1B\$@" => true, "\x1B\$B" => true, "\x1B\$(@" => true, "\x1B\$(B" => true, "\x1B\$(D" => true, "\x1B(B" => true, "\x1B(H" => true, "\x1B(J" => true, "\x1B(I" => true];
|
|
for ($i = 0; $i <= 0xFF; $i++) {
|
|
for ($j = 0; $j <= 0xFF; $j++) {
|
|
$escapeSequence = "\x1B" . chr($i) . chr($j);
|
|
if ($escapeSequence === "\x1B\$(")
|
|
continue;
|
|
if (isset($validEscapes[$escapeSequence])) {
|
|
testValid($escapeSequence, "", 'JIS', false);
|
|
testValid($escapeSequence, "", 'ISO-2022-JP', false);
|
|
} else {
|
|
identifyInvalidString($escapeSequence, 'JIS');
|
|
identifyInvalidString($escapeSequence, 'ISO-2022-JP');
|
|
}
|
|
}
|
|
}
|
|
for ($i = 0; $i <= 0xFF; $i++) {
|
|
$escapeSequence = "\x1B\$(" . chr($i);
|
|
if (isset($validEscapes[$escapeSequence])) {
|
|
testValid($escapeSequence, "", 'JIS', false);
|
|
testValid($escapeSequence, "", 'ISO-2022-JP', false);
|
|
} else {
|
|
identifyInvalidString($escapeSequence, 'JIS');
|
|
identifyInvalidString($escapeSequence, 'ISO-2022-JP');
|
|
}
|
|
}
|
|
/* Also try a bare ESC */
|
|
identifyInvalidString("\x1B", 'JIS');
|
|
identifyInvalidString("\x1B", 'ISO-2022-JP');
|
|
|
|
echo "All escape sequences work as expected\n";
|
|
|
|
foreach (['JIS', 'ISO-2022-JP'] as $encoding) {
|
|
testValidString("\x22\x25", "\x1B\$B!B\x1B(B", 'UTF-16BE', $encoding, false);
|
|
testValidString("\xFF\x0D", "\x1B\$B!]\x1B(B", 'UTF-16BE', $encoding, false);
|
|
testValidString("\xFF\xE0", "\x1B\$B!q\x1B(B", 'UTF-16BE', $encoding, false);
|
|
testValidString("\xFF\xE1", "\x1B\$B!r\x1B(B", 'UTF-16BE', $encoding, false);
|
|
testValidString("\xFF\xE2", "\x1B\$B\"L\x1B(B", 'UTF-16BE', $encoding, false);
|
|
|
|
testValidString("\x00\xA5", "\x1B(J\x5C\x1B(B", 'UTF-16BE', $encoding, false);
|
|
}
|
|
testValidString("\x20\x3E", "\x1B\$B!1\x1B(B", 'UTF-16BE', 'ISO-2022-JP', false);
|
|
|
|
echo "Other mappings from Unicode -> ISO-2022-JP are OK\n";
|
|
|
|
convertInvalidString("\xFF\xFE", "%", "UTF-16BE", "JIS", false);
|
|
convertInvalidString("\xFF\xFE", "%", "UTF-16BE", "ISO-2022-JP", false);
|
|
|
|
// Test "long" illegal character markers
|
|
mb_substitute_character("long");
|
|
convertInvalidString("\xE0", "%", "JIS", "UTF-8");
|
|
convertInvalidString("\xE0", "%", "ISO-2022-JP", "UTF-8");
|
|
convertInvalidString("\x1B\$(X", "%\$(X", "JIS", "UTF-8"); // Invalid escape
|
|
convertInvalidString("\x1B\$(X", "%\$(X", "ISO-2022-JP", "UTF-8"); // Invalid escape
|
|
convertInvalidString("\x1B\$B!", "%", "JIS", "UTF-8"); // Truncated character
|
|
convertInvalidString("\x1B\$B!", "%", "ISO-2022-JP", "UTF-8"); // Truncated character
|
|
|
|
echo "Done!\n";
|
|
?>
|
|
--EXPECT--
|
|
ASCII support OK
|
|
JIS X 0201 support OK
|
|
JIS X 0208 support OK
|
|
JIS X 0212 support OK
|
|
All escape sequences work as expected
|
|
Other mappings from Unicode -> ISO-2022-JP are OK
|
|
Done!
|