php/archived-php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2026-03-24 00:02:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix corrupted result after recursive tokenization during token_get_all()

Browse Source 
Fixes GH-19507
Closes GH-19547

Co-authored-by: Arnaud Le Blanc <arnaud.lb@gmail.com>
This commit is contained in:
Kuba Werłos
2025-08-21 21:57:18 +02:00
committed by Arnaud Le Blanc
parent 73b98a3858
commit 94d0b41db1
4 changed files with 71 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
NEWS
View File

@@ -54,6 +54,10 @@ PHP NEWS
(Girgias)
. Added support for partitioned cookies. (nielsdos)
- Tokenizer:
. Fixed bug GH-19507 (Corrupted result after recursive tokenization during
token_get_all()). (kubawerlos, nielsdos, Arnaud)
14 Aug 2025, PHP 8.5.0beta1
- Core:

18
Zend/zend_language_scanner.l
View File

@@ -210,6 +210,7 @@ void shutdown_scanner(void)
zend_ptr_stack_destroy(&SCNG(heredoc_label_stack));
SCNG(heredoc_scan_ahead) = 0;
SCNG(on_event) = NULL;
SCNG(on_event_context) = NULL;
}
ZEND_API void zend_save_lexical_state(zend_lex_state *lex_state)
@@ -581,6 +582,8 @@ ZEND_API zend_result open_file_for_scanning(zend_file_handle *file_handle)
zend_set_compiled_filename(compiled_filename);
zend_string_release_ex(compiled_filename, 0);
SCNG(on_event) = NULL;
SCNG(on_event_context) = NULL;
RESET_DOC_COMMENT();
CG(zend_lineno) = 1;
CG(increment_lineno) = 0;
@@ -766,6 +769,8 @@ ZEND_API void zend_prepare_string_for_scanning(zval *str, zend_string *filename)
zend_set_compiled_filename(filename);
CG(zend_lineno) = 1;
CG(increment_lineno) = 0;
SCNG(on_event) = NULL;
SCNG(on_event_context) = NULL;
RESET_DOC_COMMENT();
}
@@ -1636,6 +1641,9 @@ OPTIONAL_WHITESPACE_OR_COMMENTS ({WHITESPACE}|{MULTI_LINE_COMMENT}|{SINGLE_LINE_
<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("integer"){TABS_AND_SPACES}")" {
if (PARSER_MODE()) {
zend_error(E_DEPRECATED, "Non-canonical cast (integer) is deprecated, use the (int) cast instead");
if (PARSER_MODE() && EG(exception)) {
RETURN_TOKEN(T_ERROR);
}
}
RETURN_TOKEN(T_INT_CAST);
}
@@ -1647,6 +1655,9 @@ OPTIONAL_WHITESPACE_OR_COMMENTS ({WHITESPACE}|{MULTI_LINE_COMMENT}|{SINGLE_LINE_
<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("double"){TABS_AND_SPACES}")" {
if (PARSER_MODE()) {
zend_error(E_DEPRECATED, "Non-canonical cast (double) is deprecated, use the (float) cast instead");
if (PARSER_MODE() && EG(exception)) {
RETURN_TOKEN(T_ERROR);
}
}
RETURN_TOKEN(T_DOUBLE_CAST);
}
@@ -1666,6 +1677,9 @@ OPTIONAL_WHITESPACE_OR_COMMENTS ({WHITESPACE}|{MULTI_LINE_COMMENT}|{SINGLE_LINE_
<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("binary"){TABS_AND_SPACES}")" {
if (PARSER_MODE()) {
zend_error(E_DEPRECATED, "Non-canonical cast (binary) is deprecated, use the (string) cast instead");
if (PARSER_MODE() && EG(exception)) {
RETURN_TOKEN(T_ERROR);
}
}
RETURN_TOKEN(T_STRING_CAST);
}
@@ -1685,6 +1699,9 @@ OPTIONAL_WHITESPACE_OR_COMMENTS ({WHITESPACE}|{MULTI_LINE_COMMENT}|{SINGLE_LINE_
<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("boolean"){TABS_AND_SPACES}")" {
if (PARSER_MODE()) {
zend_error(E_DEPRECATED, "Non-canonical cast (boolean) is deprecated, use the (bool) cast instead");
if (PARSER_MODE() && EG(exception)) {
RETURN_TOKEN(T_ERROR);
}
}
RETURN_TOKEN(T_BOOL_CAST);
}
@@ -2741,6 +2758,7 @@ skip_escape_conversion:
SCNG(heredoc_scan_ahead) = 1;
SCNG(heredoc_indentation) = 0;
SCNG(heredoc_indentation_uses_spaces) = 0;
SCNG(on_event_context) = NULL;
LANG_SCNG(on_event) = NULL;
CG(doc_comment) = NULL;

25
ext/tokenizer/tests/gh19507_eval.phpt Normal file
View File

@@ -0,0 +1,25 @@
--TEST--
GH-19507: Corrupted result after recursive tokenization during token_get_all() (error handler with eval)
--EXTENSIONS--
tokenizer
--FILE--
<?php
set_error_handler(function (int $errno, string $msg) {
eval('123;');
echo 'error handler called: ', $msg, "\n";
});
$tokens = PhpToken::tokenize('<?php (double) $x;', TOKEN_PARSE);
foreach ($tokens as $token) {
echo $token->getTokenName(), "\n";
}
?>
--EXPECT--
error handler called: Non-canonical cast (double) is deprecated, use the (float) cast instead
T_OPEN_TAG
T_DOUBLE_CAST
T_WHITESPACE
T_VARIABLE
;

24
ext/tokenizer/tests/gh19507_throw.phpt Normal file
View File

@@ -0,0 +1,24 @@
--TEST--
GH-19507: Corrupted result after recursive tokenization during token_get_all() (error handler with throw)
--EXTENSIONS--
tokenizer
--FILE--
<?php
set_error_handler(function (int $errno, string $msg) {
throw new RuntimeException('error handler called: ' . $msg);
});
$tokens = PhpToken::tokenize('<?php (double) $x;', TOKEN_PARSE);
foreach ($tokens as $token) {
echo $token->getTokenName(), "\n";
}
?>
--EXPECTF--
Fatal error: Uncaught RuntimeException: error handler called: Non-canonical cast (double) is deprecated, use the (float) cast instead in %s:%d
Stack trace:
#0 [internal function]: {closure:%s:%d}(%d, 'Non-canonical c...', '', 1)
#1 %s(%d): PhpToken::tokenize('<?php (double) ...', %d)
#2 {main}
thrown in %s on line %d