php/archived-php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2026-03-24 00:02:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

TAILCALL VM

Browse Source 
Introduce the TAILCALL VM, a more efficient variant of the CALL VM:

 * Each opcode handler tailcalls the next opcode handler directly instead of
   returning to the interpreter loop. This eliminates call and interpreter loop
   overhead.
 * Opcode handlers use the preserve_none calling convention to eliminate
   register saving overhead.
 * preserve_none uses non-volatile registers for its first arguments, so
   execute_data and opline are usually kept in these registers and no code is
   required to forward them to the next handlers.

Generated machine code is similar to a direct-threaded VM with register pinning,
like the HYBRID VM.

JIT+TAILCALL VM also benefits from this compared to JIT+CALL VM:

 * JIT uses the registers of the execute_data and opline args as fixed regs,
   eliminating the need to move them in prologue.
 * Traces exit by tailcalling the next handler. No code is needed to forward
   execute_data and opline.
 * No register saving/restoring in epilogue/prologue.

The TAILCALL VM is used when the HYBRID VM is not supported, and the compiler
supports the musttail and preserve_none attributes: The HYBRID VM is used when
compiling with GCC, the TAILCALL VM when compiling with Clang>=19 on x86_64 or
aarch64, and the CALL VM otherwise.

This makes binaries built with Clang>=19 as fast as binaries built with GCC.
Before, these were considerably slower (by 2.8% to 44% depending on benchmark,
and by 5% to 77% before 76d7c616bb).

Closes GH-17849
Closes GH-18720
This commit is contained in:
Arnaud Le Blanc
2025-02-13 19:22:10 +01:00
parent c6f02a4ee3
commit 73b98a3858
15 changed files with 60177 additions and 1459 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
NEWS
View File

@@ -19,6 +19,8 @@ PHP NEWS
(alexandre-daubois)
. Fixed bug GH-19544 (GC treats ZEND_WEAKREF_TAG_MAP references as WeakMap
references). (Arnaud, timwolla)
. Introduced the TAILCALL VM, enabled by default when compiling with Clang>=19
on x86_64 or aarch64. (Arnaud)
- Filter:
. Added support for configuring the URI parser for FILTER_VALIDATE_URL

5
UPGRADING
View File

@@ -883,6 +883,11 @@ PHP 8.5 UPGRADE NOTES
. Creating exception objects is now much faster.
. The parts of the code that used SSE2 have been adapted to use SIMD
with ARM NEON as well.
. Introduced the TAILCALL VM, enabled by default when compiling with Clang>=19
on x86_64 or aarch64. The TAILCALL VM is as fast as the HYBRID VM used when
compiling with GCC. This makes PHP binaries built with Clang>=19 as fast as
binaries built with GCC. The performance of the CALL VM, used with other
compilers, has also improved considerably.
- Intl:
. Now avoids creating extra string copies when converting strings

101
Zend/Zend.m4
View File

@@ -170,6 +170,7 @@ ZEND_CHECK_STACK_DIRECTION
ZEND_CHECK_FLOAT_PRECISION
ZEND_DLSYM_CHECK
ZEND_CHECK_GLOBAL_REGISTER_VARIABLES
ZEND_CHECK_PRESERVE_NONE
ZEND_CHECK_CPUID_COUNT
AC_MSG_CHECKING([whether to enable thread safety])
@@ -464,3 +465,103 @@ AS_VAR_IF([ZEND_MAX_EXECUTION_TIMERS], [yes],
AC_MSG_CHECKING([whether to enable Zend max execution timers])
AC_MSG_RESULT([$ZEND_MAX_EXECUTION_TIMERS])
])
dnl
dnl ZEND_CHECK_PRESERVE_NONE
dnl
dnl Check if the preserve_none calling convention is supported and matches our
dnl expectations.
dnl
AC_DEFUN([ZEND_CHECK_PRESERVE_NONE], [dnl
AC_CACHE_CHECK([for preserve_none calling convention],
[php_cv_preverve_none],
[AC_RUN_IFELSE([AC_LANG_SOURCE([[
#include <stdio.h>
#include <stdint.h>
const char * const1 = "str1";
const char * const2 = "str2";
const char * const3 = "str3";
uint64_t key = UINT64_C(0x9d7f71d2bd296364);
uintptr_t _a = 0;
uintptr_t _b = 0;
uintptr_t __attribute__((preserve_none)) fun(uintptr_t a, uintptr_t b) {
_a = a;
_b = b;
return (uintptr_t)const3;
}
uintptr_t __attribute__((preserve_none)) test(void) {
uintptr_t ret;
#if defined(__x86_64__)
__asm__ __volatile__(
/* XORing to make it unlikely the value exists in any other register */
"movq %1, %%r12\n"
"xorq %3, %%r12\n"
"movq %2, %%r13\n"
"xorq %3, %%r13\n"
"xorq %%rax, %%rax\n"
"call fun\n"
: "=a" (ret)
: "r" (const1), "r" (const2), "r" (key)
: "r12", "r13"
);
#elif defined(__aarch64__)
__asm__ __volatile__(
/* XORing to make it unlikely the value exists in any other register */
"eor x20, %1, %3\n"
"eor x21, %2, %3\n"
"eor x0, x0, x0\n"
"bl fun\n"
"mov %0, x0\n"
: "=r" (ret)
: "r" (const1), "r" (const2), "r" (key)
: "x0", "x21", "x22", "x30"
);
#else
# error
#endif
return ret;
}
int main(void) {
/* JIT is making the following expectations about preserve_none:
* - The registers used for integer args 1 and 2
* - The register used for a single integer return value
*
* We check these expectations here:
*/
uintptr_t ret = test();
if (_a != ((uintptr_t)const1 ^ key)) {
fprintf(stderr, "arg1 mismatch\n");
return 1;
}
if (_b != ((uintptr_t)const2 ^ key)) {
fprintf(stderr, "arg2 mismatch\n");
return 2;
}
if (ret != (uintptr_t)const3) {
fprintf(stderr, "ret mismatch\n");
return 3;
}
fprintf(stderr, "OK\n");
return 0;
}]])],
[php_cv_preserve_none=yes],
[php_cv_preserve_none=no],
[php_cv_preserve_none=no])
])
AS_VAR_IF([php_cv_preserve_none], [yes], [
AC_DEFINE([HAVE_PRESERVE_NONE], [1],
[Define to 1 if you have preserve_none support.])
])
])

9
Zend/zend_portability.h
View File

@@ -321,6 +321,15 @@ char *alloca();
# define ZEND_FASTCALL
#endif
#ifdef HAVE_PRESERVE_NONE
# define ZEND_PRESERVE_NONE __attribute__((preserve_none))
#endif
#if __has_attribute(musttail)
# define HAVE_MUSTTAIL
# define ZEND_MUSTTAIL __attribute__((musttail))
#endif
#if (defined(__GNUC__) && __GNUC__ >= 3 && !defined(__INTEL_COMPILER) && !defined(__APPLE__) && !defined(__hpux) && !defined(_AIX) && !defined(__osf__)) || __has_attribute(noreturn)
# define HAVE_NORETURN
# define ZEND_NORETURN __attribute__((noreturn))

7
Zend/zend_vm_def.h
View File

@@ -2962,7 +2962,10 @@ ZEND_VM_HOT_HELPER(zend_leave_helper, ANY, ANY)
{
zend_execute_data *old_execute_data;
uint32_t call_info = EX_CALL_INFO();
#if ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL
/* zend_leave_helper may be called with opline=call_leave_op in TAILCALL VM */
SAVE_OPLINE();
#endif
if (EXPECTED((call_info & (ZEND_CALL_CODE|ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) == 0)) {
EG(current_execute_data) = EX(prev_execute_data);
@@ -4917,7 +4920,7 @@ ZEND_VM_HOT_SEND_HANDLER(116, ZEND_SEND_VAL_EX, CONST|TMP, CONST|UNUSED|NUM, SPE
ZEND_VM_C_GOTO(send_val_by_ref);
}
} else if (ARG_MUST_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
ZEND_VM_C_LABEL(send_val_by_ref):
ZEND_VM_C_LABEL(send_val_by_ref):;
ZEND_VM_DISPATCH_TO_HELPER(zend_cannot_pass_by_ref_helper, _arg_num, arg_num, _arg, arg);
}
value = GET_OP1_ZVAL_PTR(BP_VAR_R);
@@ -6162,7 +6165,7 @@ ZEND_VM_HANDLER(181, ZEND_FETCH_CLASS_CONSTANT, VAR|CONST|UNUSED|CLASS_FETCH, CO
}
bool is_constant_deprecated = ZEND_CLASS_CONST_FLAGS(c) & ZEND_ACC_DEPRECATED;
if (UNEXPECTED(is_constant_deprecated) && !CONST_IS_RECURSIVE(c)) {
if (UNEXPECTED(is_constant_deprecated) && !CONST_IS_RECURSIVE(c)) {
if (c->ce->type == ZEND_USER_CLASS) {
/* Recursion protection only applied to user constants, GH-18463 */
CONST_PROTECT_RECURSION(c);

60713
Zend/zend_vm_execute.h generated
View File

File diff suppressed because it is too large Load Diff

6
Zend/zend_vm_execute.skl
View File

@@ -136,9 +136,7 @@ ZEND_API void ZEND_FASTCALL zend_deserialize_opcode_handler(zend_op *op)
ZEND_API const void* ZEND_FASTCALL zend_get_opcode_handler_func(const zend_op *op)
{
#if ZEND_VM_KIND == ZEND_VM_KIND_CALL
return op->handler;
#elif ZEND_VM_KIND == ZEND_VM_KIND_HYBRID
#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
zval *zv;
if (!zend_handlers_table) {
@@ -147,6 +145,8 @@ ZEND_API const void* ZEND_FASTCALL zend_get_opcode_handler_func(const zend_op *o
zv = zend_hash_index_find(zend_handlers_table, (zend_long)(uintptr_t)op->handler);
ZEND_ASSERT(zv != NULL);
return zend_opcode_handler_funcs[Z_LVAL_P(zv)];
#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL
return op->handler;
#else
return NULL;
#endif

370
Zend/zend_vm_gen.php
View File

@@ -49,10 +49,11 @@ DATA;
error_reporting(E_ALL);
const ZEND_VM_KIND_CALL = 1;
const ZEND_VM_KIND_SWITCH = 2;
const ZEND_VM_KIND_GOTO = 3;
const ZEND_VM_KIND_HYBRID = 4;
const ZEND_VM_KIND_CALL = 1;
const ZEND_VM_KIND_SWITCH = 2;
const ZEND_VM_KIND_GOTO = 3;
const ZEND_VM_KIND_HYBRID = 4;
const ZEND_VM_KIND_TAILCALL = 5;
$vm_op_flags = array(
"ZEND_VM_OP_SPEC" => 1<<0,
@@ -574,7 +575,7 @@ function is_hot_helper($name) {
}
// Returns name of specialized helper
function helper_name($name, $spec, $op1, $op2, $extra_spec) {
function helper_name($name, $spec, $op1, $op2, $extra_spec, $kind) {
global $prefix, $helpers;
$extra = "";
@@ -616,13 +617,15 @@ function helper_name($name, $spec, $op1, $op2, $extra_spec) {
}
}
return $name . ($spec ? "_SPEC" : "") . $prefix[$op1] . $prefix[$op2] . $extra;
$variant = $kind === ZEND_VM_KIND_TAILCALL ? '_TAILCALL' : '';
return $name . ($spec ? "_SPEC" : "") . $prefix[$op1] . $prefix[$op2] . $extra . $variant;
}
function opcode_name($name, $spec, $op1, $op2, $extra_spec) {
function opcode_name($name, $spec, $op1, $op2, $extra_spec, $kind) {
global $prefix, $opnames, $opcodes;
$extra = "";
$variant = $kind === ZEND_VM_KIND_TAILCALL ? '_TAILCALL' : '';
if (isset($opnames[$name])) {
$opcode = $opcodes[$opnames[$name]];
@@ -642,7 +645,7 @@ function opcode_name($name, $spec, $op1, $op2, $extra_spec) {
$op1 = "ANY";
} else if ($spec) {
/* dispatch to invalid handler from unreachable code */
return "ZEND_NULL";
return "ZEND_NULL$variant";
}
}
if (!isset($opcode["op2"][$op2])) {
@@ -659,7 +662,7 @@ function opcode_name($name, $spec, $op1, $op2, $extra_spec) {
$op2 = "ANY";
} else if ($spec) {
/* dispatch to unknown handler in unreachable code */
return "ZEND_NULL";
return "ZEND_NULL$variant";
}
}
/* forward common specs (e.g. in ZEND_VM_DISPATCH_TO_HANDLER) */
@@ -668,7 +671,7 @@ function opcode_name($name, $spec, $op1, $op2, $extra_spec) {
}
}
return $name . ($spec ? "_SPEC" : "") . $prefix[$op1] . $prefix[$op2] . $extra;
return $name . ($spec ? "_SPEC" : "") . $prefix[$op1] . $prefix[$op2] . $extra . $variant;
}
// Formats condition, protecting it by parentheses when needed.
@@ -732,8 +735,8 @@ function gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec=null)
"/FREE_OP2_IF_VAR\(\)/" => $op2_free_op_if_var[$op2],
"/\!ZEND_VM_SPEC/m" => ($op1!="ANY"||$op2!="ANY"||$extra_spec)?"0":"1",
"/ZEND_VM_SPEC/m" => ($op1!="ANY"||$op2!="ANY"||$extra_spec)?"1":"0",
"/ZEND_VM_C_LABEL\(\s*([A-Za-z_]*)\s*\)/m" => "\\1".(($spec && $kind != ZEND_VM_KIND_CALL)?("_SPEC".$prefix[$op1].$prefix[$op2].extra_spec_name($extra_spec)):""),
"/ZEND_VM_C_GOTO\(\s*([A-Za-z_]*)\s*\)/m" => "goto \\1".(($spec && $kind != ZEND_VM_KIND_CALL)?("_SPEC".$prefix[$op1].$prefix[$op2].extra_spec_name($extra_spec)):""),
"/ZEND_VM_C_LABEL\(\s*([A-Za-z_]*)\s*\)/m" => "\\1".(($spec && $kind != ZEND_VM_KIND_CALL && $kind != ZEND_VM_KIND_TAILCALL)?("_SPEC".$prefix[$op1].$prefix[$op2].extra_spec_name($extra_spec)):""),
"/ZEND_VM_C_GOTO\(\s*([A-Za-z_]*)\s*\)/m" => "goto \\1".(($spec && $kind != ZEND_VM_KIND_CALL && $kind != ZEND_VM_KIND_TAILCALL)?("_SPEC".$prefix[$op1].$prefix[$op2].extra_spec_name($extra_spec)):""),
"/^#(\s*)if\s+1\s*\\|\\|.*[^\\\\]$/m" => "#\\1if 1",
"/^#(\s*)if\s+0\s*&&.*[^\\\\]$/m" => "#\\1if 0",
"/^#(\s*)elif\s+1\s*\\|\\|.*[^\\\\]$/m" => "#\\1elif 1",
@@ -796,7 +799,7 @@ function gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec=null)
"/ZEND_VM_DISPATCH_TO_HANDLER\(\s*([A-Z_]*)\s*\)/m",
"/ZEND_VM_DISPATCH_TO_HELPER\(\s*([A-Za-z_]*)\s*(,[^)]*)?\)/m",
),
function($matches) use ($spec, $prefix, $op1, $op2, $extra_spec) {
function($matches) use ($spec, $prefix, $op1, $op2, $extra_spec, $kind) {
if (strncasecmp($matches[0], "EXECUTE_DATA", strlen("EXECUTE_DATA")) == 0) {
return "execute_data";
} else if (strncasecmp($matches[0], "ZEND_VM_DISPATCH_TO_HANDLER", strlen("ZEND_VM_DISPATCH_TO_HANDLER")) == 0) {
@@ -804,35 +807,36 @@ function gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec=null)
$name = $matches[1];
$opcode = $opcodes[$opnames[$name]];
return "goto " . opcode_name($name, $spec, $op1, $op2, $extra_spec) . "_LABEL";
return "goto " . opcode_name($name, $spec, $op1, $op2, $extra_spec, $kind) . "_LABEL";
} else {
// ZEND_VM_DISPATCH_TO_HELPER
if (is_hot_helper($matches[1])) {
if (isset($matches[2])) {
// extra args
$args = preg_replace("/,\s*([A-Za-z0-9_]*)\s*,\s*([^,)\s]*)\s*/", "$1 = $2; ", $matches[2]);
return $args . "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec) . "_LABEL";
return $args . "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . "_LABEL";
}
return "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec) . "_LABEL";
return "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . "_LABEL";
}
if (isset($matches[2])) {
// extra args
$args = substr(preg_replace("/,\s*[A-Za-z0-9_]*\s*,\s*([^,)\s]*)\s*/", ", $1", $matches[2]), 2);
return "ZEND_VM_TAIL_CALL(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec) . "(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX " . $args . "))";
return "ZEND_VM_TAIL_CALL(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . "(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX " . $args . "))";
}
return "ZEND_VM_TAIL_CALL(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec) . "(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU))";
return "ZEND_VM_TAIL_CALL(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . "(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU))";
}
},
$code);
break;
case ZEND_VM_KIND_CALL:
case ZEND_VM_KIND_TAILCALL:
$code = preg_replace_callback(
array(
"/EXECUTE_DATA(?=[^_])/m",
"/ZEND_VM_DISPATCH_TO_HANDLER\(\s*([A-Z_]*)\s*\)/m",
"/ZEND_VM_DISPATCH_TO_HELPER\(\s*([A-Za-z_]*)\s*(,[^)]*)?\)/m",
),
function($matches) use ($spec, $prefix, $op1, $op2, $extra_spec, $name) {
function($matches) use ($spec, $prefix, $op1, $op2, $extra_spec, $name, $kind) {
if (strncasecmp($matches[0], "EXECUTE_DATA", strlen("EXECUTE_DATA")) == 0) {
return "execute_data";
} else if (strncasecmp($matches[0], "ZEND_VM_DISPATCH_TO_HANDLER", strlen("ZEND_VM_DISPATCH_TO_HANDLER")) == 0) {
@@ -846,15 +850,19 @@ function gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec=null)
is_hot_handler($opcode["hot"], $op1, $op2, $extra_spec) &&
is_hot_handler($opcodes[$opnames[$name]]["hot"], $op1, $op2, $extra_spec) ?
"_INLINE" : "";
return "ZEND_VM_TAIL_CALL(" . opcode_name($handler, $spec, $op1, $op2, $extra_spec) . $inline . "_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU))";
return "ZEND_VM_TAIL_CALL(" . opcode_name($handler, $spec, $op1, $op2, $extra_spec, $kind) . $inline . "_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU))";
} else {
// ZEND_VM_DISPATCH_TO_HELPER
if (isset($matches[2])) {
// extra args
$args = substr(preg_replace("/,\s*[A-Za-z0-9_]*\s*,\s*([^,)\s]*)\s*/", ", $1", $matches[2]), 2);
return "ZEND_VM_TAIL_CALL(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec) . "(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX " . $args . "))";
return "ZEND_VM_DISPATCH_TO_HELPER(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . "(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX " . $args . "))";
}
if ($kind === ZEND_VM_KIND_TAILCALL && $matches[1] === 'zend_leave_helper') {
return "ZEND_VM_DISPATCH_TO_LEAVE_HELPER(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . ")";
} else {
return "ZEND_VM_TAIL_CALL(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . "(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU))";
}
return "ZEND_VM_TAIL_CALL(" . helper_name($matches[1], $spec, $op1, $op2, $extra_spec) . "(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU))";
}
},
$code);
@@ -866,19 +874,19 @@ function gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec=null)
"/ZEND_VM_DISPATCH_TO_HANDLER\(\s*([A-Z_]*)\s*\)/m",
"/ZEND_VM_DISPATCH_TO_HELPER\(\s*([A-Za-z_]*)\s*(,[^)]*)?\)/m",
),
function($matches) use ($spec, $prefix, $op1, $op2, $extra_spec) {
function($matches) use ($spec, $prefix, $op1, $op2, $extra_spec, $kind) {
if (strncasecmp($matches[0], "EXECUTE_DATA", strlen("EXECUTE_DATA")) == 0) {
return "execute_data";
} else if (strncasecmp($matches[0], "ZEND_VM_DISPATCH_TO_HANDLER", strlen("ZEND_VM_DISPATCH_TO_HANDLER")) == 0) {
return "goto " . opcode_name($matches[1], $spec, $op1, $op2, $extra_spec) . "_LABEL";
return "goto " . opcode_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . "_LABEL";
} else {
// ZEND_VM_DISPATCH_TO_HELPER
if (isset($matches[2])) {
// extra args
$args = preg_replace("/,\s*([A-Za-z0-9_]*)\s*,\s*([^,)\s]*)\s*/", "$1 = $2; ", $matches[2]);
return $args . "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec);
return $args . "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind);
}
return "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec);
return "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind);
}
},
$code);
@@ -890,19 +898,19 @@ function gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec=null)
"/ZEND_VM_DISPATCH_TO_HANDLER\(\s*([A-Z_]*)\s*\)/m",
"/ZEND_VM_DISPATCH_TO_HELPER\(\s*([A-Za-z_]*)\s*(,[^)]*)?\)/m",
),
function($matches) use ($spec, $prefix, $op1, $op2, $extra_spec) {
function($matches) use ($spec, $prefix, $op1, $op2, $extra_spec, $kind) {
if (strncasecmp($matches[0], "EXECUTE_DATA", strlen("EXECUTE_DATA")) == 0) {
return "execute_data";
} else if (strncasecmp($matches[0], "ZEND_VM_DISPATCH_TO_HANDLER", strlen("ZEND_VM_DISPATCH_TO_HANDLER")) == 0) {
return "goto " . opcode_name($matches[1], $spec, $op1, $op2, $extra_spec) . "_LABEL";
return "goto " . opcode_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind) . "_LABEL";
} else {
// ZEND_VM_DISPATCH_TO_HELPER
if (isset($matches[2])) {
// extra args
$args = preg_replace("/,\s*([A-Za-z0-9_]*)\s*,\s*([^,)\s]*)\s*/", "$1 = $2; ", $matches[2]);
return $args . "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec);
return $args . "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind);
}
return "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec);
return "goto " . helper_name($matches[1], $spec, $op1, $op2, $extra_spec, $kind);
}
},
$code);
@@ -1047,7 +1055,8 @@ function gen_handler($f, $spec, $kind, $name, $op1, $op2, $use, $code, $lineno,
// Generate opcode handler's entry point according to selected threading model
$additional_func = false;
$spec_name = $name.($spec?"_SPEC":"").$prefix[$op1].$prefix[$op2].($spec?extra_spec_name($extra_spec):"");
$variant = $kind === ZEND_VM_KIND_TAILCALL ? '_TAILCALL' : '';
$spec_name = $name.($spec?"_SPEC":"").$prefix[$op1].$prefix[$op2].($spec?extra_spec_name($extra_spec):"").$variant;
switch ($kind) {
case ZEND_VM_KIND_HYBRID:
if (is_inline_hybrid_handler($name, $opcode["hot"], $op1, $op2, $extra_spec)) {
@@ -1079,17 +1088,19 @@ function gen_handler($f, $spec, $kind, $name, $op1, $op2, $use, $code, $lineno,
}
return;
case ZEND_VM_KIND_CALL:
case ZEND_VM_KIND_TAILCALL:
$cconv = $kind === ZEND_VM_KIND_TAILCALL ? 'ZEND_OPCODE_HANDLER_CCONV' : 'ZEND_OPCODE_HANDLER_FUNC_CCONV';
if ($opcode["hot"] && ZEND_VM_KIND == ZEND_VM_KIND_HYBRID && is_hot_handler($opcode["hot"], $op1, $op2, $extra_spec)) {
if (isset($opcode["use"])) {
out($f,"static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL {$spec_name}_INLINE_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f,"static zend_always_inline ZEND_OPCODE_HANDLER_RET {$cconv} {$spec_name}_INLINE_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
$additional_func = true;
} else {
out($f,"static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL {$spec_name}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f,"static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET {$cconv} {$spec_name}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
}
} else if ($opcode["hot"] && is_cold_handler($opcode["hot"], $op1, $op2, $extra_spec)) {
out($f,"static ZEND_VM_COLD ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL {$spec_name}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f,"static ZEND_VM_COLD ZEND_OPCODE_HANDLER_RET {$cconv} {$spec_name}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
} else {
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL {$spec_name}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f,"static ZEND_OPCODE_HANDLER_RET {$cconv} {$spec_name}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
}
break;
case ZEND_VM_KIND_SWITCH:
@@ -1116,7 +1127,8 @@ function gen_handler($f, $spec, $kind, $name, $op1, $op2, $use, $code, $lineno,
gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec);
if ($additional_func) {
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL {$spec_name}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
$cconv = $kind === ZEND_VM_KIND_TAILCALL ? 'ZEND_OPCODE_HANDLER_CCONV' : 'ZEND_OPCODE_HANDLER_FUNC_CCONV';
out($f,"static ZEND_OPCODE_HANDLER_RET {$cconv} {$spec_name}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f,"{\n");
out($f,"\tZEND_VM_TAIL_CALL({$spec_name}_INLINE_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU));\n");
out($f,"}\n");
@@ -1125,7 +1137,7 @@ function gen_handler($f, $spec, $kind, $name, $op1, $op2, $use, $code, $lineno,
}
// Generates helper
function gen_helper($f, $spec, $kind, $name, $op1, $op2, $param, $code, $lineno, $inline, $cold = false, $hot = false, $extra_spec = null) {
function gen_helper($f, $spec, $kind, $name, $op1, $op2, $param, $code, $lineno, $inline, $cold = false, $hot = false, $extra_spec = null, $signature_only = false) {
global $definition_file, $prefix;
if ($kind == ZEND_VM_KIND_HYBRID && !$hot) {
@@ -1140,7 +1152,8 @@ function gen_helper($f, $spec, $kind, $name, $op1, $op2, $param, $code, $lineno,
out($f, "#line $lineno \"$definition_file\"\n");
}
$spec_name = $name.($spec?"_SPEC":"").$prefix[$op1].$prefix[$op2].($spec?extra_spec_name($extra_spec):"");
$variant = $kind === ZEND_VM_KIND_TAILCALL ? '_TAILCALL' : '';
$spec_name = $name.($spec?"_SPEC":"").$prefix[$op1].$prefix[$op2].($spec?extra_spec_name($extra_spec):"").$variant;
// Generate helper's entry point according to selected threading model
switch ($kind) {
@@ -1148,8 +1161,11 @@ function gen_helper($f, $spec, $kind, $name, $op1, $op2, $param, $code, $lineno,
out($f, $spec_name . "_LABEL:\n");
break;
case ZEND_VM_KIND_CALL:
case ZEND_VM_KIND_TAILCALL:
if ($inline) {
$zend_attributes = " zend_always_inline";
$zend_cconv = "";
$zend_cconv_ex = "";
$zend_fastcall = "";
} else {
if ($cold) {
@@ -1157,14 +1173,22 @@ function gen_helper($f, $spec, $kind, $name, $op1, $op2, $param, $code, $lineno,
} else {
$zend_attributes = " zend_never_inline";
}
$zend_fastcall = " ZEND_FASTCALL";
if ($kind !== ZEND_VM_KIND_TAILCALL) {
$zend_cconv = " ZEND_OPCODE_HANDLER_FUNC_CCONV ";
$zend_cconv_ex = " ZEND_OPCODE_HANDLER_FUNC_CCONV_EX ";
} else {
$zend_cconv = " ZEND_OPCODE_HANDLER_CCONV ";
$zend_cconv_ex = " ZEND_OPCODE_HANDLER_CCONV_EX ";
}
$zend_fastcall = " ZEND_FASTCALL ";
}
$semi = $signature_only ? ';' : '';
if ($param == null) {
// Helper without parameters
out($f, "static$zend_attributes ZEND_OPCODE_HANDLER_RET$zend_fastcall $spec_name(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f, "static$zend_attributes ZEND_OPCODE_HANDLER_RET$zend_cconv $spec_name(ZEND_OPCODE_HANDLER_ARGS)$semi\n");
} else {
// Helper with parameter
out($f, "static$zend_attributes ZEND_OPCODE_HANDLER_RET$zend_fastcall $spec_name(ZEND_OPCODE_HANDLER_ARGS_EX $param)\n");
out($f, "static$zend_attributes ZEND_OPCODE_HANDLER_RET$zend_cconv_ex $spec_name(ZEND_OPCODE_HANDLER_ARGS_EX $param)$semi\n");
}
break;
case ZEND_VM_KIND_SWITCH:
@@ -1175,16 +1199,20 @@ function gen_helper($f, $spec, $kind, $name, $op1, $op2, $param, $code, $lineno,
break;
}
// Generate helper's code
gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec);
if (!$signature_only) {
// Generate helper's code
gen_code($f, $spec, $kind, $code, $op1, $op2, $name, $extra_spec);
}
}
function gen_null_label($f, $kind, $prolog) {
switch ($kind) {
case ZEND_VM_KIND_CALL:
out($f,$prolog."ZEND_NULL_HANDLER,\n");
break;
case ZEND_VM_KIND_TAILCALL:
out($f,$prolog."ZEND_NULL_TAILCALL_HANDLER,\n");
break;
case ZEND_VM_KIND_SWITCH:
out($f,$prolog."-1,\n");
break;
@@ -1387,6 +1415,9 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array()
case ZEND_VM_KIND_CALL:
out($f,"$prolog{$spec_name}_HANDLER,\n");
break;
case ZEND_VM_KIND_TAILCALL:
out($f,"$prolog{$spec_name}_TAILCALL_HANDLER,\n");
break;
case ZEND_VM_KIND_SWITCH:
out($f,$prolog."$switch_labels[$spec_name],\n");
break;
@@ -1433,7 +1464,8 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array()
// to handler of undefined opcode
switch ($kind) {
case ZEND_VM_KIND_CALL:
out($f,$prolog."ZEND_NULL_HANDLER,\n");
case ZEND_VM_KIND_TAILCALL:
out($f,$prolog."ZEND_NULL${variant}_HANDLER,\n");
break;
case ZEND_VM_KIND_SWITCH:
out($f,$prolog."-1,\n");
@@ -1453,6 +1485,7 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array()
// Emit pointer to unspecialized handler
switch ($kind) {
case ZEND_VM_KIND_CALL:
case ZEND_VM_KIND_TAILCALL:
out($f,$prolog.$dsc['alias']."_HANDLER,\n");
break;
case ZEND_VM_KIND_GOTO:
@@ -1464,6 +1497,7 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array()
// Emit pointer to unspecialized handler
switch ($kind) {
case ZEND_VM_KIND_CALL:
case ZEND_VM_KIND_TAILCALL:
out($f,$prolog.$dsc["op"]."_HANDLER,\n");
break;
case ZEND_VM_KIND_SWITCH:
@@ -1477,7 +1511,8 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array()
} else {
switch ($kind) {
case ZEND_VM_KIND_CALL:
out($f,$prolog."ZEND_NULL_HANDLER,\n");
case ZEND_VM_KIND_TAILCALL:
out($f,$prolog."ZEND_NULL${variant}_HANDLER,\n");
break;
case ZEND_VM_KIND_SWITCH:
out($f,$prolog."-1,\n");
@@ -1496,6 +1531,9 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array()
case ZEND_VM_KIND_CALL:
out($f,$prolog."ZEND_NULL_HANDLER\n");
break;
case ZEND_VM_KIND_TAILCALL:
out($f,$prolog."ZEND_NULL_TAILCALL_HANDLER\n");
break;
case ZEND_VM_KIND_SWITCH:
out($f,$prolog."-1\n");
break;
@@ -1533,22 +1571,24 @@ function gen_specs($f, $prolog, $specs) {
}
// Generates handler for undefined opcodes (CALL threading model)
function gen_null_handler($f) {
static $done = 0;
function gen_null_handler($f, $kind) {
$cconv = $kind === ZEND_VM_KIND_TAILCALL ? 'ZEND_OPCODE_HANDLER_CCONV' : 'ZEND_OPCODE_HANDLER_FUNC_CCONV';
$variant = $kind === ZEND_VM_KIND_TAILCALL ? '_TAILCALL' : '';
out($f,"static ZEND_OPCODE_HANDLER_RET {$cconv} ZEND_NULL{$variant}_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f,"{\n");
out($f,"\tUSE_OPLINE\n");
out($f,"\n");
out($f,"\tSAVE_OPLINE();\n");
out($f,"\tzend_error_noreturn(E_ERROR, \"Invalid opcode %d/%d/%d.\", OPLINE->opcode, OPLINE->op1_type, OPLINE->op2_type);\n");
out($f,"\tZEND_VM_NEXT_OPCODE(); /* Never reached */\n");
out($f,"}\n\n");
}
// New and all executors with CALL threading model can use the same handler
// for undefined opcodes, do we emit code for it only once
if (!$done) {
$done = 1;
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_NULL_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f,"{\n");
out($f,"\tUSE_OPLINE\n");
out($f,"\n");
out($f,"\tSAVE_OPLINE();\n");
out($f,"\tzend_error_noreturn(E_ERROR, \"Invalid opcode %d/%d/%d.\", OPLINE->opcode, OPLINE->op1_type, OPLINE->op2_type);\n");
out($f,"\tZEND_VM_NEXT_OPCODE(); /* Never reached */\n");
out($f,"}\n\n");
}
function gen_halt_handler($f, $kind) {
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV ZEND_HALT_TAILCALL_HANDLER(ZEND_OPCODE_HANDLER_ARGS)\n");
out($f,"{\n");
out($f,"\treturn (zend_op*) ZEND_VM_ENTER_BIT;\n");
out($f,"}\n\n");
}
function extra_spec_name($extra_spec) {
@@ -1671,6 +1711,8 @@ function read_order_file($fn) {
function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array()) {
global $list, $opcodes, $helpers, $op_types_ex, $gen_order;
$delayed_helpers = fopen("php://memory", "w+");
if ($spec) {
// Produce specialized executor
$op1t = $op_types_ex;
@@ -1698,7 +1740,13 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array())
if (isset($helpers[$num]["op1"][$op1]) &&
isset($helpers[$num]["op2"][$op2])) {
// Generate helper code
gen_helper($f, 1, $kind, $num, $op1, $op2, $helpers[$num]["param"], $helpers[$num]["code"], $lineno, $helpers[$num]["inline"], $helpers[$num]["cold"], $helpers[$num]["hot"], $extra_spec);
if ($kind === ZEND_VM_KIND_TAILCALL && $helpers[$num]["param"] !== null) {
$out = $delayed_helpers;
gen_helper($f, 1, $kind, $num, $op1, $op2, $helpers[$num]["param"], $helpers[$num]["code"], $lineno, $helpers[$num]["inline"], $helpers[$num]["cold"], $helpers[$num]["hot"], $extra_spec, signature_only: true);
} else {
$out = $f;
}
gen_helper($out, 1, $kind, $num, $op1, $op2, $helpers[$num]["param"], $helpers[$num]["code"], $lineno, $helpers[$num]["inline"], $helpers[$num]["cold"], $helpers[$num]["hot"], $extra_spec);
}
}
} else {
@@ -1722,7 +1770,13 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array())
} else if (isset($dsc["helper"])) {
$num = $dsc["helper"];
// Generate helper code
gen_helper($f, 0, $kind, $num, "ANY", "ANY", $helpers[$num]["param"], $helpers[$num]["code"], $lineno, $helpers[$num]["inline"], $helpers[$num]["cold"], $helpers[$num]["hot"]);
if ($kind === ZEND_VM_KIND_TAILCALL && $helpers[$num]["param"] !== null) {
$out = $delayed_helpers;
gen_helper($out, 0, $kind, $num, "ANY", "ANY", $helpers[$num]["param"], $helpers[$num]["code"], $lineno, $helpers[$num]["inline"], $helpers[$num]["cold"], $helpers[$num]["hot"], signature_only: true);
} else {
$out = $f;
}
gen_helper($out, 0, $kind, $num, "ANY", "ANY", $helpers[$num]["param"], $helpers[$num]["code"], $lineno, $helpers[$num]["inline"], $helpers[$num]["cold"], $helpers[$num]["hot"]);
} else {
var_dump($dsc);
die("??? $kind:$num\n");
@@ -1746,7 +1800,11 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array())
// Generate handler for undefined opcodes
switch ($kind) {
case ZEND_VM_KIND_CALL:
gen_null_handler($f);
gen_null_handler($f, $kind);
break;
case ZEND_VM_KIND_TAILCALL:
gen_null_handler($f, $kind);
gen_halt_handler($f, $kind);
break;
case ZEND_VM_KIND_SWITCH:
out($f,"default: ZEND_NULL_LABEL:\n");
@@ -1774,6 +1832,21 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array())
out($f,"\t\t\t\tHYBRID_BREAK(); /* Never reached */\n");
break;
}
rewind($delayed_helpers);
$delayed_helpers = stream_get_contents($delayed_helpers);
if ($delayed_helpers !== '') {
out($f, "/* The following helpers can not tailcall due to signature mismatch. Redefine some macros so they do not enforce tailcall. */\n");
out($f, "#pragma push_macro(\"ZEND_VM_CONTINUE\")\n");
out($f, "#undef ZEND_VM_CONTINUE\n");
out($f, "#pragma push_macro(\"ZEND_VM_INTERRUPT\")\n");
out($f, "#undef ZEND_VM_INTERRUPT\n");
out($f, "#define ZEND_VM_CONTINUE(handler) return opline\n");
out($f, "#define ZEND_VM_INTERRUPT() return zend_interrupt_helper".($spec?"_SPEC":"")."(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU)\n");
out($f, $delayed_helpers);
out($f, "#pragma pop_macro(\"ZEND_VM_INTERRUPT\")\n");
out($f, "#pragma pop_macro(\"ZEND_VM_CONTINUE\")\n");
}
}
function skip_blanks($f, $prolog, $epilog) {
@@ -1821,12 +1894,17 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
out($f,"static zend_op hybrid_halt_op;\n");
out($f,"#endif\n");
}
out($f,"#if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID) || !ZEND_VM_SPEC\n");
if ($kind == ZEND_VM_KIND_HYBRID || $kind == ZEND_VM_KIND_CALL) {
out($f,"#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL\n\n");
out($f,"static zend_vm_opcode_handler_func_t const * zend_opcode_handler_funcs;\n");
out($f,"#endif\n");
}
out($f,"#if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) || !ZEND_VM_SPEC\n");
out($f,"static zend_vm_opcode_handler_t zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op);\n");
out($f,"#endif\n\n");
if ($kind == ZEND_VM_KIND_HYBRID) {
out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n");
out($f,"static const void *zend_vm_get_opcode_handler_func(uint8_t opcode, const zend_op* op);\n");
out($f,"#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL\n");
out($f,"static zend_vm_opcode_handler_func_t zend_vm_get_opcode_handler_func(uint8_t opcode, const zend_op* op);\n");
out($f,"#else\n");
out($f,"# define zend_vm_get_opcode_handler_func zend_vm_get_opcode_handler\n");
out($f,"#endif\n\n");
@@ -1882,11 +1960,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
out($f,"#if defined(ZEND_VM_FP_GLOBAL_REG) && defined(ZEND_VM_IP_GLOBAL_REG)\n");
out($f,"# define ZEND_OPCODE_HANDLER_RET void\n");
out($f,"# define ZEND_VM_TAIL_CALL(call) call; return\n");
out($f,"# ifdef ZEND_VM_TAIL_CALL_DISPATCH\n");
out($f,"# define ZEND_VM_CONTINUE() ((opcode_handler_t)OPLINE->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); return\n");
out($f,"# else\n");
out($f,"# define ZEND_VM_CONTINUE() return\n");
out($f,"# endif\n");
out($f,"# define ZEND_VM_CONTINUE() return\n");
if ($kind == ZEND_VM_KIND_HYBRID) {
out($f,"# if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n");
out($f,"# define ZEND_VM_RETURN() opline = &hybrid_halt_op; return\n");
@@ -1911,8 +1985,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
}
out($f,"# define ZEND_VM_COLD ZEND_COLD ZEND_OPT_SIZE\n");
out($f,"#endif\n");
out($f,"\n");
out($f,"typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *opcode_handler_t) (ZEND_OPCODE_HANDLER_ARGS);\n");
out($f,"#define ZEND_VM_DISPATCH_TO_HELPER(call) ZEND_VM_TAIL_CALL(call)\n");
out($f,"\n");
out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
out($f,"# define DCL_OPLINE\n");
@@ -1952,13 +2025,13 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
out($f,"#define ZEND_VM_INTERRUPT() ZEND_VM_TAIL_CALL(zend_interrupt_helper".($spec?"_SPEC":"")."(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU));\n");
out($f,"#define ZEND_VM_LOOP_INTERRUPT() zend_interrupt_helper".($spec?"_SPEC":"")."(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
if ($kind == ZEND_VM_KIND_HYBRID) {
out($f,"#define ZEND_VM_DISPATCH(opcode, opline) ZEND_VM_TAIL_CALL(((opcode_handler_t)zend_vm_get_opcode_handler_func(opcode, opline))(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU));\n");
out($f,"#define ZEND_VM_DISPATCH(opcode, opline) return zend_vm_get_opcode_handler_func(opcode, opline)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
} else {
out($f,"#define ZEND_VM_DISPATCH(opcode, opline) ZEND_VM_TAIL_CALL(((opcode_handler_t)zend_vm_get_opcode_handler(opcode, opline))(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU));\n");
out($f,"#define ZEND_VM_DISPATCH(opcode, opline) return zend_vm_get_opcode_handler(opcode, opline)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
}
out($f,"\n");
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_interrupt_helper".($spec?"_SPEC":"")."(ZEND_OPCODE_HANDLER_ARGS);\n");
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_NULL_HANDLER(ZEND_OPCODE_HANDLER_ARGS);\n");
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_FUNC_CCONV zend_interrupt_helper".($spec?"_SPEC":"")."(ZEND_OPCODE_HANDLER_ARGS);\n");
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_FUNC_CCONV ZEND_NULL_HANDLER(ZEND_OPCODE_HANDLER_ARGS);\n");
out($f,"\n");
break;
case ZEND_VM_KIND_SWITCH:
@@ -2034,6 +2107,50 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
// out($f,"# define ZEND_VM_INTERRUPT() goto zend_interrupt_helper_SPEC_LABEL\n");
out($f,"#endif\n\n");
}
if ($kind == ZEND_VM_KIND_HYBRID || $kind == ZEND_VM_KIND_CALL) {
/* Generate both CALL and TAILCALL handlers.
* TAILCALL handlers are used as zend_vm_opcode_handler_t
* when supported.
* CALL handlers are used as zend_vm_opcode_handler_func_t.
*/
out($f,"#if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL\n");
out($f,"\n");
out($f,"# undef ZEND_VM_TAIL_CALL\n");
out($f,"# undef ZEND_VM_CONTINUE\n");
out($f,"# undef ZEND_VM_RETURN\n");
out($f,"# undef ZEND_VM_DISPATCH_TO_HELPER\n");
out($f,"# undef ZEND_VM_INTERRUPT\n");
out($f,"\n");
out($f,"# define ZEND_VM_TAIL_CALL(call) ZEND_MUSTTAIL return call\n");
out($f,"# define ZEND_VM_CONTINUE() ZEND_VM_TAIL_CALL(opline->handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU))\n");
out($f,"# define ZEND_VM_RETURN() opline = &call_halt_op; ZEND_VM_CONTINUE()\n");
out($f,"# define ZEND_VM_DISPATCH_TO_HELPER(call) \\\n");
out($f," do { \\\n");
out($f," opline = call; \\\n");
out($f," ZEND_VM_TAIL_CALL(opline->handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU)); \\\n");
out($f," } while (0)\n");
out($f,"# define ZEND_VM_DISPATCH_TO_LEAVE_HELPER(helper) opline = &call_leave_op; SAVE_OPLINE(); ZEND_VM_CONTINUE()\n");
out($f,"# define ZEND_VM_INTERRUPT() ZEND_VM_TAIL_CALL(zend_interrupt_helper".($spec?"_SPEC":"")."_TAILCALL(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU))\n");
out($f,"\n");
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_interrupt_helper".($spec?"_SPEC":"")."_TAILCALL(ZEND_OPCODE_HANDLER_ARGS);\n");
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV ZEND_NULL_TAILCALL_HANDLER(ZEND_OPCODE_HANDLER_ARGS);\n");
out($f,"static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV ZEND_HALT_TAILCALL_HANDLER(ZEND_OPCODE_HANDLER_ARGS);\n");
out($f,"static zend_never_inline const zend_op *ZEND_OPCODE_HANDLER_CCONV zend_leave_helper_SPEC_TAILCALL(zend_execute_data *ex, const zend_op *opline);\n");
out($f,"\n");
out($f,"static const zend_op call_halt_op = {\n");
out($f," .handler = ZEND_HALT_TAILCALL_HANDLER,\n");
out($f,"};\n");
out($f,"static const zend_op call_leave_op = {\n");
out($f," .handler = zend_leave_helper_SPEC_TAILCALL,\n");
out($f,"};\n");
out($f,"\n");
gen_executor_code($f, $spec, ZEND_VM_KIND_TAILCALL, $m[1]);
out($f,"#endif /* ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL */\n");
}
break;
case "EXECUTOR_NAME":
out($f, $m[1].$executor_name.$m[3]."\n");
@@ -2094,7 +2211,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
}
$prolog = $m[1];
out($f,$prolog."if (UNEXPECTED(execute_data == NULL)) {\n");
out($f,$prolog."\tstatic const void * const labels[] = {\n");
out($f,$prolog."\tstatic zend_vm_opcode_handler_t const labels[] = {\n");
gen_labels($f, $spec, ($kind == ZEND_VM_KIND_HYBRID) ? ZEND_VM_KIND_GOTO : $kind, $prolog."\t\t", $specs);
out($f,$prolog."\t};\n");
out($f,$prolog."\tzend_opcode_handlers = (zend_vm_opcode_handler_t*) labels;\n");
@@ -2143,10 +2260,10 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
out($f,"#else /* ZEND_VM_KIND != ZEND_VM_KIND_HYBRID */\n");
case ZEND_VM_KIND_CALL:
out($f,"#if defined(ZEND_VM_FP_GLOBAL_REG) && defined(ZEND_VM_IP_GLOBAL_REG)\n");
out($f, $m[1]."((opcode_handler_t)OPLINE->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, $m[1]."(OPLINE->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, $m[1]."if (UNEXPECTED(!OPLINE))".$m[3]."\n");
out($f,"#else\n");
out($f, $m[1]."opline = ((opcode_handler_t)opline->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, $m[1]."opline = (opline->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, $m[1]."if (UNEXPECTED(((uintptr_t)opline & ZEND_VM_ENTER_BIT)))".$m[3]."\n");
out($f,"#endif\n");
if ($kind == ZEND_VM_KIND_HYBRID) {
@@ -2194,7 +2311,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
break;
case "EXTERNAL_EXECUTOR":
if ($kind == ZEND_VM_KIND_CALL) {
gen_executor_code($f, $spec, $kind, $m[1]);
gen_executor_code($f, $spec, ZEND_VM_KIND_CALL, $m[1]);
}
break;
case "INITIALIZER_NAME":
@@ -2212,29 +2329,38 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
out($f,$prolog."zend_spec_handlers = specs;\n");
out($f,$prolog.$executor_name."_ex(NULL);\n");
} else {
out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n");
out($f,$prolog."static zend_vm_opcode_handler_func_t const labels[] = {\n");
out($f,"#else\n");
out($f,$prolog."static zend_vm_opcode_handler_t const labels[] = {\n");
out($f,"#endif\n");
gen_labels($f, $spec, ($kind == ZEND_VM_KIND_HYBRID) ? ZEND_VM_KIND_CALL : $kind, $prolog."\t", $specs, $switch_labels);
out($f,$prolog."static zend_vm_opcode_handler_func_t const funcs[] = {\n");
gen_labels($f, $spec, ZEND_VM_KIND_CALL, $prolog."\t", $specs);
out($f,$prolog."};\n");
out($f, "#if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL || ZEND_VM_KIND == ZEND_VM_KIND_SWITCH\n");
out($f,$prolog."static zend_vm_opcode_handler_t const handlers[] = {\n");
gen_labels($f, $spec, $kind === ZEND_VM_KIND_HYBRID || $kind === ZEND_VM_KIND_CALL ? ZEND_VM_KIND_TAILCALL : $kind, $prolog."\t", $specs, $switch_labels);
out($f,$prolog."};\n");
out($f,$prolog."zend_handlers_count = sizeof(handlers) / sizeof(handlers[0]);\n");
out($f, "#elif ZEND_VM_KIND != ZEND_VM_KIND_HYBRID\n");
out($f,$prolog."static zend_vm_opcode_handler_t const *handlers = funcs;\n");
out($f,$prolog."zend_handlers_count = sizeof(funcs) / sizeof(funcs[0]);\n");
out($f, "#endif\n");
out($f,$prolog."static const uint32_t specs[] = {\n");
gen_specs($f, $prolog."\t", $specs);
out($f,$prolog."};\n");
out($f,"#if 0\n");
if ($kind == ZEND_VM_KIND_HYBRID) {
out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n");
out($f,$prolog."zend_opcode_handler_funcs = labels;\n");
out($f,"#elif (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n");
out($f,$prolog."zend_opcode_handler_funcs = funcs;\n");
out($f,$prolog."zend_spec_handlers = specs;\n");
out($f,$prolog.$executor_name."_ex(NULL);\n");
out($f,"#else\n");
}
out($f,$prolog."zend_opcode_handlers = labels;\n");
out($f,$prolog."zend_handlers_count = sizeof(labels) / sizeof(labels[0]);\n");
if ($kind == ZEND_VM_KIND_HYBRID || $kind == ZEND_VM_KIND_CALL) {
out($f,"#elif ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL\n");
out($f,$prolog."zend_opcode_handler_funcs = funcs;\n");
out($f,$prolog."zend_opcode_handlers = handlers;\n");
out($f,$prolog."zend_spec_handlers = specs;\n");
}
out($f,"#else\n");
out($f,$prolog."zend_opcode_handlers = handlers;\n");
out($f,$prolog."zend_spec_handlers = specs;\n");
if ($kind == ZEND_VM_KIND_HYBRID) {
out($f,"#endif\n");
}
out($f,"#endif\n");
}
break;
default:
@@ -2346,16 +2472,23 @@ function gen_vm_opcodes_header(
$str .= "#define ZEND_VM_KIND_SWITCH\t" . ZEND_VM_KIND_SWITCH . "\n";
$str .= "#define ZEND_VM_KIND_GOTO\t" . ZEND_VM_KIND_GOTO . "\n";
$str .= "#define ZEND_VM_KIND_HYBRID\t" . ZEND_VM_KIND_HYBRID . "\n";
$str .= "#define ZEND_VM_KIND_TAILCALL\t" . ZEND_VM_KIND_TAILCALL . "\n";
$str .= "#if 0\n";
if ($GLOBALS["vm_kind_name"][ZEND_VM_KIND] === "ZEND_VM_KIND_HYBRID") {
$str .= "/* HYBRID requires support for computed GOTO and global register variables*/\n";
$str .= "#if (defined(__GNUC__) && defined(HAVE_GCC_GLOBAL_REGS))\n";
$str .= "#elif (defined(__GNUC__) && defined(HAVE_GCC_GLOBAL_REGS))\n";
$str .= "# define ZEND_VM_KIND\t\tZEND_VM_KIND_HYBRID\n";
}
if ($GLOBALS["vm_kind_name"][ZEND_VM_KIND] === "ZEND_VM_KIND_HYBRID" || $GLOBALS["vm_kind_name"][ZEND_VM_KIND] === "ZEND_VM_KIND_CALL") {
$str .= "#elif defined(HAVE_MUSTTAIL) && defined(HAVE_PRESERVE_NONE) && (defined(__x86_64__) || defined(__aarch64__))\n";
$str .= "# define ZEND_VM_KIND\t\tZEND_VM_KIND_TAILCALL\n";
$str .= "#else\n";
$str .= "# define ZEND_VM_KIND\t\tZEND_VM_KIND_CALL\n";
$str .= "#endif\n";
} else {
$str .= "#else\n";
$str .= "#define ZEND_VM_KIND\t\t" . $GLOBALS["vm_kind_name"][ZEND_VM_KIND] . "\n";
}
$str .= "#endif\n";
$str .= "\n";
$str .= "#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)\n";
$str .= "# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64))\n";
@@ -2363,12 +2496,22 @@ function gen_vm_opcodes_header(
$str .= "# endif\n";
$str .= "#endif\n";
$str .= "\n";
$str .= "#if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL\n";
$str .= "# define ZEND_OPCODE_HANDLER_CCONV ZEND_PRESERVE_NONE\n";
$str .= "# define ZEND_OPCODE_HANDLER_CCONV_EX ZEND_FASTCALL\n";
$str .= "#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL\n";
$str .= "# define ZEND_OPCODE_HANDLER_CCONV ZEND_FASTCALL\n";
$str .= "# define ZEND_OPCODE_HANDLER_CCONV_EX ZEND_FASTCALL\n";
$str .= "#endif\n";
$str .= "#define ZEND_OPCODE_HANDLER_FUNC_CCONV ZEND_FASTCALL\n";
$str .= "#define ZEND_OPCODE_HANDLER_FUNC_CCONV_EX ZEND_FASTCALL\n";
$str .= "\n";
$str .= "#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID\n";
$str .= "typedef const void* zend_vm_opcode_handler_t;\n";
$str .= "typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_func_t)(void);\n";
$str .= "#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL\n";
$str .= "typedef const struct _zend_op *(ZEND_FASTCALL *zend_vm_opcode_handler_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);\n";
$str .= "typedef const struct _zend_op *(ZEND_FASTCALL *zend_vm_opcode_handler_func_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);\n";
$str .= "#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL\n";
$str .= "typedef const struct _zend_op *(ZEND_OPCODE_HANDLER_CCONV *zend_vm_opcode_handler_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);\n";
$str .= "typedef const struct _zend_op *(ZEND_OPCODE_HANDLER_FUNC_CCONV *zend_vm_opcode_handler_func_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);\n";
$str .= "#elif ZEND_VM_KIND == ZEND_VM_KIND_SWITCH\n";
$str .= "typedef int zend_vm_opcode_handler_t;\n";
$str .= "#elif ZEND_VM_KIND == ZEND_VM_KIND_GOTO\n";
@@ -2857,7 +3000,7 @@ function gen_vm($def, $skel) {
out($f, "\treturn (spec & SPEC_START_MASK) + offset;\n");
}
out($f, "}\n\n");
out($f, "#if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID) || !ZEND_VM_SPEC\n");
out($f, "#if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) || !ZEND_VM_SPEC\n");
out($f, "static zend_vm_opcode_handler_t zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op)\n");
out($f, "{\n");
if (!ZEND_VM_SPEC) {
@@ -2868,10 +3011,10 @@ function gen_vm($def, $skel) {
out($f, "}\n");
out($f, "#endif\n\n");
if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) {
if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID || ZEND_VM_KIND == ZEND_VM_KIND_CALL) {
// Generate zend_vm_get_opcode_handler_func() function
out($f, "#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID\n");
out($f,"static const void *zend_vm_get_opcode_handler_func(uint8_t opcode, const zend_op* op)\n");
out($f, "#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL\n");
out($f,"static zend_vm_opcode_handler_func_t zend_vm_get_opcode_handler_func(uint8_t opcode, const zend_op* op)\n");
out($f, "{\n");
out($f, "\tuint32_t spec = zend_spec_handlers[opcode];\n");
if (!ZEND_VM_SPEC) {
@@ -2983,8 +3126,8 @@ function gen_vm($def, $skel) {
out($f, "ZEND_API int ZEND_FASTCALL zend_vm_call_opcode_handler(zend_execute_data* ex)\n");
out($f, "{\n");
if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) {
out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n");
out($f, "\topcode_handler_t handler;\n");
out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL)\n");
out($f, "\tzend_vm_opcode_handler_func_t handler;\n");
out($f,"#endif\n");
}
out($f, "\tDCL_OPLINE;\n");
@@ -3003,12 +3146,12 @@ function gen_vm($def, $skel) {
out($f,"#if defined(ZEND_VM_FP_GLOBAL_REG) && defined(ZEND_VM_IP_GLOBAL_REG)\n");
if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) {
out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n");
out($f, "\thandler = (opcode_handler_t)zend_vm_get_opcode_handler_func(zend_user_opcodes[opline->opcode], opline);\n");
out($f, "\thandler = zend_vm_get_opcode_handler_func(zend_user_opcodes[opline->opcode], opline);\n");
out($f, "\thandler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, "\tif (EXPECTED(opline != &hybrid_halt_op)) {\n");
out($f,"#else\n");
}
out($f, "\t((opcode_handler_t)OPLINE->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, "\t(OPLINE->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) {
out($f, "\tif (EXPECTED(opline)) {\n");
out($f,"#endif\n");
@@ -3021,9 +3164,14 @@ function gen_vm($def, $skel) {
out($f, "\t\tret = -1;\n");
out($f, "\t}\n");
out($f, "#else\n");
out($f, "\topline = ((opcode_handler_t)OPLINE->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, "# if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL\n");
out($f, "\thandler = zend_vm_get_opcode_handler_func(zend_user_opcodes[opline->opcode], opline);\n");
out($f, "\topline = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, "# else\n");
out($f, "\topline = (OPLINE->handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);\n");
out($f, "# endif\n");
out($f, "if (UNEXPECTED(((uintptr_t)opline & ZEND_VM_ENTER_BIT))) {\n");
out($f, "\tif (UNEXPECTED(((uintptr_t)opline & ZEND_VM_ENTER_BIT))) {\n");
out($f, "\t\topline = (const zend_op*)((uintptr_t)opline & ~ZEND_VM_ENTER_BIT);\n");
out($f, "\t\tif (EXPECTED(opline)) {\n");
out($f, "\t\t\t/* ZEND_VM_ENTER() or ZEND_VM_LEAVE() */\n");

22
Zend/zend_vm_opcodes.h generated
View File

@@ -29,9 +29,13 @@
#define ZEND_VM_KIND_SWITCH 2
#define ZEND_VM_KIND_GOTO 3
#define ZEND_VM_KIND_HYBRID 4
#define ZEND_VM_KIND_TAILCALL 5
#if 0
/* HYBRID requires support for computed GOTO and global register variables*/
#if (defined(__GNUC__) && defined(HAVE_GCC_GLOBAL_REGS))
#elif (defined(__GNUC__) && defined(HAVE_GCC_GLOBAL_REGS))
# define ZEND_VM_KIND ZEND_VM_KIND_HYBRID
#elif defined(HAVE_MUSTTAIL) && defined(HAVE_PRESERVE_NONE) && (defined(__x86_64__) || defined(__aarch64__))
# define ZEND_VM_KIND ZEND_VM_KIND_TAILCALL
#else
# define ZEND_VM_KIND ZEND_VM_KIND_CALL
#endif
@@ -42,12 +46,22 @@
# endif
#endif
#if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
# define ZEND_OPCODE_HANDLER_CCONV ZEND_PRESERVE_NONE
# define ZEND_OPCODE_HANDLER_CCONV_EX ZEND_FASTCALL
#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL
# define ZEND_OPCODE_HANDLER_CCONV ZEND_FASTCALL
# define ZEND_OPCODE_HANDLER_CCONV_EX ZEND_FASTCALL
#endif
#define ZEND_OPCODE_HANDLER_FUNC_CCONV ZEND_FASTCALL
#define ZEND_OPCODE_HANDLER_FUNC_CCONV_EX ZEND_FASTCALL
#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID
typedef const void* zend_vm_opcode_handler_t;
typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_func_t)(void);
#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL
typedef const struct _zend_op *(ZEND_FASTCALL *zend_vm_opcode_handler_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);
typedef const struct _zend_op *(ZEND_FASTCALL *zend_vm_opcode_handler_func_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);
#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
typedef const struct _zend_op *(ZEND_OPCODE_HANDLER_CCONV *zend_vm_opcode_handler_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);
typedef const struct _zend_op *(ZEND_OPCODE_HANDLER_FUNC_CCONV *zend_vm_opcode_handler_func_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);
#elif ZEND_VM_KIND == ZEND_VM_KIND_SWITCH
typedef int zend_vm_opcode_handler_t;
#elif ZEND_VM_KIND == ZEND_VM_KIND_GOTO

21
ext/opcache/jit/zend_jit.c
View File

@@ -98,7 +98,11 @@ static zend_vm_opcode_handler_t zend_jit_func_trace_counter_handler = NULL;
static zend_vm_opcode_handler_t zend_jit_ret_trace_counter_handler = NULL;
static zend_vm_opcode_handler_t zend_jit_loop_trace_counter_handler = NULL;
static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS);
#if ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS);
#else
static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_FUNC_CCONV zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS);
#endif
static int zend_jit_trace_op_len(const zend_op *opline);
static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
@@ -3074,7 +3078,11 @@ jit_failure:
}
/* Run-time JIT handler */
static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS)
#if ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS)
#else
static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_FUNC_CCONV zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS)
#endif
{
#if GCC_GLOBAL_REGS
zend_execute_data *execute_data;
@@ -3126,7 +3134,8 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_runtime_jit(ZEND_OPCODE_HANDLE
#if GCC_GLOBAL_REGS
return; // ZEND_VM_CONTINUE
#else
return orig_opline; // ZEND_VM_CONTINUE
opline = orig_opline;
ZEND_OPCODE_RETURN();
#endif
}
@@ -3314,7 +3323,7 @@ int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
jit_extension->op_array = op_array;
jit_extension->orig_handler = (void*)opline->handler;
jit_extension->orig_handler = opline->handler;
ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
opline->handler = zend_jit_runtime_jit_handler;
zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
@@ -3344,7 +3353,7 @@ int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
jit_extension->op_array = op_array;
jit_extension->orig_handler = (void*)opline->handler;
jit_extension->orig_handler = opline->handler;
ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
opline->handler = zend_jit_profile_jit_handler;
zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
@@ -3707,7 +3716,7 @@ void zend_jit_init(void)
#endif
}
#if ZEND_VM_KIND != ZEND_VM_KIND_CALL && ZEND_VM_KIND != ZEND_VM_KIND_HYBRID
#if ZEND_VM_KIND != ZEND_VM_KIND_CALL && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL && ZEND_VM_KIND != ZEND_VM_KIND_HYBRID
# error JIT is compatible only with CALL and HYBRID VM
#endif

41
ext/opcache/jit/zend_jit_internal.h
View File

@@ -197,10 +197,6 @@ extern const zend_op *zend_jit_halt_op;
handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
return; \
} while(0)
# define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX arg); \
return; \
} while(0)
# define ZEND_VM_ENTER_BIT 0
#else
# define EXECUTE_DATA_D zend_execute_data* execute_data
@@ -212,17 +208,21 @@ extern const zend_op *zend_jit_halt_op;
# define OPLINE_DC , OPLINE_D
# define OPLINE_CC , OPLINE_C
# define ZEND_OPCODE_HANDLER_RET const zend_op *
# if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
# define ZEND_OPCODE_TAIL_CALL(handler) do { \
ZEND_MUSTTAIL return (handler)(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
} while(0)
# define ZEND_OPCODE_RETURN() ZEND_OPCODE_TAIL_CALL((zend_vm_opcode_handler_t)opline->handler)
# else
# define ZEND_OPCODE_TAIL_CALL(handler) do { \
return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
} while(0)
# define ZEND_OPCODE_RETURN() return opline
# endif
# define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D OPLINE_DC
# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU EXECUTE_DATA_C OPLINE_CC
# define ZEND_OPCODE_HANDLER_ARGS_EX EXECUTE_DATA_D OPLINE_DC,
# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX EXECUTE_DATA_C OPLINE_CC,
# define ZEND_OPCODE_RETURN() return opline
# define ZEND_OPCODE_TAIL_CALL(handler) do { \
return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
} while(0)
# define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX arg); \
} while(0)
# define ZEND_VM_ENTER_BIT 1ULL
#endif
@@ -230,11 +230,16 @@ extern const zend_op *zend_jit_halt_op;
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(ZEND_OPCODE_HANDLER_ARGS);
#if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_leave_func_helper_tailcall(ZEND_OPCODE_HANDLER_ARGS);
#endif
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS);
#if ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
#endif
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper_no_skip_recv(ZEND_OPCODE_HANDLER_ARGS);
@@ -666,9 +671,11 @@ struct _zend_jit_trace_stack_frame {
(frame)->_info |= TRACE_FRAME_MASK_ALWAYS_RELEASE_THIS; \
} while (0)
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
#if ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
#endif
int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline);
zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data,

206
ext/opcache/jit/zend_jit_ir.c
View File

@@ -27,11 +27,21 @@
# define ZREG_IP 7 /* IR_REG_RDI */
# define ZREG_FIRST_FPR 8
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */
# if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
# error
# endif
#elif defined(IR_TARGET_X64)
# define IR_REG_SP 4 /* IR_REG_RSP */
# define IR_REG_FP 5 /* IR_REG_RBP */
# define ZREG_FP 14 /* IR_REG_R14 */
# define ZREG_IP 15 /* IR_REG_R15 */
# if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
/* Use the first two arg registers of the preserve_none calling convention for FP/IP
* https://github.com/llvm/llvm-project/blob/68bfe91b5a34f80dbcc4f0a7fa5d7aa1cdf959c2/llvm/lib/Target/X86/X86CallingConv.td#L1029 */
# define ZREG_FP 12 /* IR_REG_R12 */
# define ZREG_IP 13 /* IR_REG_R13 */
# else
# define ZREG_FP 14 /* IR_REG_R14 */
# define ZREG_IP 15 /* IR_REG_R15 */
# endif
# define ZREG_FIRST_FPR 16
# if defined(_WIN64)
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15))
@@ -46,9 +56,17 @@
# endif
#elif defined(IR_TARGET_AARCH64)
# define IR_REG_SP 31 /* IR_REG_RSP */
# define IR_REG_LR 30 /* IR_REG_X30 */
# define IR_REG_FP 29 /* IR_REG_X29 */
# define ZREG_FP 27 /* IR_REG_X27 */
# define ZREG_IP 28 /* IR_REG_X28 */
# if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
/* Use the first two arg registers of the preserve_none calling convention for FP/IP
* https://github.com/llvm/llvm-project/blob/68bfe91b5a34f80dbcc4f0a7fa5d7aa1cdf959c2/llvm/lib/Target/AArch64/AArch64CallingConvention.td#L541 */
# define ZREG_FP 20 /* IR_REG_X20 */
# define ZREG_IP 21 /* IR_REG_X21 */
# else
# define ZREG_FP 27 /* IR_REG_X27 */
# define ZREG_IP 28 /* IR_REG_X28 */
# endif
# define ZREG_FIRST_FPR 32
# define IR_REGSET_PRESERVED ((1<<19) | (1<<20) | (1<<21) | (1<<22) | (1<<23) | \
(1<<24) | (1<<25) | (1<<26) | (1<<27) | (1<<28)) /* all preserved registers */
@@ -64,12 +82,22 @@
#undef _ir_CTX
#define _ir_CTX (&jit->ctx)
#if GCC_GLOBAL_REGS
# define IR_OPCODE_HANDLER_RET IR_VOID
#else
# define IR_OPCODE_HANDLER_RET IR_ADDR
#endif
#undef ir_CONST_ADDR
#define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr))
#define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)
#define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)
#define ir_CAST_FC_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \
ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_I32))
# define ir_CONST_OPCODE_HANDLER_FUNC(_addr) \
jit_CONST_OPCODE_HANDLER_FUNC(jit, _addr)
# define ir_CAST_OPCODE_HANDLER_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \
ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_OPCODE_HANDLER_RET))
#define ir_CONST_FUNC_PROTO(_addr, _proto) \
jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto))
@@ -549,6 +577,11 @@ static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags)
return jit_CONST_FUNC_PROTO(jit, addr, proto);
}
static ir_ref jit_CONST_OPCODE_HANDLER_FUNC(zend_jit_ctx *jit, zend_vm_opcode_handler_t handler)
{
return jit_CONST_FUNC(jit, (uintptr_t)handler, IR_FASTCALL_FUNC);
}
static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset)
{
if (offset) {
@@ -1922,6 +1955,20 @@ static void zend_jit_vm_leave(zend_jit_ctx *jit, ir_ref to_opline)
ir_RETURN(ir_OR_A(to_opline, ir_CONST_ADDR(ZEND_VM_ENTER_BIT)));
}
static void zend_jit_tailcall_handler(zend_jit_ctx *jit, ir_ref handler)
{
#if defined(IR_TARGET_X86)
if (!IR_IS_CONST_REF(handler)) {
handler = ir_CAST_OPCODE_HANDLER_FUNC(handler);
}
#endif
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
ir_TAILCALL(IR_OPCODE_HANDLER_RET, handler);
} else {
ir_TAILCALL_2(IR_ADDR, handler, jit_FP(jit), jit_IP(jit));
}
}
/* stubs */
static int zend_jit_exception_handler_stub(zend_jit_ctx *jit)
@@ -1934,10 +1981,10 @@ static int zend_jit_exception_handler_stub(zend_jit_ctx *jit)
} else {
zend_vm_opcode_handler_t handler = EG(exception_op)->handler;
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_jit_tailcall_handler(jit, ir_CONST_OPCODE_HANDLER_FUNC(handler));
} else {
ir_ref ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit));
ir_ref ref = ir_CALL_2(IR_ADDR, ir_CONST_OPCODE_HANDLER_FUNC(handler), jit_FP(jit), jit_IP(jit));
zend_jit_vm_enter(jit, ref);
}
}
@@ -2036,8 +2083,8 @@ static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit)
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
}
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit)));
} else {
zend_jit_vm_enter(jit, jit_IP(jit));
}
@@ -2046,6 +2093,10 @@ static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit)
static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit)
{
#if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
ir_TAILCALL(IR_OPCODE_HANDLER_RET, ir_CONST_OPCODE_HANDLER_FUNC(zend_jit_leave_func_helper_tailcall));
return 1;
#else
ir_ref call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
ir_ref if_top = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_TOP)));
@@ -2074,6 +2125,7 @@ static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit)
}
return 1;
#endif /* ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL */
}
static int zend_jit_negative_shift_stub(zend_jit_ctx *jit)
@@ -2234,7 +2286,7 @@ static int zend_jit_leave_throw_stub(zend_jit_ctx *jit)
// JIT: opline = EG(exception_op);
jit_STORE_IP(jit, jit_EG(exception_op));
if (GCC_GLOBAL_REGS) {
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
ir_STORE(jit_EX(opline), jit_IP(jit));
// JIT: HANDLE_EXCEPTION()
@@ -2252,7 +2304,7 @@ static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit)
return 0;
}
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_runtime_jit));
ir_CALL(IR_VOID, ir_CONST_OPCODE_HANDLER_FUNC(zend_runtime_jit));
ir_IJMP(ir_LOAD_A(jit_IP(jit)));
return 1;
}
@@ -2382,7 +2434,7 @@ static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost)
ir_IJMP(_zend_jit_orig_opline_handler(jit, offset));
ir_IF_TRUE(if_halt);
ir_IJMP(ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
ir_IJMP(ir_CONST_OPCODE_HANDLER_FUNC(zend_jit_halt_op->handler));
return 1;
}
@@ -2420,7 +2472,7 @@ static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit)
static int zend_jit_trace_halt_stub(zend_jit_ctx *jit)
{
if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
ir_TAILCALL(IR_VOID, ir_CONST_OPCODE_HANDLER_FUNC(zend_jit_halt_op->handler));
} else if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, IR_NULL);
ir_RETURN(IR_VOID);
@@ -2432,8 +2484,8 @@ static int zend_jit_trace_halt_stub(zend_jit_ctx *jit)
static int zend_jit_trace_escape_stub(zend_jit_ctx *jit)
{
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit)));
} else {
zend_jit_vm_enter(jit, jit_IP(jit));
}
@@ -2459,8 +2511,8 @@ static int zend_jit_trace_exit_stub(zend_jit_ctx *jit)
ref = ir_LOAD_A(jit_EX(opline));
jit_STORE_IP(jit, ref);
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit)));
} else {
zend_jit_vm_enter(jit, ref);
}
@@ -2479,11 +2531,11 @@ static int zend_jit_trace_exit_stub(zend_jit_ctx *jit)
zend_jit_check_timeout(jit, NULL, NULL);
addr = zend_jit_orig_opline_handler(jit);
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, addr);
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_jit_tailcall_handler(jit, addr);
} else {
#if defined(IR_TARGET_X86)
addr = ir_CAST_FC_FUNC(addr);
addr = ir_CAST_OPCODE_HANDLER_FUNC(addr);
#endif
ref = ir_CALL_2(IR_ADDR, addr, jit_FP(jit), jit_IP(jit));
zend_jit_vm_enter(jit, ref);
@@ -2497,7 +2549,7 @@ static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit)
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key));
} else {
ir_TAILCALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit), jit_IP(jit));
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit));
}
return 1;
@@ -2508,7 +2560,7 @@ static int zend_jit_undefined_key_stub(zend_jit_ctx *jit)
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key));
} else {
ir_TAILCALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit), jit_IP(jit));
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit));
}
return 1;
@@ -2676,11 +2728,13 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
jit->ctx.fixed_regset = (1<<ZREG_FP) | (1<<ZREG_IP);
if (!(flags & IR_FUNCTION)) {
jit->ctx.flags |= IR_NO_STACK_COMBINE;
if (ZEND_VM_KIND == ZEND_VM_KIND_CALL) {
if (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
jit->ctx.flags |= IR_FUNCTION;
/* Stack must be 16 byte aligned */
/* TODO: select stack size ??? */
#if defined(IR_TARGET_AARCH64)
#if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 5; /* 5 spill slots (8 bytes) or 10 spill slots (4 bytes) */
#elif defined(IR_TARGET_AARCH64)
jit->ctx.flags |= IR_USE_FRAME_POINTER;
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 bytes) */
#elif defined(_WIN64)
@@ -2696,6 +2750,23 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
*/
if (GCC_GLOBAL_REGS) {
jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<<ZREG_FP) | (1<<ZREG_IP));
} else if (ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
/* The only preserved register on x86 is RBP:
* https://github.com/llvm/llvm-project/blob/a414877a7a5f000d01370acb1162eb1dea87f48c/llvm/lib/Target/X86/X86RegisterInfo.cpp#L319
* https://github.com/llvm/llvm-project/blob/68bfe91b5a34f80dbcc4f0a7fa5d7aa1cdf959c2/llvm/lib/Target/X86/X86CallingConv.td#L1183
* On AArch64 it's LR, FP:
* https://github.com/llvm/llvm-project/blob/68bfe91b5a34f80dbcc4f0a7fa5d7aa1cdf959c2/llvm/lib/Target/AArch64/AArch64CallingConvention.td#L681
*
* Add them to the fixed_regset to prevent usage or these regs.
* It's cheaper to not use them than to save them.
*/
#if defined(IR_TARGET_X64)
jit->ctx.fixed_regset |= (1<<IR_REG_FP);
#elif defined(IR_TARGET_AARCH64)
jit->ctx.fixed_regset |= (1<<IR_REG_FP) | (1<<IR_REG_LR);
#else
ZEND_UNREACHABLE();
#endif
} else {
jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED;
//#ifdef _WIN64
@@ -3146,13 +3217,15 @@ static void zend_jit_calc_trace_prologue_size(void)
void *entry;
size_t size;
zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) ? 0 : IR_START_BR_TARGET);
if (!GCC_GLOBAL_REGS) {
ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1);
ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2);
jit_STORE_FP(jit, execute_data_ref);
jit_STORE_IP(jit, opline_ref);
if (ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) {
ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1);
ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2);
jit_STORE_FP(jit, execute_data_ref);
jit_STORE_IP(jit, opline_ref);
}
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
@@ -3983,7 +4056,7 @@ static void zend_jit_recv_entry(zend_jit_ctx *jit, int b)
/* Insert a MERGE block with additional ENTRY input between predecessor and this one */
ir_ENTRY(ref, bb->start);
if (!GCC_GLOBAL_REGS) {
if (!GCC_GLOBAL_REGS && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) {
/* 2 and 3 are hardcoded reference to IR_PARAMs */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
@@ -4004,7 +4077,7 @@ static void zend_jit_osr_entry(zend_jit_ctx *jit, int b)
/* Insert a MERGE block with additional ENTRY input between predecessor and this one */
ir_ENTRY(ref, bb->start);
if (!GCC_GLOBAL_REGS) {
if (!GCC_GLOBAL_REGS && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) {
/* 2 and 3 are hardcoded reference to IR_PARAMs */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
@@ -4020,7 +4093,7 @@ static void zend_jit_osr_entry(zend_jit_ctx *jit, int b)
static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label)
{
ir_ENTRY(src, label);
if (!GCC_GLOBAL_REGS) {
if (!GCC_GLOBAL_REGS && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) {
/* 2 and 3 are hardcoded reference to IR_PARAMs */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
@@ -4038,6 +4111,10 @@ static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_th
if (GCC_GLOBAL_REGS) {
zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(opline);
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
} else if (ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(opline);
ir_ref ip = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit));
jit_STORE_IP(jit, ip);
} else {
zend_vm_opcode_handler_t handler = opline->handler;
ir_ref ip = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit));
@@ -4090,8 +4167,8 @@ static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline)
}
} else {
zend_vm_opcode_handler_t handler = opline->handler;
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_jit_tailcall_handler(jit, ir_CONST_OPCODE_HANDLER_FUNC(handler));
} else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)
&& (opline->opcode == ZEND_CATCH
|| opline->opcode == ZEND_FAST_CALL
@@ -4099,10 +4176,10 @@ static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline)
|| opline->opcode == ZEND_MATCH_ERROR
|| opline->opcode == ZEND_THROW
|| opline->opcode == ZEND_VERIFY_NEVER_TYPE)) {
ir_ref ip = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit));
ir_ref ip = ir_CALL_2(IR_ADDR, ir_CONST_OPCODE_HANDLER_FUNC(handler), jit_FP(jit), jit_IP(jit));
zend_jit_vm_enter(jit, ip);
} else {
ir_TAILCALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit));
ir_TAILCALL_2(IR_ADDR, ir_CONST_OPCODE_HANDLER_FUNC(handler), jit_FP(jit), jit_IP(jit));
}
}
if (jit->b >= 0) {
@@ -10205,6 +10282,8 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
}
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, helper);
} else if (ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
ir_CALL_2(IR_ADDR, helper, jit_FP(jit), jit_IP(jit));
} else {
ir_ref ref = ir_CALL_2(IR_ADDR, helper, jit_FP(jit), jit_IP(jit));
jit_STORE_IP(jit, ref);
@@ -10379,8 +10458,8 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
}
}
/* fallback to indirect JMP or RETURN */
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit)));
} else {
zend_jit_vm_enter(jit, jit_IP(jit));
}
@@ -11147,8 +11226,8 @@ static int zend_jit_leave_func(zend_jit_ctx *jit,
jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
}
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit)));
} else {
zend_jit_vm_leave(jit, jit_IP(jit));
}
@@ -16578,7 +16657,7 @@ static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend
int i, count;
zend_basic_block *bb;
zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_BR_TARGET));
zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_BR_TARGET));
jit->ctx.spill_base = ZREG_FP;
@@ -16595,10 +16674,12 @@ static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend
jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref));
if (!GCC_GLOBAL_REGS) {
ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1);
ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2);
jit_STORE_FP(jit, execute_data_ref);
jit_STORE_IP(jit, opline_ref);
if (ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) {
ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1);
ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2);
jit_STORE_FP(jit, execute_data_ref);
jit_STORE_IP(jit, opline_ref);
}
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
@@ -16973,8 +17054,8 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
size_t offset = jit_extension->offset;
const void *handler =
(zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
zend_vm_opcode_handler_func_t handler =
ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
ir_ref ref;
zend_jit_set_ip(jit, opline);
@@ -16987,7 +17068,8 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_GENERATOR_CREATE) {
opline->opcode == ZEND_GENERATOR_CREATE ||
opline->opcode == ZEND_INCLUDE_OR_EVAL) {
jit_STORE_IP(jit, ir_AND_A(ref, ir_CONST_ADDR(~ZEND_VM_ENTER_BIT)));
} else {
@@ -17004,14 +17086,16 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
trace++;
}
if (!GCC_GLOBAL_REGS
&& (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
if ((!GCC_GLOBAL_REGS
&& (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN))
|| ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_GENERATOR_CREATE) {
opline->opcode == ZEND_GENERATOR_CREATE ||
opline->opcode == ZEND_INCLUDE_OR_EVAL) {
ir_ref addr = jit_EG(current_execute_data);
@@ -17138,7 +17222,7 @@ static int zend_jit_deoptimizer_start(zend_jit_ctx *jit,
uint32_t trace_num,
uint32_t exit_num)
{
zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) ? 0 : IR_START_BR_TARGET);
jit->ctx.spill_base = ZREG_FP;
@@ -17159,7 +17243,7 @@ static int zend_jit_trace_start(zend_jit_ctx *jit,
zend_jit_trace_info *parent,
uint32_t exit_num)
{
zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) ? 0 : IR_START_BR_TARGET);
jit->ctx.spill_base = ZREG_FP;
@@ -17170,10 +17254,12 @@ static int zend_jit_trace_start(zend_jit_ctx *jit,
if (!GCC_GLOBAL_REGS) {
if (!parent) {
ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1);
ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2);
jit_STORE_FP(jit, execute_data_ref);
jit_STORE_IP(jit, opline_ref);
if (ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) {
ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1);
ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2);
jit_STORE_FP(jit, execute_data_ref);
jit_STORE_IP(jit, opline_ref);
}
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
}
@@ -17276,11 +17362,11 @@ static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *
static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline)
{
if (GCC_GLOBAL_REGS) {
if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) {
if (!original_handler) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit)));
} else {
ir_TAILCALL(IR_VOID, zend_jit_orig_opline_handler(jit));
zend_jit_tailcall_handler(jit, zend_jit_orig_opline_handler(jit));
}
} else {
if (original_handler) {

1
ext/opcache/jit/zend_jit_trace.c
View File

@@ -20,6 +20,7 @@
#include "zend_jit_internal.h"
#include "zend_shared_alloc.h"
#include "ir/ir.h"
#include "zend_vm_opcodes.h"
static zend_jit_trace_info *zend_jit_traces = NULL;
static const void **zend_jit_exit_groups = NULL;

130
ext/opcache/jit/zend_jit_vm_helpers.c
View File

@@ -46,6 +46,60 @@ register const zend_op* volatile opline __asm__("x28");
# pragma GCC diagnostic warning "-Wvolatile-register-var"
#endif
#if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_leave_func_helper_tailcall(ZEND_OPCODE_HANDLER_ARGS)
{
zend_execute_data *old_execute_data;
uint32_t call_info = EX_CALL_INFO();
if (!(call_info & ZEND_CALL_TOP)) {
if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
zend_clean_and_cache_symbol_table(EX(symbol_table));
}
zend_vm_stack_free_extra_args_ex(call_info, execute_data);
if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) {
OBJ_RELEASE(Z_OBJ(execute_data->This));
} else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
}
if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
zend_free_extra_named_params(EX(extra_named_params));
}
old_execute_data = execute_data;
execute_data = EX(prev_execute_data);
zend_vm_stack_free_call_frame_ex(call_info, old_execute_data);
if (UNEXPECTED(EG(exception) != NULL)) {
const zend_op *old_opline = EX(opline);
zend_throw_exception_internal(NULL);
if (old_opline->result_type != IS_UNDEF) {
zval_ptr_dtor(EX_VAR(old_opline->result.var));
}
opline = EG(current_execute_data)->opline;
} else {
opline = ++EX(opline);
}
ZEND_OPCODE_TAIL_CALL(opline->handler);
} else {
if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) {
if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
zend_clean_and_cache_symbol_table(EX(symbol_table));
}
zend_vm_stack_free_extra_args_ex(call_info, execute_data);
}
if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
zend_free_extra_named_params(EX(extra_named_params));
}
if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
}
return (zend_op*)ZEND_VM_ENTER_BIT;
}
}
#endif
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info)
{
zend_execute_data *old_execute_data;
@@ -75,14 +129,16 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(ZEND_OPC
zval_ptr_dtor(EX_VAR(old_opline->result.var));
}
#ifndef HAVE_GCC_GLOBAL_REGS
return (zend_op*)((uintptr_t)EG(current_execute_data)->opline | ZEND_VM_ENTER_BIT);
return EG(current_execute_data)->opline;
#endif
} else {
EX(opline)++;
#ifdef HAVE_GCC_GLOBAL_REGS
opline = EX(opline);
#elif ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
return EX(opline);
#else
return (zend_op*)((uintptr_t)EX(opline) | ZEND_VM_ENTER_BIT);
return (const zend_op*)((uintptr_t)EX(opline) | ZEND_VM_ENTER_BIT);
#endif
}
}
@@ -101,11 +157,11 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(ZEND_OPCODE
if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
}
execute_data = EG(current_execute_data);
#ifdef HAVE_GCC_GLOBAL_REGS
execute_data = EG(current_execute_data);
opline = zend_jit_halt_op;
#else
return (const zend_op*)ZEND_VM_ENTER_BIT; // ZEND_VM_RETURN
return (const zend_op*)ZEND_VM_ENTER_BIT;
#endif
}
@@ -114,9 +170,9 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(ZEND_OPCODE_HAN
uint32_t call_info = EX_CALL_INFO();
if (call_info & ZEND_CALL_TOP) {
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info);
return zend_jit_leave_top_func_helper(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX call_info);
} else {
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info);
return zend_jit_leave_nested_func_helper(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX call_info);
}
}
@@ -298,18 +354,18 @@ void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D)
ZVAL_NULL(result);
}
#if ZEND_VM_KIND != ZEND_VM_KIND_HYBRID
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)
#if ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)
{
zend_op_array *op_array = (zend_op_array*)EX(func);
zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t) jit_extension->orig_handler;
zend_vm_opcode_handler_t handler = jit_extension->orig_handler;
++*(uintptr_t*)(EX(run_time_cache) + zend_jit_profile_counter_rid);
++zend_jit_profile_counter;
ZEND_OPCODE_TAIL_CALL(handler);
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
{
zend_jit_op_array_hot_extension *jit_extension =
(zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
@@ -326,7 +382,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_H
}
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
{
zend_jit_op_array_hot_extension *jit_extension =
(zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
@@ -338,7 +394,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_H
zend_jit_hot_func(execute_data, opline);
ZEND_OPCODE_RETURN();
} else {
zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
zend_vm_opcode_handler_t handler = jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
ZEND_OPCODE_TAIL_CALL(handler);
}
}
@@ -404,15 +460,13 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key)
return _zend_quick_get_constant(key, 0, 1);
}
#if ZEND_VM_KIND != ZEND_VM_KIND_HYBRID
static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t cost)
#if ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_trace_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
{
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
size_t offset = jit_extension->offset;
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT;
if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
@@ -427,31 +481,55 @@ static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_c
opline = execute_data ? EX(opline) : NULL;
#ifdef HAVE_GCC_GLOBAL_REGS
return;
#elif ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
if (EXPECTED(opline)) {
ZEND_OPCODE_TAIL_CALL((zend_vm_opcode_handler_t)opline->handler);
} else {
return (const zend_op*)ZEND_VM_ENTER_BIT;
}
#else
return (const zend_op*)((uintptr_t)opline | ZEND_VM_ENTER_BIT); // ZEND_VM_ENTER() / ZEND_VM_RETURN()
#endif
} else {
zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
zend_vm_opcode_handler_t handler = ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
ZEND_OPCODE_TAIL_CALL(handler);
}
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
size_t offset = jit_extension->offset;
uint32_t cost = ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func));
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
ZEND_OPCODE_TAIL_CALL(zend_jit_trace_counter_helper);
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
size_t offset = jit_extension->offset;
uint32_t cost = ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return));
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
ZEND_OPCODE_TAIL_CALL(zend_jit_trace_counter_helper);
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
size_t offset = jit_extension->offset;
uint32_t cost = ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop));
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
ZEND_OPCODE_TAIL_CALL(zend_jit_trace_counter_helper);
}
#endif

2
sapi/phpdbg/phpdbg_prompt.c
View File

@@ -41,7 +41,7 @@
#include "phpdbg_lexer.h"
#include "phpdbg_parser.h"
#if ZEND_VM_KIND != ZEND_VM_KIND_CALL && ZEND_VM_KIND != ZEND_VM_KIND_HYBRID
#if ZEND_VM_KIND != ZEND_VM_KIND_CALL && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL && ZEND_VM_KIND != ZEND_VM_KIND_HYBRID
#error "phpdbg can only be built with CALL zend vm kind"
#endif