php/archived-php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2026-03-24 00:02:20 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
76d7c616bb8775520f18456b41121bf934cac391
archived-php-src/ext/opcache/jit/zend_jit_vm_helpers.c
Arnaud Le Blanc 76d7c616bb Pass opline as argument to opcode handlers in CALL VM 
This changes the signature of opcode handlers in the CALL VM so that the opline
is passed directly via arguments. This reduces the number of memory operations
on EX(opline), and makes the CALL VM considerably faster.

Additionally, this unifies the CALL and HYBRID VMs a bit, as EX(opline) is now
handled in the same way in both VMs.

This is a part of GH-17849.

Currently we have two VMs:

 * HYBRID: Used when compiling with GCC. execute_data and opline are global
   register variables
 * CALL: Used when compiling with something else. execute_data is passed as
   opcode handler arg, but opline is passed via execute_data->opline
   (EX(opline)).

The Call VM looks like this:

    while (1) {
        ret = execute_data->opline->handler(execute_data);
        if (UNEXPECTED(ret != 0)) {
            if (ret > 0) { // returned by ZEND_VM_ENTER() / ZEND_VM_LEAVE()
                execute_data = EG(current_execute_data);
            } else {       // returned by ZEND_VM_RETURN()
                return;
            }
        }
    }

    // example op handler
    int ZEND_INIT_FCALL_SPEC_CONST_HANDLER(zend_execute_data *execute_data) {
        // load opline
        const zend_op *opline = execute_data->opline;

        // instruction execution

        // dispatch
        // ZEND_VM_NEXT_OPCODE():
        execute_data->opline++;
        return 0; // ZEND_VM_CONTINUE()
    }

Opcode handlers return a positive value to signal that the loop must load a
new execute_data from EG(current_execute_data), typically when entering
or leaving a function.

Here I make the following changes:

 * Pass opline as opcode handler argument
 * Return next opline from opcode handlers
 * ZEND_VM_ENTER / ZEND_VM_LEAVE return opline|(1<<0) to signal that
   execute_data must be reloaded from EG(current_execute_data)

This gives us:

    while (1) {
        opline = opline->handler(execute_data, opline);
        if (UNEXPECTED((uintptr_t) opline & ZEND_VM_ENTER_BIT) {
            opline = opline & ~ZEND_VM_ENTER_BIT;
            if (opline != 0) { // ZEND_VM_ENTER() / ZEND_VM_LEAVE()
                execute_data = EG(current_execute_data);
            } else {           // ZEND_VM_RETURN()
                return;
            }
        }
    }

    // example op handler
    const zend_op * ZEND_INIT_FCALL_SPEC_CONST_HANDLER(zend_execute_data *execute_data, const zend_op *opline) {
        // opline already loaded

        // instruction execution

        // dispatch
        // ZEND_VM_NEXT_OPCODE():
        return ++opline;
    }

bench.php is 23% faster on Linux / x86_64, 18% faster on MacOS / M1.

Symfony Demo is 2.8% faster.

When using the HYBRID VM, JIT'ed code stores execute_data/opline in two fixed
callee-saved registers and rarely touches EX(opline), just like the VM.

Since the registers are callee-saved, the JIT'ed code doesn't have to
save them before calling other functions, and can assume they always
contain execute_data/opline. The code also avoids saving/restoring them in
prologue/epilogue, as execute_ex takes care of that (JIT'ed code is called
exclusively from there).

The CALL VM can now use a fixed register for execute_data/opline as well, but
we can't rely on execute_ex to save the registers for us as it may use these
registers itself. So we have to save/restore the two registers in JIT'ed code
prologue/epilogue.

Closes GH-17952
2025-04-15 18:51:54 +02:00

1323 lines
42 KiB
C
Raw Blame History

/*
+----------------------------------------------------------------------+
| Zend JIT |
+----------------------------------------------------------------------+
| Copyright (c) The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
| available through the world-wide-web at the following url: |
| https://www.php.net/license/3_01.txt |
| If you did not receive a copy of the PHP license and are unable to |
| obtain it through the world-wide-web, please send a note to |
| license@php.net so we can mail you a copy immediately. |
+----------------------------------------------------------------------+
| Authors: Dmitry Stogov <dmitry@php.net> |
| Xinchen Hui <laruence@php.net> |
+----------------------------------------------------------------------+
*/
#include "Zend/zend_execute.h"
#include "Zend/zend_exceptions.h"
#include "Zend/zend_vm.h"
#include "Zend/zend_closures.h"
#include "Zend/zend_constants.h"
#include "Zend/zend_API.h"
#include <ZendAccelerator.h>
#include "Optimizer/zend_func_info.h"
#include "Optimizer/zend_call_graph.h"
#include "zend_jit.h"
#include "zend_jit_internal.h"
#ifdef HAVE_GCC_GLOBAL_REGS
# pragma GCC diagnostic ignored "-Wvolatile-register-var"
# if defined(__x86_64__)
register zend_execute_data* volatile execute_data __asm__("%r14");
register const zend_op* volatile opline __asm__("%r15");
# elif defined(i386)
register zend_execute_data* volatile execute_data __asm__("%esi");
register const zend_op* volatile opline __asm__("%edi");
# elif defined(__aarch64__)
register zend_execute_data* volatile execute_data __asm__("x27");
register const zend_op* volatile opline __asm__("x28");
# endif
# pragma GCC diagnostic warning "-Wvolatile-register-var"
#endif
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info)
{
zend_execute_data *old_execute_data;
if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
zend_clean_and_cache_symbol_table(EX(symbol_table));
}
zend_vm_stack_free_extra_args_ex(call_info, execute_data);
if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) {
OBJ_RELEASE(Z_OBJ(execute_data->This));
} else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
}
if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
zend_free_extra_named_params(EX(extra_named_params));
}
old_execute_data = execute_data;
execute_data = EX(prev_execute_data);
zend_vm_stack_free_call_frame_ex(call_info, old_execute_data);
if (UNEXPECTED(EG(exception) != NULL)) {
const zend_op *old_opline = EX(opline);
zend_throw_exception_internal(NULL);
if (old_opline->result_type != IS_UNDEF) {
zval_ptr_dtor(EX_VAR(old_opline->result.var));
}
#ifndef HAVE_GCC_GLOBAL_REGS
return (zend_op*)((uintptr_t)EG(current_execute_data)->opline | ZEND_VM_ENTER_BIT);
#endif
} else {
EX(opline)++;
#ifdef HAVE_GCC_GLOBAL_REGS
opline = EX(opline);
#else
return (zend_op*)((uintptr_t)EX(opline) | ZEND_VM_ENTER_BIT);
#endif
}
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info)
{
if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) {
if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
zend_clean_and_cache_symbol_table(EX(symbol_table));
}
zend_vm_stack_free_extra_args_ex(call_info, execute_data);
}
if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
zend_free_extra_named_params(EX(extra_named_params));
}
if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
}
execute_data = EG(current_execute_data);
#ifdef HAVE_GCC_GLOBAL_REGS
opline = zend_jit_halt_op;
#else
return (const zend_op*)ZEND_VM_ENTER_BIT; // ZEND_VM_RETURN
#endif
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(ZEND_OPCODE_HANDLER_ARGS)
{
uint32_t call_info = EX_CALL_INFO();
if (call_info & ZEND_CALL_TOP) {
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info);
} else {
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info);
}
}
static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper_ex(ZEND_OPCODE_HANDLER_ARGS_EX bool skip_recv)
{
zend_op_array *op_array = &EX(func)->op_array;
if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) {
uint32_t first_extra_arg = op_array->num_args;
uint32_t num_args = EX_NUM_ARGS();
zval *end, *src, *dst;
uint32_t type_flags = 0;
if (skip_recv && EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) {
/* Skip useless ZEND_RECV and ZEND_RECV_INIT opcodes */
opline += first_extra_arg;
}
/* move extra args into separate array after all CV and TMP vars */
end = EX_VAR_NUM(first_extra_arg - 1);
src = end + (num_args - first_extra_arg);
dst = src + (op_array->last_var + op_array->T - first_extra_arg);
if (EXPECTED(src != dst)) {
do {
type_flags |= Z_TYPE_INFO_P(src);
ZVAL_COPY_VALUE(dst, src);
ZVAL_UNDEF(src);
src--;
dst--;
} while (src != end);
if (type_flags & (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT)) {
ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
}
} else {
do {
if (Z_REFCOUNTED_P(src)) {
ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
break;
}
src--;
} while (src != end);
}
}
#ifndef HAVE_GCC_GLOBAL_REGS
return opline;
#endif
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper(ZEND_OPCODE_HANDLER_ARGS)
{
return zend_jit_copy_extra_args_helper_ex(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX true);
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper_no_skip_recv(ZEND_OPCODE_HANDLER_ARGS)
{
return zend_jit_copy_extra_args_helper_ex(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX false);
}
bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D)
{
zend_execute_data *call = (zend_execute_data *) opline;
zend_function *fbc = call->func;
zend_deprecated_function(fbc);
if (EG(exception)) {
#ifndef HAVE_GCC_GLOBAL_REGS
zend_execute_data *execute_data = EG(current_execute_data);
#endif
const zend_op *opline = EG(opline_before_exception);
if (opline && RETURN_VALUE_USED(opline)) {
ZVAL_UNDEF(EX_VAR(opline->result.var));
}
zend_vm_stack_free_args(call);
if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) {
OBJ_RELEASE(Z_OBJ(call->This));
}
zend_vm_stack_free_call_frame(call);
return 0;
}
return 1;
}
bool ZEND_FASTCALL zend_jit_nodiscard_helper(OPLINE_D)
{
zend_execute_data *call = (zend_execute_data *) opline;
zend_function *fbc = call->func;
zend_nodiscard_function(fbc);
if (EG(exception)) {
#ifndef HAVE_GCC_GLOBAL_REGS
zend_execute_data *execute_data = EG(current_execute_data);
#endif
const zend_op *opline = EG(opline_before_exception);
if (opline && RETURN_VALUE_USED(opline)) {
ZVAL_UNDEF(EX_VAR(opline->result.var));
}
zend_vm_stack_free_args(call);
if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) {
OBJ_RELEASE(Z_OBJ(call->This));
}
zend_vm_stack_free_call_frame(call);
return 0;
}
return 1;
}
bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(OPLINE_D)
{
zend_execute_data *call = (zend_execute_data *) opline;
zend_function *fbc = call->func;
if (fbc->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (zend_jit_deprecated_helper(OPLINE_C) == 0) {
return 0;
}
}
if (fbc->common.fn_flags & ZEND_ACC_NODISCARD) {
if (zend_jit_nodiscard_helper(OPLINE_C) == 0) {
return 0;
}
}
return 1;
}
void ZEND_FASTCALL zend_jit_undefined_long_key(EXECUTE_DATA_D)
{
const zend_op *opline = EX(opline);
zval *result = EX_VAR(opline->result.var);
zval *dim;
if (opline->op2_type == IS_CONST) {
dim = RT_CONSTANT(opline, opline->op2);
} else {
dim = EX_VAR(opline->op2.var);
}
ZEND_ASSERT(Z_TYPE_P(dim) == IS_LONG);
zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, Z_LVAL_P(dim));
ZVAL_NULL(result);
}
void ZEND_FASTCALL zend_jit_undefined_long_key_ex(zend_long key EXECUTE_DATA_DC)
{
const zend_op *opline = EX(opline);
zval *result = EX_VAR(opline->result.var);
zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, key);
ZVAL_NULL(result);
}
void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D)
{
const zend_op *opline = EX(opline);
zval *result = EX_VAR(opline->result.var);
zval *dim;
zend_ulong lval;
if (opline->op2_type == IS_CONST) {
dim = RT_CONSTANT(opline, opline->op2);
} else {
dim = EX_VAR(opline->op2.var);
}
ZEND_ASSERT(Z_TYPE_P(dim) == IS_STRING);
if (ZEND_HANDLE_NUMERIC(Z_STR_P(dim), lval)) {
zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, lval);
} else {
zend_error(E_WARNING, "Undefined array key \"%s\"", Z_STRVAL_P(dim));
}
ZVAL_NULL(result);
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)
{
zend_op_array *op_array = (zend_op_array*)EX(func);
zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t) jit_extension->orig_handler;
++*(uintptr_t*)(EX(run_time_cache) + zend_jit_profile_counter_rid);
++zend_jit_profile_counter;
ZEND_OPCODE_TAIL_CALL(handler);
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
{
zend_jit_op_array_hot_extension *jit_extension =
(zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
*(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func));
if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
*(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
zend_jit_hot_func(execute_data, opline);
ZEND_OPCODE_RETURN();
} else {
zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
ZEND_OPCODE_TAIL_CALL(handler);
}
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
{
zend_jit_op_array_hot_extension *jit_extension =
(zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
*(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop));
if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
*(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
zend_jit_hot_func(execute_data, opline);
ZEND_OPCODE_RETURN();
} else {
zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
ZEND_OPCODE_TAIL_CALL(handler);
}
}
static zend_always_inline zend_constant* _zend_quick_get_constant(
const zval *key, uint32_t flags, int check_defined_only)
{
#ifndef HAVE_GCC_GLOBAL_REGS
zend_execute_data *execute_data = EG(current_execute_data);
#endif
const zend_op *opline = EX(opline);
zval *zv;
zend_constant *c = NULL;
/* null/true/false are resolved during compilation, so don't check for them here. */
zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
if (zv) {
c = (zend_constant*)Z_PTR_P(zv);
} else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
key++;
zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
if (zv) {
c = (zend_constant*)Z_PTR_P(zv);
}
}
if (!c) {
if (!check_defined_only) {
zend_throw_error(NULL, "Undefined constant \"%s\"", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)));
ZVAL_UNDEF(EX_VAR(opline->result.var));
}
CACHE_PTR(opline->extended_value, ENCODE_SPECIAL_CACHE_NUM(zend_hash_num_elements(EG(zend_constants))));
return NULL;
}
if (!check_defined_only) {
if (ZEND_CONSTANT_FLAGS(c) & CONST_DEPRECATED) {
zend_error(E_DEPRECATED, "Constant %s is deprecated", ZSTR_VAL(c->name));
if (EG(exception)) {
return NULL;
}
return c;
}
}
CACHE_PTR(opline->extended_value, c);
return c;
}
zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags)
{
return _zend_quick_get_constant(key, flags, 0);
}
zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key)
{
return _zend_quick_get_constant(key, 0, 1);
}
static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t cost)
{
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
size_t offset = jit_extension->offset;
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT;
if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
#ifdef HAVE_GCC_GLOBAL_REGS
opline = NULL;
return;
#else
return (const zend_op*)ZEND_VM_ENTER_BIT; // ZEND_VM_RETURN()
#endif
}
execute_data = EG(current_execute_data);
opline = execute_data ? EX(opline) : NULL;
#ifdef HAVE_GCC_GLOBAL_REGS
return;
#else
return (const zend_op*)((uintptr_t)opline | ZEND_VM_ENTER_BIT); // ZEND_VM_ENTER() / ZEND_VM_RETURN()
#endif
} else {
zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
ZEND_OPCODE_TAIL_CALL(handler);
}
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
}
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
#define TRACE_RECORD(_op, _info, _ptr) \
trace_buffer[idx].info = _op | (_info); \
trace_buffer[idx].ptr = _ptr; \
idx++; \
if (idx >= JIT_G(max_trace_length) - 2) { \
stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
break; \
}
#define TRACE_RECORD_VM(_op, _ptr, _op1_type, _op2_type, _op3_type) \
trace_buffer[idx].op = _op; \
trace_buffer[idx].op1_type = _op1_type; \
trace_buffer[idx].op2_type = _op2_type; \
trace_buffer[idx].op3_type = _op3_type; \
trace_buffer[idx].ptr = _ptr; \
idx++; \
if (idx >= JIT_G(max_trace_length) - 2) { \
stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
break; \
}
#define TRACE_START(_op, _start, _ptr1, _ptr2) \
trace_buffer[0].op = _op; \
trace_buffer[0].start = _start; \
trace_buffer[0].level = 0; \
trace_buffer[0].ptr = _ptr1; \
trace_buffer[1].last = 0; \
trace_buffer[1].ptr = _ptr2; \
idx = ZEND_JIT_TRACE_START_REC_SIZE;
#define TRACE_END(_op, _stop, _ptr) \
trace_buffer[1].last = idx; \
trace_buffer[idx].op = _op; \
trace_buffer[idx].start = trace_buffer[idx].start; \
trace_buffer[idx].stop = trace_buffer[0].stop = _stop; \
trace_buffer[idx].level = trace_buffer[0].level = ret_level ? ret_level + 1 : 0; \
trace_buffer[idx].ptr = _ptr;
static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level)
{
int i;
int count = 0;
for (i = ret_level; i < level; i++) {
count += (unrolled_calls[i] == op_array);
}
return count;
}
static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level)
{
int i;
int count = 0;
for (i = 0; i < ret_level; i++) {
count += (unrolled_calls[i] == op_array);
}
return count;
}
static int zend_jit_trace_has_recursive_ret(zend_execute_data *ex, const zend_op_array *orig_op_array, const zend_op *orig_opline, int ret_level)
{
while (ex != NULL && ex->func != NULL && ret_level < ZEND_JIT_TRACE_MAX_RET_DEPTH) {
if (&ex->func->op_array == orig_op_array && ex->opline + 1 == orig_opline) {
return 1;
}
ex = ex->prev_execute_data;
ret_level++;
}
return 0;
}
static uint8_t zend_jit_trace_bad_stop_event(const zend_op *opline, int count)
{
const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
uint8_t *cache_count = JIT_G(bad_root_cache_count);
uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
uint32_t i;
if (count < 0) {
count = 0;
}
for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
if (cache_opline[i] == opline) {
if (cache_count[i] >= count) {
return cache_stop[i];
}
break;
}
}
return 0;
}
#define ZEND_CALL_MEGAMORPHIC ZEND_CALL_JIT_RESERVED
static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t init_level)
{
zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
do {
zend_function *func;
zend_jit_op_array_trace_extension *jit_extension;
if (call->prev_execute_data) {
idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, init_level + 1);
if (idx < 0) {
return idx;
}
}
func = call->func;
if (func->type == ZEND_INTERNAL_FUNCTION
&& (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
func = NULL;
} else if (func->type == ZEND_USER_FUNCTION) {
jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE))
|| (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))
|| (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) {
func = NULL;
} else if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) {
func = (zend_function*)jit_extension->op_array;
}
}
if (!func
|| (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)
|| (func->common.fn_flags & ZEND_ACC_NEVER_CACHE)
|| func->common.prop_info) {
/* continue recording */
func = NULL;
} else if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM
/* TODO: use more accurate check ??? */
&& ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
|| func->common.scope)) {
func = NULL;
ZEND_ADD_CALL_FLAG(call, ZEND_CALL_MEGAMORPHIC);
}
TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INFO(init_level), func);
} while (0);
return idx;
}
static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic)
{
return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, 0);
}
static int zend_jit_trace_subtrace(zend_jit_trace_rec *trace_buffer, int start, int end, uint8_t event, const zend_op_array *op_array, const zend_op *opline)
{
int idx;
TRACE_START(ZEND_JIT_TRACE_START, event, op_array, opline);
memmove(trace_buffer + idx, trace_buffer + start, (end - start) * sizeof(zend_jit_trace_rec));
return idx + (end - start);
}
/*
* Trace Linking Rules
* ===================
*
* flags
* +----------+----------+----------++----------+----------+----------+
* | || JIT |
* +----------+----------+----------++----------+----------+----------+
* start | LOOP | ENTER | RETURN || LOOP | ENTER | RETURN |
* +========+==========+==========+==========++==========+==========+==========+
* | LOOP | loop | | loop-ret || COMPILED | LINK | LINK |
* +--------+----------+----------+----------++----------+----------+----------+
* | ENTER |INNER_LOOP| rec-call | return || LINK | LINK | LINK |
* +--------+----------+----------+----------++----------+----------+----------+
* | RETURN |INNER_LOOP| | rec-ret || LINK | | LINK |
* +--------+----------+----------+----------++----------+----------+----------+
* | SIDE | unroll | | side-ret || LINK | LINK | LINK |
* +--------+----------+----------+----------++----------+----------+----------+
*
* loop: LOOP if "cycle" and level == 0, otherwise INNER_LOOP
* INNER_LOOP: abort recording and start new one (wait for loop)
* COMPILED: abort recording (wait while side exit creates outer loop)
* unroll: continue recording while unroll limit reached
* rec-call: RECURSIVE_CALL if "cycle" and level > N, otherwise continue
* loop-ret: LOOP_EXIT if level == 0, otherwise continue (wait for loop)
* return: RETURN if level == 0
* rec_ret: RECURSIVE_RET if "cycle" and ret_level > N, otherwise continue
* side_ret: RETURN if level == 0 && ret_level == ret_depth, otherwise continue
*
*/
zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
const zend_op *op,
zend_jit_trace_rec *trace_buffer,
uint8_t start,
uint32_t is_megamorphic,
int ret_depth)
{
#ifdef HAVE_GCC_GLOBAL_REGS
zend_execute_data *save_execute_data = execute_data;
const zend_op *save_opline = opline;
#endif
const zend_op *orig_opline, *end_opline;
zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
zend_jit_trace_stop halt = 0;
int level = 0;
int ret_level = 0;
zend_vm_opcode_handler_t handler;
const zend_op_array *op_array;
zend_jit_op_array_trace_extension *jit_extension;
size_t offset;
int idx, count;
uint8_t trace_flags, op1_type, op2_type, op3_type;
zend_class_entry *ce1, *ce2;
const zend_op *link_to_enter_opline = NULL;
int backtrack_link_to_enter = -1;
int backtrack_recursion = -1;
int backtrack_ret_recursion = -1;
int backtrack_ret_recursion_level = 0;
int loop_unroll_limit = 0;
int last_loop = -1;
int last_loop_level = -1;
const zend_op *last_loop_opline = NULL;
const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
zend_execute_data *prev_execute_data = ex;
#ifdef HAVE_GCC_GLOBAL_REGS
execute_data = ex;
opline = EX(opline) = op;
#else
zend_execute_data *execute_data = ex;
const zend_op *opline = EX(opline) = op;
#endif
zend_execute_data *prev_call = EX(call);
orig_opline = opline;
op_array = &EX(func)->op_array;
jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
offset = jit_extension->offset;
if (!op_array->function_name
|| (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
op_array = jit_extension->op_array;
}
TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline);
if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
/* Abort trace because of exception */
TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_EXCEPTION, opline);
#ifdef HAVE_GCC_GLOBAL_REGS
execute_data = save_execute_data;
opline = save_opline;
#endif
return ZEND_JIT_TRACE_STOP_EXCEPTION;
}
trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_NOT_SUPPORTED, opline);
#ifdef HAVE_GCC_GLOBAL_REGS
execute_data = save_execute_data;
opline = save_opline;
#endif
return ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
}
if (prev_call) {
int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic);
if (ret < 0) {
TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_BAD_FUNC, opline);
#ifdef HAVE_GCC_GLOBAL_REGS
execute_data = save_execute_data;
opline = save_opline;
#endif
return ZEND_JIT_TRACE_STOP_BAD_FUNC;
}
idx = ret;
}
while (1) {
ce1 = ce2 = NULL;
op1_type = op2_type = op3_type = IS_UNKNOWN;
if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV))
&& opline->opcode != ZEND_ROPE_ADD
&& opline->opcode != ZEND_ROPE_END
&& opline->opcode != ZEND_NEW
&& opline->opcode != ZEND_FETCH_CLASS_CONSTANT
&& opline->opcode != ZEND_INIT_STATIC_METHOD_CALL) {
zval *zv = EX_VAR(opline->op1.var);
op1_type = Z_TYPE_P(zv);
uint8_t flags = 0;
if (op1_type == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
op1_type = Z_TYPE_P(zv);
flags |= IS_TRACE_INDIRECT;
}
if (op1_type == IS_REFERENCE) {
zv = Z_REFVAL_P(zv);
op1_type = Z_TYPE_P(zv);
flags |= IS_TRACE_REFERENCE;
}
if (Z_TYPE_P(zv) == IS_OBJECT) {
ce1 = Z_OBJCE_P(zv);
} else if (Z_TYPE_P(zv) == IS_ARRAY) {
if (HT_IS_PACKED(Z_ARRVAL_P(zv))) {
flags |= IS_TRACE_PACKED;
}
}
op1_type |= flags;
} else if (opline->op1_type == IS_UNUSED && (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
uint32_t op1_flags = ZEND_VM_OP1_FLAGS(zend_get_opcode_flags(opline->opcode));
if ((op1_flags & ZEND_VM_OP_MASK) == ZEND_VM_OP_THIS) {
op1_type = IS_OBJECT;
ce1 = Z_OBJCE(EX(This));
}
}
if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)
&& opline->opcode != ZEND_INSTANCEOF
&& opline->opcode != ZEND_UNSET_STATIC_PROP
&& opline->opcode != ZEND_ISSET_ISEMPTY_STATIC_PROP
&& opline->opcode != ZEND_ASSIGN_STATIC_PROP
&& opline->opcode != ZEND_ASSIGN_STATIC_PROP_REF
&& opline->opcode != ZEND_ASSIGN_STATIC_PROP_OP
&& opline->opcode != ZEND_PRE_INC_STATIC_PROP
&& opline->opcode != ZEND_POST_INC_STATIC_PROP
&& opline->opcode != ZEND_PRE_DEC_STATIC_PROP
&& opline->opcode != ZEND_POST_DEC_STATIC_PROP
&& opline->opcode != ZEND_FETCH_STATIC_PROP_R
&& opline->opcode != ZEND_FETCH_STATIC_PROP_W
&& opline->opcode != ZEND_FETCH_STATIC_PROP_RW
&& opline->opcode != ZEND_FETCH_STATIC_PROP_IS
&& opline->opcode != ZEND_FETCH_STATIC_PROP_FUNC_ARG
&& opline->opcode != ZEND_FETCH_STATIC_PROP_UNSET
&& (opline->op2_type == IS_CV
|| (opline->opcode != ZEND_FE_FETCH_R
&& opline->opcode != ZEND_FE_FETCH_RW))) {
zval *zv = EX_VAR(opline->op2.var);
uint8_t flags = 0;
op2_type = Z_TYPE_P(zv);
if (op2_type == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
op2_type = Z_TYPE_P(zv);
flags |= IS_TRACE_INDIRECT;
}
if (op2_type == IS_REFERENCE) {
zv = Z_REFVAL_P(zv);
op2_type = Z_TYPE_P(zv);
flags |= IS_TRACE_REFERENCE;
}
if (Z_TYPE_P(zv) == IS_OBJECT) {
ce2 = Z_OBJCE_P(zv);
}
op2_type |= flags;
}
if (opline->opcode == ZEND_ASSIGN_DIM ||
opline->opcode == ZEND_ASSIGN_OBJ ||
opline->opcode == ZEND_ASSIGN_STATIC_PROP ||
opline->opcode == ZEND_ASSIGN_DIM_OP ||
opline->opcode == ZEND_ASSIGN_OBJ_OP ||
opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP ||
opline->opcode == ZEND_ASSIGN_OBJ_REF ||
opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF) {
if ((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
zval *zv = EX_VAR((opline+1)->op1.var);
uint8_t flags = 0;
op3_type = Z_TYPE_P(zv);
if (op3_type == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
op3_type = Z_TYPE_P(zv);
flags |= IS_TRACE_INDIRECT;
}
if (op3_type == IS_REFERENCE) {
zv = Z_REFVAL_P(zv);
op3_type = Z_TYPE_P(zv);
flags |= IS_TRACE_REFERENCE;
}
op3_type |= flags;
}
}
TRACE_RECORD_VM(ZEND_JIT_TRACE_VM, opline, op1_type, op2_type, op3_type);
if (ce1) {
TRACE_RECORD(ZEND_JIT_TRACE_OP1_TYPE, 0, ce1);
}
if (ce2) {
TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2);
}
switch (opline->opcode) {
case ZEND_FETCH_DIM_R:
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_DIM_RW:
case ZEND_FETCH_DIM_IS:
case ZEND_FETCH_DIM_FUNC_ARG:
case ZEND_FETCH_DIM_UNSET:
case ZEND_FETCH_LIST_R:
case ZEND_FETCH_LIST_W:
case ZEND_ASSIGN_DIM:
case ZEND_ASSIGN_DIM_OP:
case ZEND_UNSET_DIM:
case ZEND_ISSET_ISEMPTY_DIM_OBJ:
if (opline->op1_type == IS_CONST) {
zval *arr = RT_CONSTANT(opline, opline->op1);
op1_type = Z_TYPE_P(arr);
}
if ((op1_type & IS_TRACE_TYPE_MASK) == IS_ARRAY
&& opline->op2_type != IS_UNDEF) {
zval *arr, *dim, *val;
uint8_t val_type = IS_UNDEF;
if (opline->op2_type == IS_CONST) {
dim = RT_CONSTANT(opline, opline->op2);
} else {
dim = EX_VAR(opline->op2.var);
}
if (Z_TYPE_P(dim) == IS_LONG || Z_TYPE_P(dim) == IS_STRING) {
if (opline->op1_type == IS_CONST) {
arr = RT_CONSTANT(opline, opline->op1);
} else {
arr = EX_VAR(opline->op1.var);
}
if (Z_TYPE_P(arr) == IS_INDIRECT) {
arr = Z_INDIRECT_P(arr);
}
if (Z_TYPE_P(arr) == IS_REFERENCE) {
arr = Z_REFVAL_P(arr);
}
ZEND_ASSERT(Z_TYPE_P(arr) == IS_ARRAY);
if (Z_TYPE_P(dim) == IS_LONG) {
val = zend_hash_index_find(Z_ARRVAL_P(arr), Z_LVAL_P(dim));
} else /*if Z_TYPE_P(dim) == IS_STRING)*/ {
val = zend_symtable_find(Z_ARRVAL_P(arr), Z_STR_P(dim));
}
if (val) {
val_type = Z_TYPE_P(val);
}
TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, val_type, 0, 0);
}
}
break;
case ZEND_FETCH_OBJ_R: {
if (opline->op2_type == IS_CONST) {
/* Remove the SIMPLE_GET flag to avoid inlining hooks. */
void **cache_slot = CACHE_ADDR(opline->extended_value & ~ZEND_FETCH_REF);
uintptr_t prop_offset = (uintptr_t)CACHED_PTR_EX(cache_slot + 1);
if (IS_HOOKED_PROPERTY_OFFSET(prop_offset)) {
CACHE_PTR_EX(cache_slot + 1, (void*)((uintptr_t)CACHED_PTR_EX(cache_slot + 1) & ~ZEND_PROPERTY_HOOK_SIMPLE_GET_BIT)); \
}
}
ZEND_FALLTHROUGH;
}
case ZEND_FETCH_OBJ_W:
case ZEND_FETCH_OBJ_RW:
case ZEND_FETCH_OBJ_IS:
case ZEND_FETCH_OBJ_FUNC_ARG:
case ZEND_FETCH_OBJ_UNSET:
case ZEND_ASSIGN_OBJ:
case ZEND_ASSIGN_OBJ_OP:
case ZEND_ASSIGN_OBJ_REF:
case ZEND_UNSET_OBJ:
case ZEND_ISSET_ISEMPTY_PROP_OBJ:
case ZEND_PRE_INC_OBJ:
case ZEND_PRE_DEC_OBJ:
case ZEND_POST_INC_OBJ:
case ZEND_POST_DEC_OBJ:
if (opline->op1_type != IS_CONST
&& opline->op2_type == IS_CONST
&& Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
&& Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
zval *obj, *val;
zend_string *prop_name = Z_STR_P(RT_CONSTANT(opline, opline->op2));
zend_property_info *prop_info;
if (opline->op1_type == IS_UNUSED) {
obj = &EX(This);
} else {
obj = EX_VAR(opline->op1.var);
}
if (Z_TYPE_P(obj) != IS_OBJECT
|| Z_OBJ_P(obj)->handlers != &std_object_handlers) {
break;
}
prop_info = zend_get_property_info(Z_OBJCE_P(obj), prop_name, 1);
if (prop_info
&& prop_info != ZEND_WRONG_PROPERTY_INFO
&& !prop_info->hooks
&& !(prop_info->flags & ZEND_ACC_STATIC)) {
val = OBJ_PROP(Z_OBJ_P(obj), prop_info->offset);
TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, Z_TYPE_P(val), 0, 0);
}
}
break;
default:
break;
}
if (opline->opcode == ZEND_DO_FCALL
|| opline->opcode == ZEND_DO_ICALL
|| opline->opcode == ZEND_DO_UCALL
|| opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (ZEND_CALL_INFO(EX(call)) & ZEND_CALL_MEGAMORPHIC) {
stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
break;
}
if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
zend_function *func = EX(call)->func;
if ((func->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)
|| (func->common.fn_flags & ZEND_ACC_NEVER_CACHE)
|| func->common.prop_info) {
/* continue recording */
func = NULL;
} else if (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) {
stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
break;
}
TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, func);
}
} else if (opline->opcode == ZEND_INCLUDE_OR_EVAL
|| opline->opcode == ZEND_CALLABLE_CONVERT) {
/* TODO: Support tracing JIT for ZEND_CALLABLE_CONVERT. */
stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
break;
}
handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
#ifdef HAVE_GCC_GLOBAL_REGS
handler();
if (UNEXPECTED(opline == zend_jit_halt_op)) {
#else
opline = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);
if (UNEXPECTED(((uintptr_t)opline & ~ZEND_VM_ENTER_BIT) == 0)) {
#endif
stop = ZEND_JIT_TRACE_STOP_RETURN;
opline = NULL;
halt = ZEND_JIT_TRACE_HALT;
break;
}
#ifndef HAVE_GCC_GLOBAL_REGS
if ((uintptr_t)opline & ZEND_VM_ENTER_BIT) {
opline = (const zend_op*)((uintptr_t)opline & ~ZEND_VM_ENTER_BIT);
execute_data = EG(current_execute_data);
}
#endif
if (UNEXPECTED(execute_data != prev_execute_data)) {
op_array = &EX(func)->op_array;
jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
if (UNEXPECTED(!jit_extension)
|| UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) {
if (execute_data->prev_execute_data != prev_execute_data) {
stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
}
break;
}
offset = jit_extension->offset;
if (!op_array->function_name
|| (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
op_array = jit_extension->op_array;
}
if (execute_data->prev_execute_data == prev_execute_data) {
/* Enter into function */
prev_call = NULL;
if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) {
stop = ZEND_JIT_TRACE_STOP_TOO_DEEP;
break;
}
if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
break;
}
if (EX(func)->op_array.prop_info) {
stop = ZEND_JIT_TRACE_STOP_PROP_HOOK_CALL;
break;
}
if (EX(func)->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE) {
stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
break;
}
TRACE_RECORD(ZEND_JIT_TRACE_ENTER,
EX(return_value) != NULL ? ZEND_JIT_TRACE_RETURN_VALUE_USED : 0,
op_array);
count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level);
if (opline == orig_opline) {
if (count + 1 >= JIT_G(max_recursive_calls)) {
stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
break;
}
backtrack_recursion = idx;
} else if (count >= JIT_G(max_recursive_calls)) {
stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
break;
}
unrolled_calls[ret_level + level] = &EX(func)->op_array;
level++;
} else {
/* Return from function */
prev_call = EX(call);
if (level == 0) {
if (start == ZEND_JIT_TRACE_START_RETURN
&& JIT_G(max_recursive_returns) > 0
&& execute_data->prev_execute_data
&& execute_data->prev_execute_data->func
&& execute_data->prev_execute_data->func->type == ZEND_USER_FUNCTION
&& zend_jit_trace_has_recursive_ret(execute_data, trace_buffer[0].op_array, orig_opline, ret_level)) {
if (ret_level > ZEND_JIT_TRACE_MAX_RET_DEPTH) {
stop = ZEND_JIT_TRACE_STOP_TOO_DEEP_RET;
break;
}
TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level);
if (opline == orig_opline) {
if (count + 1 >= JIT_G(max_recursive_returns)) {
stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
break;
}
backtrack_ret_recursion = idx;
backtrack_ret_recursion_level = ret_level;
} else if (count >= JIT_G(max_recursive_returns)) {
stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
break;
}
unrolled_calls[ret_level] = &EX(func)->op_array;
ret_level++;
last_loop_opline = NULL;
if (prev_call) {
int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0);
if (ret < 0) {
stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
break;
}
idx = ret;
}
} else if (start & ZEND_JIT_TRACE_START_LOOP
&& zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
/* Fail to try close the loop.
If this doesn't work terminate it. */
stop = ZEND_JIT_TRACE_STOP_LOOP_EXIT;
break;
} else if (start & ZEND_JIT_TRACE_START_ENTER
&& EX(prev_execute_data)
&& EX(func) == EX(prev_execute_data)->func
&& zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
ZEND_JIT_TRACE_STOP_RECURSION_EXIT) {
stop = ZEND_JIT_TRACE_STOP_RECURSION_EXIT;
break;
} else if ((start & ZEND_JIT_TRACE_START_SIDE)
&& ret_level < ret_depth) {
TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
ret_level++;
last_loop_opline = NULL;
if (prev_call) {
int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0);
if (ret < 0) {
stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
break;
}
idx = ret;
}
} else {
stop = ZEND_JIT_TRACE_STOP_RETURN;
break;
}
} else {
level--;
if (level < last_loop_level) {
last_loop_opline = NULL;
}
TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
}
}
prev_execute_data = execute_data;
}
if (EX(call) != prev_call) {
if (EX(call)
&& EX(call)->prev_execute_data == prev_call) {
zend_function *func;
uint32_t info = 0;
zend_jit_op_array_trace_extension *jit_extension;
func = EX(call)->func;
if (func->type == ZEND_INTERNAL_FUNCTION
&& (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
func = NULL;
} else if (func->type == ZEND_USER_FUNCTION) {
jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE))
|| (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))
|| (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) {
func = NULL;
} else if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) {
func = (zend_function*)jit_extension->op_array;
}
}
if (!func
|| (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)
|| (func->common.fn_flags & ZEND_ACC_NEVER_CACHE)
|| func->common.prop_info) {
/* continue recording */
func = NULL;
} else if (JIT_G(max_polymorphic_calls) == 0
&& zend_jit_may_be_polymorphic_call(opline - 1)) {
func = NULL;
ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC);
} else if ((is_megamorphic == ZEND_JIT_EXIT_METHOD_CALL
|| is_megamorphic == ZEND_JIT_EXIT_CLOSURE_CALL)
&& trace_buffer[1].opline == opline - 1) {
func = NULL;
ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC);
}
if (!func) {
info = ZEND_JIT_TRACE_NUM_ARGS_INFO(ZEND_CALL_NUM_ARGS(EX(call)));
}
TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, info, func);
}
prev_call = EX(call);
}
if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
/* Abort trace because of exception */
stop = ZEND_JIT_TRACE_STOP_EXCEPTION;
break;
}
trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
if (trace_flags) {
if (trace_flags & ZEND_JIT_TRACE_JITED) {
if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
if ((start & ZEND_JIT_TRACE_START_LOOP) != 0
&& level + ret_level == 0
&& zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
ZEND_JIT_TRACE_STOP_COMPILED_LOOP) {
/* Fail to try close outer loop through side exit.
If this doesn't work just link. */
stop = ZEND_JIT_TRACE_STOP_COMPILED_LOOP;
break;
} else {
stop = ZEND_JIT_TRACE_STOP_LINK;
break;
}
} else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
if (start != ZEND_JIT_TRACE_START_RETURN) {
// TODO: We may try to inline function ???
stop = ZEND_JIT_TRACE_STOP_LINK;
break;
}
if (backtrack_link_to_enter < 0) {
backtrack_link_to_enter = idx;
link_to_enter_opline = opline;
}
} else {
stop = ZEND_JIT_TRACE_STOP_LINK;
break;
}
} else if (trace_flags & ZEND_JIT_TRACE_BLACKLISTED) {
stop = ZEND_JIT_TRACE_STOP_BLACK_LIST;
break;
} else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
uint8_t bad_stop;
if (start != ZEND_JIT_TRACE_START_SIDE) {
if (opline == orig_opline && level + ret_level == 0) {
stop = ZEND_JIT_TRACE_STOP_LOOP;
break;
}
}
if (start != ZEND_JIT_TRACE_START_SIDE
|| level + ret_level != 0) {
/* First try creating a trace for inner loop.
If this doesn't work try loop unroling. */
bad_stop = zend_jit_trace_bad_stop_event(opline,
JIT_G(blacklist_root_trace) / 2);
if (bad_stop != ZEND_JIT_TRACE_STOP_INNER_LOOP
&& bad_stop != ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
if (start == ZEND_JIT_TRACE_START_SIDE
|| zend_jit_trace_bad_stop_event(orig_opline,
JIT_G(blacklist_root_trace) / 2) != ZEND_JIT_TRACE_STOP_INNER_LOOP) {
stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
break;
}
}
}
if (opline == last_loop_opline
&& level == last_loop_level) {
idx = zend_jit_trace_subtrace(trace_buffer,
last_loop, idx, ZEND_JIT_TRACE_START_LOOP, op_array, opline);
start = ZEND_JIT_TRACE_START_LOOP;
stop = ZEND_JIT_TRACE_STOP_LOOP;
ret_level = 0;
break;
} else if (loop_unroll_limit < JIT_G(max_loop_unrolls)) {
last_loop = idx;
last_loop_opline = opline;
last_loop_level = level;
loop_unroll_limit++;
} else {
stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL;
break;
}
} else if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
TRACE_RECORD(ZEND_JIT_TRACE_VM, 0, opline);
stop = ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
break;
}
}
}
end_opline = opline;
if (!ZEND_JIT_TRACE_STOP_OK(stop)) {
if (backtrack_recursion > 0) {
idx = backtrack_recursion;
stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
end_opline = orig_opline;
} else if (backtrack_ret_recursion > 0) {
idx = backtrack_ret_recursion;
ret_level = backtrack_ret_recursion_level;
stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
end_opline = orig_opline;
} else if (backtrack_link_to_enter > 0) {
if (stop == ZEND_JIT_TRACE_STOP_DEEP_RECURSION
&& zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) / 2) ==
ZEND_JIT_TRACE_STOP_DEEP_RECURSION) {
idx = backtrack_link_to_enter;
stop = ZEND_JIT_TRACE_STOP_LINK;
end_opline = link_to_enter_opline;
}
}
}
if (stop == ZEND_JIT_TRACE_STOP_LINK) {
/* Shrink fake INIT_CALLs */
while (trace_buffer[idx-1].op == ZEND_JIT_TRACE_INIT_CALL
&& (trace_buffer[idx-1].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
idx--;
}
}
TRACE_END(ZEND_JIT_TRACE_END, stop, end_opline);
if (!halt) {
EX(opline) = opline;
}
#ifdef HAVE_GCC_GLOBAL_REGS
execute_data = save_execute_data;
opline = save_opline;
#endif
return stop | halt;
}
Reference in New Issue View Git Blame Copy Permalink